From 7a083b920d19570f7bcc4bac04dca09d21438817 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 30 Aug 2019 14:55:07 +0800 Subject: [PATCH 1/5] add black list and white list support for migrating repositories --- custom/conf/app.ini.sample | 8 +++ .../doc/advanced/config-cheat-sheet.en-us.md | 6 +++ .../doc/advanced/config-cheat-sheet.zh-cn.md | 6 +++ modules/matchlist/matchlist.go | 43 +++++++++++++++ modules/migrations/migrate.go | 52 +++++++++++++++++++ modules/migrations/migrate_test.go | 38 ++++++++++++++ modules/setting/migrate.go | 25 +++++++++ routers/init.go | 5 ++ vendor/modules.txt | 1 + 9 files changed, 184 insertions(+) create mode 100644 modules/matchlist/matchlist.go create mode 100644 modules/migrations/migrate_test.go create mode 100644 modules/setting/migrate.go diff --git a/custom/conf/app.ini.sample b/custom/conf/app.ini.sample index a2ac7248e770b..962bfa0f0dba7 100644 --- a/custom/conf/app.ini.sample +++ b/custom/conf/app.ini.sample @@ -803,3 +803,11 @@ IS_INPUT_FILE = false ENABLED = false ; If you want to add authorization, specify a token here TOKEN = + +[migration] +; Whitelist for migrating, default is blank. Blank means everything will be allowed. +; Multiple domains could be separated by commas. +WHITELISTED_DOMAINS = +; Blacklist for migrating, default is blank. Multiple domains could be separated by commas. +; When WHITELISTED_DOMAINS is not blank, this option will be ignored. +BLACKLISTED_DOMAINS = \ No newline at end of file diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md index 10c6110f373f2..628a1fb57e865 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md +++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md @@ -511,9 +511,15 @@ Two special environment variables are passed to the render command: - `GITEA_PREFIX_RAW`, which contains the current URL prefix in the `raw` path tree. To be used as prefix for image paths. ## Time (`time`) + - `FORMAT`: Time format to diplay on UI. i.e. RFC1123 or 2006-01-02 15:04:05 - `DEFAULT_UI_LOCATION`: Default location of time on the UI, so that we can display correct user's time on UI. i.e. Shanghai/Asia +## Migraions (`migration`) + +- `WHITELISTED_DOMAINS`: ****: Domains whitelist for migrating repositories, default is blank. It means everything will be allowed. Multiple domains could be separated by commas. +- `BLACKLISTED_DOMAINS`: ****: Domains blacklist for migrating repositories, default is blank. Multiple domains could be separated by commas. When `WHITELISTED_DOMAINS` is not blank, this option will be ignored. + ## Other (`other`) - `SHOW_FOOTER_BRANDING`: **false**: Show Gitea branding in the footer. diff --git a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md index b4b9e4e3a92a6..c9fba9fa07e31 100644 --- a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md +++ b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md @@ -240,9 +240,15 @@ IS_INPUT_FILE = false - IS_INPUT_FILE: 输入方式是最后一个参数为文件路径还是从标准输入读取。 ## Time (`time`) + - `FORMAT`: 显示在界面上的时间格式。比如： RFC1123 或者 2006-01-02 15:04:05 - `DEFAULT_UI_LOCATION`: 默认显示在界面上的时区，默认为本地时区。比如： Asia/Shanghai +## Migraions (`migration`) + +- `WHITELISTED_DOMAINS`: ****: 迁移仓库的域名白名单，默认为空，表示允许从任意域名迁移仓库，多个域名用逗号分隔。 +- `BLACKLISTED_DOMAINS`: ****: 迁移仓库的域名黑名单，默认为空，多个域名用逗号分隔。如果 `WHITELISTED_DOMAINS` 不为空，此选项将会被忽略。 + ## Other (`other`) - `SHOW_FOOTER_BRANDING`: 为真则在页面底部显示Gitea的字样。 diff --git a/modules/matchlist/matchlist.go b/modules/matchlist/matchlist.go new file mode 100644 index 0000000000000..b417e38099c99 --- /dev/null +++ b/modules/matchlist/matchlist.go @@ -0,0 +1,43 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package matchlist + +import ( + "github.com/gobwas/glob" +) + +// Matchlist represents a black or white list +type Matchlist struct { + rules []string + ruleGlobs []glob.Glob +} + +// NewMatchlist creates a new black or white list +func NewMatchlist(rules ...string) (*Matchlist, error) { + list := Matchlist{ + rules: rules, + ruleGlobs: make([]glob.Glob, 0, len(rules)), + } + + for _, rule := range list.rules { + rg, err := glob.Compile(rule) + if err != nil { + return nil, err + } + list.ruleGlobs = append(list.ruleGlobs, rg) + } + + return &list, nil +} + +// Match will matches +func (b *Matchlist) Match(u string) bool { + for _, r := range b.ruleGlobs { + if r.Match(u) { + return true + } + } + return false +} \ No newline at end of file diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go index 27782cb94034e..5fe1c027eec43 100644 --- a/modules/migrations/migrate.go +++ b/modules/migrations/migrate.go @@ -6,9 +6,15 @@ package migrations import ( + "fmt" + "net/url" + "strings" + "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/matchlist" "code.gitea.io/gitea/modules/migrations/base" + "code.gitea.io/gitea/modules/setting" ) // MigrateOptions is equal to base.MigrateOptions @@ -23,8 +29,34 @@ func RegisterDownloaderFactory(factory base.DownloaderFactory) { factories = append(factories, factory) } +func isMigrateURLAllowed(remoteURL string) (bool, error) { + u, err := url.Parse(remoteURL) + if err != nil { + return false, err + } + + if strings.EqualFold(u.Scheme, "http") || strings.EqualFold(u.Scheme, "https") { + if len(setting.Migration.WhitelistedDomains) > 0 { + if !whitelist.Match(u.Host) { + return false, fmt.Errorf("Migrate from %v is not allowed", u.Host) + } + } else { + if blacklist.Match(u.Host) { + return false, fmt.Errorf("Migrate from %v is not allowed", u.Host) + } + } + } + + return true, nil +} + // MigrateRepository migrate repository according MigrateOptions func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOptions) (*models.Repository, error) { + allowed, err := isMigrateURLAllowed(opts.RemoteURL) + if !allowed { + return nil, err + } + var ( downloader base.Downloader uploader = NewGiteaLocalUploader(doer, ownerName, opts.Name) @@ -250,3 +282,23 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts return nil } + +var ( + whitelist *matchlist.Matchlist + blacklist *matchlist.Matchlist +) + +// Init migrations service +func Init() error { + var err error + whitelist, err = matchlist.NewMatchlist(setting.Migration.WhitelistedDomains...) + if err != nil { + return fmt.Errorf("Init migration whitelist domains failed: %v", err) + } + + blacklist, err = matchlist.NewMatchlist(setting.Migration.BlacklistedDomains...) + if err != nil { + return fmt.Errorf("Init migration blacklist domains failed: %v", err) + } + return nil +} \ No newline at end of file diff --git a/modules/migrations/migrate_test.go b/modules/migrations/migrate_test.go new file mode 100644 index 0000000000000..275f48d1a949c --- /dev/null +++ b/modules/migrations/migrate_test.go @@ -0,0 +1,38 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "testing" + + "code.gitea.io/gitea/modules/setting" + + "github.com/stretchr/testify/assert" +) + +func TestMigrateWhiteBlacklist(t *testing.T) { + setting.Migration.WhitelistedDomains = []string{"github.com"} + assert.NoError(t, Init()) + + allowed, err := isMigrateURLAllowed("https://gitlab.com/gitlab/gitlab.git") + assert.False(t, allowed) + assert.Error(t, err) + + allowed, err = isMigrateURLAllowed("https://github.com/go-gitea/gitea.git") + assert.True(t, allowed) + assert.NoError(t, err) + + setting.Migration.WhitelistedDomains = []string{} + setting.Migration.BlacklistedDomains = []string{"github.com"} + assert.NoError(t, Init()) + + allowed, err = isMigrateURLAllowed("https://gitlab.com/gitlab/gitlab.git") + assert.True(t, allowed) + assert.NoError(t, err) + + allowed, err = isMigrateURLAllowed("https://github.com/go-gitea/gitea.git") + assert.False(t, allowed) + assert.Error(t, err) +} \ No newline at end of file diff --git a/modules/setting/migrate.go b/modules/setting/migrate.go new file mode 100644 index 0000000000000..eb634ef23fdce --- /dev/null +++ b/modules/setting/migrate.go @@ -0,0 +1,25 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package setting + +import ( + "fmt" +) + +var Migration = struct { + WhitelistedDomains []string + BlacklistedDomains []string +} { + WhitelistedDomains: []string{}, + BlacklistedDomains: []string{}, +} + +// InitMigrationConfig represents load migration configurations +func InitMigrationConfig() error { + if err := Cfg.Section("migration").MapTo(&Migration); err != nil { + return fmt.Errorf("Failed to map Migration settings: %v", err) + } + return nil +} \ No newline at end of file diff --git a/routers/init.go b/routers/init.go index fdf90904ce759..b641de8560394 100644 --- a/routers/init.go +++ b/routers/init.go @@ -19,6 +19,7 @@ import ( "code.gitea.io/gitea/modules/mailer" "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/markup/external" + repo_migrations "code.gitea.io/gitea/modules/migrations" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/ssh" @@ -101,6 +102,10 @@ func GlobalInit() { models.InitSyncMirrors() models.InitDeliverHooks() models.InitTestPullRequests() + + if err := repo_migrations.Init(); err != nil { + log.Fatal("Failed to initialize migrations: %v", err) + } } if setting.EnableSQLite3 { log.Info("SQLite3 Supported") diff --git a/vendor/modules.txt b/vendor/modules.txt index 27dc32d705be2..498377dd67749 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -524,6 +524,7 @@ golang.org/x/tools/go/internal/gcimporter golang.org/x/tools/internal/fastwalk golang.org/x/tools/internal/module # google.golang.org/appengine v1.6.2 +# google.golang.org/appengine v1.6.1 google.golang.org/appengine/cloudsql google.golang.org/appengine/urlfetch google.golang.org/appengine/internal From 3f2b34ba86e4ba4f767e5369b837e38d3ebd7fb0 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 30 Aug 2019 15:11:27 +0800 Subject: [PATCH 2/5] fix fmt --- modules/matchlist/matchlist.go | 6 +++--- modules/migrations/migrate.go | 4 ++-- modules/migrations/migrate_test.go | 2 +- modules/setting/migrate.go | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/modules/matchlist/matchlist.go b/modules/matchlist/matchlist.go index b417e38099c99..656d8fc99174c 100644 --- a/modules/matchlist/matchlist.go +++ b/modules/matchlist/matchlist.go @@ -10,14 +10,14 @@ import ( // Matchlist represents a black or white list type Matchlist struct { - rules []string + rules []string ruleGlobs []glob.Glob } // NewMatchlist creates a new black or white list func NewMatchlist(rules ...string) (*Matchlist, error) { list := Matchlist{ - rules: rules, + rules: rules, ruleGlobs: make([]glob.Glob, 0, len(rules)), } @@ -40,4 +40,4 @@ func (b *Matchlist) Match(u string) bool { } } return false -} \ No newline at end of file +} diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go index 5fe1c027eec43..67f3e308d27d3 100644 --- a/modules/migrations/migrate.go +++ b/modules/migrations/migrate.go @@ -56,7 +56,7 @@ func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOpt if !allowed { return nil, err } - + var ( downloader base.Downloader uploader = NewGiteaLocalUploader(doer, ownerName, opts.Name) @@ -301,4 +301,4 @@ func Init() error { return fmt.Errorf("Init migration blacklist domains failed: %v", err) } return nil -} \ No newline at end of file +} diff --git a/modules/migrations/migrate_test.go b/modules/migrations/migrate_test.go index 275f48d1a949c..534625018913b 100644 --- a/modules/migrations/migrate_test.go +++ b/modules/migrations/migrate_test.go @@ -35,4 +35,4 @@ func TestMigrateWhiteBlacklist(t *testing.T) { allowed, err = isMigrateURLAllowed("https://github.com/go-gitea/gitea.git") assert.False(t, allowed) assert.Error(t, err) -} \ No newline at end of file +} diff --git a/modules/setting/migrate.go b/modules/setting/migrate.go index eb634ef23fdce..d26683692fe11 100644 --- a/modules/setting/migrate.go +++ b/modules/setting/migrate.go @@ -11,7 +11,7 @@ import ( var Migration = struct { WhitelistedDomains []string BlacklistedDomains []string -} { +}{ WhitelistedDomains: []string{}, BlacklistedDomains: []string{}, } @@ -22,4 +22,4 @@ func InitMigrationConfig() error { return fmt.Errorf("Failed to map Migration settings: %v", err) } return nil -} \ No newline at end of file +} From ad5a226cce1da48b2d5df22e51b6391f5e07dc93 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 30 Aug 2019 15:25:10 +0800 Subject: [PATCH 3/5] fix lint --- modules/setting/migrate.go | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/setting/migrate.go b/modules/setting/migrate.go index d26683692fe11..657bb969f590b 100644 --- a/modules/setting/migrate.go +++ b/modules/setting/migrate.go @@ -8,6 +8,7 @@ import ( "fmt" ) +// Migration represents migrations' settings var Migration = struct { WhitelistedDomains []string BlacklistedDomains []string From 7280964e5f14fed5f43effffec2d1339a9ddc4de Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 30 Aug 2019 15:43:37 +0800 Subject: [PATCH 4/5] fix vendor --- vendor/modules.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/vendor/modules.txt b/vendor/modules.txt index 498377dd67749..ab99c226856cc 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -444,6 +444,7 @@ go.mongodb.org/mongo-driver/bson/bsoncodec go.mongodb.org/mongo-driver/bson/bsonrw go.mongodb.org/mongo-driver/x/bsonx/bsoncore # golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472 +# golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586 golang.org/x/crypto/acme/autocert golang.org/x/crypto/argon2 golang.org/x/crypto/bcrypt From cdff51c8946a9f6b06316c871d769b6cd4fef114 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 7 Sep 2019 11:41:58 +0800 Subject: [PATCH 5/5] fix modules.txt --- vendor/modules.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/vendor/modules.txt b/vendor/modules.txt index ab99c226856cc..27dc32d705be2 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -444,7 +444,6 @@ go.mongodb.org/mongo-driver/bson/bsoncodec go.mongodb.org/mongo-driver/bson/bsonrw go.mongodb.org/mongo-driver/x/bsonx/bsoncore # golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472 -# golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586 golang.org/x/crypto/acme/autocert golang.org/x/crypto/argon2 golang.org/x/crypto/bcrypt @@ -525,7 +524,6 @@ golang.org/x/tools/go/internal/gcimporter golang.org/x/tools/internal/fastwalk golang.org/x/tools/internal/module # google.golang.org/appengine v1.6.2 -# google.golang.org/appengine v1.6.1 google.golang.org/appengine/cloudsql google.golang.org/appengine/urlfetch google.golang.org/appengine/internal