Gitea 2 Gitea migration (#12657)

* first draft

* update gitea sdk to 9e280adb4da

* adapt feat of updated sdk

* releases now works

* break the Reactions loop

* use convertGiteaLabel

* fix endless loop because paggination is not supported there !!!

* rename gitea local uploader files

* pagination can bite you in the ass

* Version Checks

* lint

* docs

* rename gitea sdk import to miss future conficts

* go-swagger: dont scan the sdk structs

* make sure gitea can shutdown gracefully

* make GetPullRequests and GetIssues similar

* rm useles

* Add Test: started ...

* ... add tests ...

* Add tests and Fixing things

* Workaround missing SHA

* Adapt: Ensure that all migration requests are cancellable
(714ab71)

* LINT: fix misspells in test set

* adapt ListMergeRequestAwardEmoji

* update sdk

* Return error when creating giteadownloader failed

* update sdk

* adapt new sdk

* adopt new features

* check version before err

* adapt: 'migrate service type switch page'

* optimize

* Fix DefaultBranch

* impruve

* handle subPath

* fix test

* Fix ReviewCommentPosition

* test GetReviews

* add DefaultBranch int test set

* rm unused

* Update SDK to v0.13.0

* addopt sdk changes

* found better link

* format template

* Update Docs

* Update Gitea SDK (v0.13.1)

Author: 6543

Makefile

@@ -120,11 +120,12 @@ endif
 
 GO_SOURCES_OWN := $(filter-out vendor/% %/bindata.go, $(GO_SOURCES))
 
-#To update swagger use: GO111MODULE=on go get -u github.com/go-swagger/go-swagger/cmd/swagger@v0.20.1
+#To update swagger use: GO111MODULE=on go get -u github.com/go-swagger/go-swagger/cmd/swagger
 SWAGGER := $(GO) run -mod=vendor github.com/go-swagger/go-swagger/cmd/swagger
 SWAGGER_SPEC := templates/swagger/v1_json.tmpl
 SWAGGER_SPEC_S_TMPL := s|"basePath": *"/api/v1"|"basePath": "{{AppSubUrl}}/api/v1"|g
 SWAGGER_SPEC_S_JSON := s|"basePath": *"{{AppSubUrl}}/api/v1"|"basePath": "/api/v1"|g
+SWAGGER_EXCLUDE := code.gitea.io/sdk
 SWAGGER_NEWLINE_COMMAND := -e '$$a\'
 
 TEST_MYSQL_HOST ?= mysql:3306
@@ -243,7 +244,7 @@ endif
 
 .PHONY: generate-swagger
 generate-swagger:
-	$(SWAGGER) generate spec -o './$(SWAGGER_SPEC)'
+	$(SWAGGER) generate spec -x "$(SWAGGER_EXCLUDE)" -o './$(SWAGGER_SPEC)'
 	$(SED_INPLACE) '$(SWAGGER_SPEC_S_TMPL)' './$(SWAGGER_SPEC)'
 	$(SED_INPLACE) $(SWAGGER_NEWLINE_COMMAND) './$(SWAGGER_SPEC)'
 

docs/content/doc/advanced/migrations.en-us.md

@@ -17,10 +17,10 @@ menu:
 
 The new migration features were introduced in Gitea 1.9.0. It defines two interfaces to support migrating
 repositories data from other git host platforms to gitea or, in the future migrating gitea data to other
-git host platforms. Currently, only the migrations from github via APIv3 to Gitea is implemented.
+git host platforms. Currently, migrations from Github, Gitlab and Gitea to Gitea is implemented.
 
 First of all, Gitea defines some standard objects in packages `modules/migrations/base`. They are
- `Repository`, `Milestone`, `Release`, `Label`, `Issue`, `Comment`, `PullRequest`, `Reaction`, `Review`, `ReviewComment`.
+ `Repository`, `Milestone`, `Release`, `ReleaseAsset`, `Label`, `Issue`, `Comment`, `PullRequest`, `Reaction`, `Review`, `ReviewComment`.
 
 ## Downloader Interfaces
 
@@ -33,6 +33,7 @@ create a Downloader.
 
 ```Go
 type Downloader interface {
+	GetAsset(relTag string, relID, id int64) (io.ReadCloser, error)
 	SetContext(context.Context)
 	GetRepoInfo() (*Repository, error)
 	GetTopics() ([]string, error)
@@ -41,15 +42,15 @@ type Downloader interface {
 	GetLabels() ([]*Label, error)
 	GetIssues(page, perPage int) ([]*Issue, bool, error)
 	GetComments(issueNumber int64) ([]*Comment, error)
-	GetPullRequests(page, perPage int) ([]*PullRequest, error)
+	GetPullRequests(page, perPage int) ([]*PullRequest, bool, error)
 	GetReviews(pullRequestNumber int64) ([]*Review, error)
 }
 ```
 
 ```Go
 type DownloaderFactory interface {
-	Match(opts MigrateOptions) (bool, error)
-	New(opts MigrateOptions) (Downloader, error)
+	New(ctx context.Context, opts MigrateOptions) (Downloader, error)
+	GitServiceType() structs.GitServiceType
 }
 ```
 
@@ -66,7 +67,7 @@ type Uploader interface {
 	CreateRepo(repo *Repository, opts MigrateOptions) error
 	CreateTopics(topic ...string) error
 	CreateMilestones(milestones ...*Milestone) error
-	CreateReleases(releases ...*Release) error
+	CreateReleases(downloader Downloader, releases ...*Release) error
 	SyncTags() error
 	CreateLabels(labels ...*Label) error
 	CreateIssues(issues ...*Issue) error

go.mod

@@ -4,6 +4,7 @@ go 1.14
 
 require (
 	code.gitea.io/gitea-vet v0.2.1
+	code.gitea.io/sdk/gitea v0.13.1
 	gitea.com/lunny/levelqueue v0.3.0
 	gitea.com/macaron/binding v0.0.0-20190822013154-a5f53841ed2b
 	gitea.com/macaron/cache v0.0.0-20190822004001-a6e7fee4ee76
@@ -49,7 +50,7 @@ require (
 	github.com/google/uuid v1.1.1
 	github.com/gorilla/context v1.1.1
 	github.com/hashicorp/go-retryablehttp v0.6.7 // indirect
-	github.com/hashicorp/go-version v0.0.0-00010101000000-000000000000
+	github.com/hashicorp/go-version v1.2.1
 	github.com/huandu/xstrings v1.3.0
 	github.com/issue9/assert v1.3.2 // indirect
 	github.com/issue9/identicon v1.0.1

go.sum

@@ -15,7 +15,7 @@ import (
 
 // AssetDownloader downloads an asset (attachment) for a release
 type AssetDownloader interface {
-	GetAsset(tag string, id int64) (io.ReadCloser, error)
+	GetAsset(relTag string, relID, id int64) (io.ReadCloser, error)
 }
 
 // Downloader downloads the site repo informations
@@ -29,7 +29,7 @@ type Downloader interface {
 	GetLabels() ([]*Label, error)
 	GetIssues(page, perPage int) ([]*Issue, bool, error)
 	GetComments(issueNumber int64) ([]*Comment, error)
-	GetPullRequests(page, perPage int) ([]*PullRequest, error)
+	GetPullRequests(page, perPage int) ([]*PullRequest, bool, error)
 	GetReviews(pullRequestNumber int64) ([]*Review, error)
 }
 
@@ -209,23 +209,24 @@ func (d *RetryDownloader) GetComments(issueNumber int64) ([]*Comment, error) {
 }
 
 // GetPullRequests returns a repository's pull requests with retry
-func (d *RetryDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, error) {
+func (d *RetryDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bool, error) {
 	var (
 		times = d.RetryTimes
 		prs   []*PullRequest
 		err   error
+		isEnd bool
 	)
 	for ; times > 0; times-- {
-		if prs, err = d.Downloader.GetPullRequests(page, perPage); err == nil {
-			return prs, nil
+		if prs, isEnd, err = d.Downloader.GetPullRequests(page, perPage); err == nil {
+			return prs, isEnd, nil
 		}
 		select {
 		case <-d.ctx.Done():
-			return nil, d.ctx.Err()
+			return nil, false, d.ctx.Err()
 		case <-time.After(time.Second * time.Duration(d.RetryDelay)):
 		}
 	}
-	return nil, err
+	return nil, false, err
 }
 
 // GetReviews returns pull requests reviews

modules/migrations/base/downloader.go

@@ -23,4 +23,5 @@ type Issue struct {
 	Closed      *time.Time
 	Labels      []*Label
 	Reactions   []*Reaction
+	Assignees   []string
 }

modules/migrations/base/issue.go

@@ -31,7 +31,6 @@ type PullRequest struct {
 	MergeCommitSHA string
 	Head           PullRequestBranch
 	Base           PullRequestBranch
-	Assignee       string
 	Assignees      []string
 	IsLocked       bool
 	Reactions      []*Reaction

modules/migrations/base/pullrequest.go

@@ -15,6 +15,7 @@ type ReleaseAsset struct {
 	DownloadCount *int
 	Created       time.Time
 	Updated       time.Time
+	DownloadURL   *string
 }
 
 // Release represents a release

modules/migrations/base/release.go

@@ -12,8 +12,6 @@ type Repository struct {
 	IsPrivate     bool
 	IsMirror      bool
 	Description   string
-	AuthUsername  string
-	AuthPassword  string
 	CloneURL      string
 	OriginalURL   string
 	DefaultBranch string

modules/migrations/base/repo.go

@@ -36,6 +36,7 @@ type ReviewComment struct {
 	TreePath  string
 	DiffHunk  string
 	Position  int
+	Line      int
 	CommitID  string
 	PosterID  int64
 	Reactions []*Reaction

modules/migrations/base/review.go

@@ -66,7 +66,7 @@ func (g *PlainGitDownloader) GetReleases() ([]*base.Release, error) {
 }
 
 // GetAsset returns an asset
-func (g *PlainGitDownloader) GetAsset(_ string, _ int64) (io.ReadCloser, error) {
+func (g *PlainGitDownloader) GetAsset(_ string, _, _ int64) (io.ReadCloser, error) {
 	return nil, ErrNotSupported
 }
 
@@ -81,8 +81,8 @@ func (g *PlainGitDownloader) GetComments(issueNumber int64) ([]*base.Comment, er
 }
 
 // GetPullRequests returns pull requests according page and perPage
-func (g *PlainGitDownloader) GetPullRequests(start, limit int) ([]*base.PullRequest, error) {
-	return nil, ErrNotSupported
+func (g *PlainGitDownloader) GetPullRequests(start, limit int) ([]*base.PullRequest, bool, error) {
+	return nil, false, ErrNotSupported
 }
 
 // GetReviews returns reviews according issue number