Skip to content

Commit dd08853

Browse files
lunnylunny
authored
Dump github/gitlab/gitea repository data to a local directory and restore to gitea (#12244)
* Dump github/gitlab repository data to a local directory * Fix lint * Adjust directory structure * Allow migration special units * Allow migration ignore release assets * Fix lint * Add restore repository * stage the changes * Merge * Fix lint * Update the interface * Add some restore methods * Finish restore * Add comments * Fix restore * Add a token flag * Fix bug * Fix test * Fix test * Fix bug * Fix bug * Fix lint * Fix restore * refactor downloader * fmt * Fix bug isEnd detection on getIssues * Refactor maxPerPage * Remove unused codes * Remove unused codes * Fix bug * Fix restore * Fix dump * Uploader should not depend downloader * use release attachment name but not id * Fix restore bug * Fix lint * Fix restore bug * Add a method of DownloadFunc for base.Release to make uploader not depend on downloader * fix Release yml marshal * Fix trace information * Fix bug when dump & restore * Save relative path on yml file * Fix bug * Use relative path * Update docs * Use git service string but not int * Recognize clone addr to service type
1 parent 212fa34 commit dd08853

29 files changed

+1484
-225
lines changed

‎cmd/dump_repo.go

Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
// Copyright 2020 The Gitea Authors. All rights reserved.
2+
// Use of this source code is governed by a MIT-style
3+
// license that can be found in the LICENSE file.
4+
5+
package cmd
6+
7+
import (
8+
"context"
9+
"errors"
10+
"strings"
11+
12+
"code.gitea.io/gitea/modules/convert"
13+
"code.gitea.io/gitea/modules/log"
14+
"code.gitea.io/gitea/modules/migrations"
15+
"code.gitea.io/gitea/modules/migrations/base"
16+
"code.gitea.io/gitea/modules/setting"
17+
"code.gitea.io/gitea/modules/structs"
18+
19+
"github.com/urfave/cli"
20+
)
21+
22+
// CmdDumpRepository represents the available dump repository sub-command.
23+
var CmdDumpRepository = cli.Command{
24+
Name: "dump-repo",
25+
Usage: "Dump the repository from git/github/gitea/gitlab",
26+
Description: "This is a command for dumping the repository data.",
27+
Action: runDumpRepository,
28+
Flags: []cli.Flag{
29+
cli.StringFlag{
30+
Name: "git_service",
31+
Value: "",
32+
Usage: "Git service, git, github, gitea, gitlab. If clone_addr could be recognized, this could be ignored.",
33+
},
34+
cli.StringFlag{
35+
Name: "repo_dir, r",
36+
Value: "./data",
37+
Usage: "Repository dir path to store the data",
38+
},
39+
cli.StringFlag{
40+
Name: "clone_addr",
41+
Value: "",
42+
Usage: "The URL will be clone, currently could be a git/github/gitea/gitlab http/https URL",
43+
},
44+
cli.StringFlag{
45+
Name: "auth_username",
46+
Value: "",
47+
Usage: "The username to visit the clone_addr",
48+
},
49+
cli.StringFlag{
50+
Name: "auth_password",
51+
Value: "",
52+
Usage: "The password to visit the clone_addr",
53+
},
54+
cli.StringFlag{
55+
Name: "auth_token",
56+
Value: "",
57+
Usage: "The personal token to visit the clone_addr",
58+
},
59+
cli.StringFlag{
60+
Name: "owner_name",
61+
Value: "",
62+
Usage: "The data will be stored on a directory with owner name if not empty",
63+
},
64+
cli.StringFlag{
65+
Name: "repo_name",
66+
Value: "",
67+
Usage: "The data will be stored on a directory with repository name if not empty",
68+
},
69+
cli.StringFlag{
70+
Name: "units",
71+
Value: "",
72+
Usage: `Which items will be migrated, one or more units should be separated as comma.
73+
wiki, issues, labels, releases, release_assets, milestones, pull_requests, comments are allowed. Empty means all units.`,
74+
},
75+
},
76+
}
77+
78+
func runDumpRepository(ctx *cli.Context) error {
79+
if err := initDB(); err != nil {
80+
return err
81+
}
82+
83+
log.Trace("AppPath: %s", setting.AppPath)
84+
log.Trace("AppWorkPath: %s", setting.AppWorkPath)
85+
log.Trace("Custom path: %s", setting.CustomPath)
86+
log.Trace("Log path: %s", setting.LogRootPath)
87+
setting.InitDBConfig()
88+
89+
var (
90+
serviceType structs.GitServiceType
91+
cloneAddr = ctx.String("clone_addr")
92+
serviceStr = ctx.String("git_service")
93+
)
94+
95+
if strings.HasPrefix(strings.ToLower(cloneAddr), "https://github.com/") {
96+
serviceStr = "github"
97+
} else if strings.HasPrefix(strings.ToLower(cloneAddr), "https://gitlab.com/") {
98+
serviceStr = "gitlab"
99+
} else if strings.HasPrefix(strings.ToLower(cloneAddr), "https://gitea.com/") {
100+
serviceStr = "gitea"
101+
}
102+
if serviceStr == "" {
103+
return errors.New("git_service missed or clone_addr cannot be recognized")
104+
}
105+
serviceType = convert.ToGitServiceType(serviceStr)
106+
107+
var opts = base.MigrateOptions{
108+
GitServiceType: serviceType,
109+
CloneAddr: cloneAddr,
110+
AuthUsername: ctx.String("auth_username"),
111+
AuthPassword: ctx.String("auth_password"),
112+
AuthToken: ctx.String("auth_token"),
113+
RepoName: ctx.String("repo_name"),
114+
}
115+
116+
if len(ctx.String("units")) == 0 {
117+
opts.Wiki = true
118+
opts.Issues = true
119+
opts.Milestones = true
120+
opts.Labels = true
121+
opts.Releases = true
122+
opts.Comments = true
123+
opts.PullRequests = true
124+
opts.ReleaseAssets = true
125+
} else {
126+
units := strings.Split(ctx.String("units"), ",")
127+
for _, unit := range units {
128+
switch strings.ToLower(unit) {
129+
case "wiki":
130+
opts.Wiki = true
131+
case "issues":
132+
opts.Issues = true
133+
case "milestones":
134+
opts.Milestones = true
135+
case "labels":
136+
opts.Labels = true
137+
case "releases":
138+
opts.Releases = true
139+
case "release_assets":
140+
opts.ReleaseAssets = true
141+
case "comments":
142+
opts.Comments = true
143+
case "pull_requests":
144+
opts.PullRequests = true
145+
}
146+
}
147+
}
148+
149+
if err := migrations.DumpRepository(
150+
context.Background(),
151+
ctx.String("repo_dir"),
152+
ctx.String("owner_name"),
153+
opts,
154+
); err != nil {
155+
log.Fatal("Failed to dump repository: %v", err)
156+
return err
157+
}
158+
159+
log.Trace("Dump finished!!!")
160+
161+
return nil
162+
}

‎cmd/restore_repo.go

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
// Copyright 2020 The Gitea Authors. All rights reserved.
2+
// Use of this source code is governed by a MIT-style
3+
// license that can be found in the LICENSE file.
4+
5+
package cmd
6+
7+
import (
8+
"context"
9+
"strings"
10+
11+
"code.gitea.io/gitea/modules/log"
12+
"code.gitea.io/gitea/modules/migrations"
13+
"code.gitea.io/gitea/modules/migrations/base"
14+
"code.gitea.io/gitea/modules/setting"
15+
"code.gitea.io/gitea/modules/storage"
16+
pull_service "code.gitea.io/gitea/services/pull"
17+
18+
"github.com/urfave/cli"
19+
)
20+
21+
// CmdRestoreRepository represents the available restore a repository sub-command.
22+
var CmdRestoreRepository = cli.Command{
23+
Name: "restore-repo",
24+
Usage: "Restore the repository from disk",
25+
Description: "This is a command for restoring the repository data.",
26+
Action: runRestoreRepository,
27+
Flags: []cli.Flag{
28+
cli.StringFlag{
29+
Name: "repo_dir, r",
30+
Value: "./data",
31+
Usage: "Repository dir path to restore from",
32+
},
33+
cli.StringFlag{
34+
Name: "owner_name",
35+
Value: "",
36+
Usage: "Restore destination owner name",
37+
},
38+
cli.StringFlag{
39+
Name: "repo_name",
40+
Value: "",
41+
Usage: "Restore destination repository name",
42+
},
43+
cli.StringFlag{
44+
Name: "units",
45+
Value: "",
46+
Usage: `Which items will be restored, one or more units should be separated as comma.
47+
wiki, issues, labels, releases, release_assets, milestones, pull_requests, comments are allowed. Empty means all units.`,
48+
},
49+
},
50+
}
51+
52+
func runRestoreRepository(ctx *cli.Context) error {
53+
if err := initDB(); err != nil {
54+
return err
55+
}
56+
57+
log.Trace("AppPath: %s", setting.AppPath)
58+
log.Trace("AppWorkPath: %s", setting.AppWorkPath)
59+
log.Trace("Custom path: %s", setting.CustomPath)
60+
log.Trace("Log path: %s", setting.LogRootPath)
61+
setting.InitDBConfig()
62+
63+
if err := storage.Init(); err != nil {
64+
return err
65+
}
66+
67+
if err := pull_service.Init(); err != nil {
68+
return err
69+
}
70+
71+
var opts = base.MigrateOptions{
72+
RepoName: ctx.String("repo_name"),
73+
}
74+
75+
if len(ctx.String("units")) == 0 {
76+
opts.Wiki = true
77+
opts.Issues = true
78+
opts.Milestones = true
79+
opts.Labels = true
80+
opts.Releases = true
81+
opts.Comments = true
82+
opts.PullRequests = true
83+
opts.ReleaseAssets = true
84+
} else {
85+
units := strings.Split(ctx.String("units"), ",")
86+
for _, unit := range units {
87+
switch strings.ToLower(unit) {
88+
case "wiki":
89+
opts.Wiki = true
90+
case "issues":
91+
opts.Issues = true
92+
case "milestones":
93+
opts.Milestones = true
94+
case "labels":
95+
opts.Labels = true
96+
case "releases":
97+
opts.Releases = true
98+
case "release_assets":
99+
opts.ReleaseAssets = true
100+
case "comments":
101+
opts.Comments = true
102+
case "pull_requests":
103+
opts.PullRequests = true
104+
}
105+
}
106+
}
107+
108+
if err := migrations.RestoreRepository(
109+
context.Background(),
110+
ctx.String("repo_dir"),
111+
ctx.String("owner_name"),
112+
ctx.String("repo_name"),
113+
); err != nil {
114+
log.Fatal("Failed to restore repository: %v", err)
115+
return err
116+
}
117+
118+
return nil
119+
}

‎docs/content/doc/usage/command-line.en-us.md

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -441,3 +441,28 @@ Manage running server operations:
441441
- `--host value`, `-H value`: Mail server host (defaults to: 127.0.0.1:25)
442442
- `--send-to value`, `-s value`: Email address(es) to send to
443443
- `--subject value`, `-S value`: Subject header of sent emails
444+
445+
### dump-repo
446+
447+
Dump-repo dumps repository data from git/github/gitea/gitlab:
448+
449+
- Options:
450+
- `--git_service service` : Git service, it could be `git`, `github`, `gitea`, `gitlab`, If clone_addr could be recognized, this could be ignored.
451+
- `--repo_dir dir`, `-r dir`: Repository dir path to store the data
452+
- `--clone_addr addr`: The URL will be clone, currently could be a git/github/gitea/gitlab http/https URL. i.e. https://github.com/lunny/tango.git
453+
- `--auth_username lunny`: The username to visit the clone_addr
454+
- `--auth_password <password>`: The password to visit the clone_addr
455+
- `--auth_token <token>`: The personal token to visit the clone_addr
456+
- `--owner_name lunny`: The data will be stored on a directory with owner name if not empty
457+
- `--repo_name tango`: The data will be stored on a directory with repository name if not empty
458+
- `--units <units>`: Which items will be migrated, one or more units should be separated as comma. wiki, issues, labels, releases, release_assets, milestones, pull_requests, comments are allowed. Empty means all units.
459+
460+
### restore-repo
461+
462+
Restore-repo restore repository data from disk dir:
463+
464+
- Options:
465+
- `--repo_dir dir`, `-r dir`: Repository dir path to restore from
466+
- `--owner_name lunny`: Restore destination owner name
467+
- `--repo_name tango`: Restore destination repository name
468+
- `--units <units>`: Which items will be restored, one or more units should be separated as comma. wiki, issues, labels, releases, release_assets, milestones, pull_requests, comments are allowed. Empty means all units.

‎main.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,8 @@ arguments - which can alternatively be run by running the subcommand web.`
7272
cmd.Cmdembedded,
7373
cmd.CmdMigrateStorage,
7474
cmd.CmdDocs,
75+
cmd.CmdDumpRepository,
76+
cmd.CmdRestoreRepository,
7577
}
7678
// Now adjust these commands to add our global configuration options
7779

‎models/admin.go

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,3 +132,16 @@ func DeleteNoticesByIDs(ids []int64) error {
132132
Delete(new(Notice))
133133
return err
134134
}
135+
136+
// GetAdminUser returns the first administrator
137+
func GetAdminUser() (*User, error) {
138+
var admin User
139+
has, err := x.Where("is_admin=?", true).Get(&admin)
140+
if err != nil {
141+
return nil, err
142+
} else if !has {
143+
return nil, ErrUserNotExist{}
144+
}
145+
146+
return &admin, nil
147+
}

‎models/task.go

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -211,10 +211,6 @@ func FinishMigrateTask(task *Task) error {
211211
if _, err := sess.ID(task.ID).Cols("status", "end_time").Update(task); err != nil {
212212
return err
213213
}
214-
task.Repo.Status = RepositoryReady
215-
if _, err := sess.ID(task.RepoID).Cols("status").Update(task.Repo); err != nil {
216-
return err
217-
}
218214

219215
return sess.Commit()
220216
}

‎modules/migrations/base/comment.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,10 @@ import "time"
99

1010
// Comment is a standard comment information
1111
type Comment struct {
12-
IssueIndex int64
13-
PosterID int64
14-
PosterName string
15-
PosterEmail string
12+
IssueIndex int64 `yaml:"issue_index"`
13+
PosterID int64 `yaml:"poster_id"`
14+
PosterName string `yaml:"poster_name"`
15+
PosterEmail string `yaml:"poster_email"`
1616
Created time.Time
1717
Updated time.Time
1818
Content string

‎modules/migrations/base/downloader.go

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,20 +7,13 @@ package base
77

88
import (
99
"context"
10-
"io"
1110
"time"
1211

1312
"code.gitea.io/gitea/modules/structs"
1413
)
1514

16-
// AssetDownloader downloads an asset (attachment) for a release
17-
type AssetDownloader interface {
18-
GetAsset(relTag string, relID, id int64) (io.ReadCloser, error)
19-
}
20-
2115
// Downloader downloads the site repo informations
2216
type Downloader interface {
23-
AssetDownloader
2417
SetContext(context.Context)
2518
GetRepoInfo() (*Repository, error)
2619
GetTopics() ([]string, error)

0 commit comments

Comments
 (0)