Skip to content

Commit 36582e7

Browse files
bl00mberbl00mber
committed
Add LFS data mirroring
1 parent 0a9a484 commit 36582e7

File tree

14 files changed

+136
-25
lines changed

14 files changed

+136
-25
lines changed

‎modules/forms/repo_form.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@ type MigrateRepoForm struct {
7373
// required: true
7474
RepoName string `json:"repo_name" binding:"Required;AlphaDashDot;MaxSize(100)"`
7575
Mirror bool `json:"mirror"`
76+
LFS bool `json:"lfs"`
7677
Private bool `json:"private"`
7778
Description string `json:"description" binding:"MaxSize(255)"`
7879
Wiki bool `json:"wiki"`

‎modules/migrations/base/options.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ type MigrateOptions struct {
2020
// required: true
2121
RepoName string `json:"repo_name" binding:"Required"`
2222
Mirror bool `json:"mirror"`
23+
LFS bool `json:"lfs"`
2324
Private bool `json:"private"`
2425
Description string `json:"description"`
2526
OriginalURL string

‎modules/migrations/gitea_uploader.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,7 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate
117117
OriginalURL: repo.OriginalURL,
118118
GitServiceType: opts.GitServiceType,
119119
Mirror: repo.IsMirror,
120+
LFS: opts.LFS,
120121
CloneAddr: repo.CloneURL,
121122
Private: repo.IsPrivate,
122123
Wiki: opts.Wiki,

‎modules/repository/repo.go

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,9 @@ package repository
77
import (
88
"context"
99
"fmt"
10+
"os"
1011
"path"
12+
"path/filepath"
1113
"strings"
1214
"time"
1315

@@ -70,6 +72,75 @@ func MigrateRepositoryGitData(ctx context.Context, u *models.User, repo *models.
7072
return repo, fmt.Errorf("Clone: %v", err)
7173
}
7274

75+
if opts.LFS {
76+
_, err = git.NewCommand("lfs", "fetch", opts.CloneAddr).RunInDir(repoPath)
77+
if err != nil {
78+
return repo, fmt.Errorf("LFS fetch failed %s: %v", opts.CloneAddr, err)
79+
}
80+
81+
lfsSrc := path.Join(repoPath, "lfs", "objects")
82+
lfsDst := path.Join(setting.LFS.Path)
83+
84+
// move LFS files
85+
err := filepath.Walk(lfsSrc, func(path string, info os.FileInfo, err error) error {
86+
var relSrcPath = strings.Replace(path, lfsSrc, "", 1)
87+
if relSrcPath == "" {
88+
return nil
89+
}
90+
if err != nil {
91+
return err
92+
}
93+
lfsSrcFull := filepath.Join(lfsSrc, relSrcPath)
94+
lfsDstFull := filepath.Join(lfsDst, relSrcPath)
95+
96+
if _, err := os.Stat(lfsDstFull); !os.IsNotExist(err) {
97+
return nil
98+
}
99+
100+
if info.IsDir() {
101+
return os.Mkdir(lfsDstFull, 0755)
102+
}
103+
104+
// generate and associate LFS OIDs
105+
file, err := os.Open(lfsSrcFull)
106+
if err != nil {
107+
return err
108+
}
109+
defer file.Close()
110+
111+
oid, err := models.GenerateLFSOid(file)
112+
if err != nil {
113+
return err
114+
}
115+
fileInfo, err := file.Stat()
116+
if err != nil {
117+
return err
118+
}
119+
120+
lfsDstFull = filepath.Join(lfsDst, oid[0:2], oid[2:4], oid[4:])
121+
err = os.Rename(lfsSrcFull, lfsDstFull)
122+
if err != nil {
123+
return err
124+
}
125+
126+
_, err = models.NewLFSMetaObject(&models.LFSMetaObject{Oid: oid, Size: fileInfo.Size(), RepositoryID: repo.ID})
127+
if err != nil {
128+
log.Error("Unable to write LFS OID[%s] size %d meta object in %v/%v to database. Error: %v", oid, fileInfo.Size(), u.Name, repoPath, err)
129+
return err
130+
}
131+
132+
return nil
133+
})
134+
if err != nil {
135+
return repo, fmt.Errorf("Failed to move LFS files %s: %v", lfsSrc, err)
136+
}
137+
138+
err = os.RemoveAll(path.Join(repoPath, "lfs"))
139+
if err != nil {
140+
return repo, fmt.Errorf("Failed to remove LFS files %s: %v", repoPath, err)
141+
}
142+
}
143+
73144
if opts.Wiki {
74145
wikiPath := models.WikiPath(u.Name, opts.RepoName)
75146
wikiRemotePath := WikiRemoteURL(opts.CloneAddr)

‎modules/structs/repo.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -253,6 +253,7 @@ type MigrateRepoOptions struct {
253253
AuthToken string `json:"auth_token"`
254254

255255
Mirror bool `json:"mirror"`
256+
LFS bool `json:"lfs"`
256257
Private bool `json:"private"`
257258
Description string `json:"description" binding:"MaxSize(255)"`
258259
Wiki bool `json:"wiki"`

‎options/locale/locale_en-US.ini

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -754,6 +754,7 @@ need_auth = Clone Authorization
754754
migrate_options = Migration Options
755755
migrate_service = Migration Service
756756
migrate_options_mirror_helper = This repository will be a <span class="text blue">mirror</span>
757+
migrate_options_mirror_lfs = Mirror LFS data
757758
migrate_options_mirror_disabled = Your site administrator has disabled new mirrors.
758759
migrate_items = Migration Items
759760
migrate_items_wiki = Wiki
@@ -770,7 +771,6 @@ migrate.clone_local_path = or a local server path
770771
migrate.permission_denied = You are not allowed to import local repositories.
771772
migrate.invalid_local_path = "The local path is invalid. It does not exist or is not a directory."
772773
migrate.failed = Migration failed: %v
773-
migrate.lfs_mirror_unsupported = Mirroring LFS objects is not supported - use 'git lfs fetch --all' and 'git lfs push --all' instead.
774774
migrate.migrate_items_options = Access Token is required to migrate additional items
775775
migrated_from = Migrated from <a href="%[1]s">%[2]s</a>
776776
migrated_from_fake = Migrated From %[1]s
@@ -931,7 +931,7 @@ ext_issues = Ext. Issues
931931
ext_issues.desc = Link to an external issue tracker.
932932
933933
projects = Projects
934-
projects.desc = Manage issues and pulls in project boards.
934+
projects.desc = Manage issues and pulls in project boards.
935935
projects.description = Description (optional)
936936
projects.description_placeholder = Description
937937
projects.create = Create Project

‎routers/api/v1/repo/migrate.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,7 @@ func Migrate(ctx *context.APIContext) {
134134
Description: form.Description,
135135
Private: form.Private || setting.Repository.ForcePrivate,
136136
Mirror: form.Mirror,
137+
LFS: form.LFS,
137138
AuthUsername: form.AuthUsername,
138139
AuthPassword: form.AuthPassword,
139140
AuthToken: form.AuthToken,

‎routers/repo/migrate.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ func Migrate(ctx *context.Context) {
4747
ctx.Data["IsForcedPrivate"] = setting.Repository.ForcePrivate
4848
ctx.Data["DisableMirrors"] = setting.Repository.DisableMirrors
4949
ctx.Data["mirror"] = ctx.Query("mirror") == "1"
50+
ctx.Data["LFS"] = ctx.Query("lfs") == "1"
5051
ctx.Data["wiki"] = ctx.Query("wiki") == "1"
5152
ctx.Data["milestones"] = ctx.Query("milestones") == "1"
5253
ctx.Data["labels"] = ctx.Query("labels") == "1"
@@ -172,6 +173,7 @@ func MigratePost(ctx *context.Context) {
172173
Description: form.Description,
173174
Private: form.Private || setting.Repository.ForcePrivate,
174175
Mirror: form.Mirror && !setting.Repository.DisableMirrors,
176+
LFS: form.LFS,
175177
AuthUsername: form.AuthUsername,
176178
AuthPassword: form.AuthPassword,
177179
AuthToken: form.AuthToken,

‎templates/repo/migrate/git.tmpl

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@
1515
<input id="clone_addr" name="clone_addr" value="{{.clone_addr}}" autofocus required>
1616
<span class="help">
1717
{{.i18n.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate.clone_local_path"}}{{end}}
18-
{{if .LFSActive}}<br/>{{.i18n.Tr "repo.migrate.lfs_mirror_unsupported"}}{{end}}
1918
</span>
2019
</div>
2120
<div class="inline field {{if .Err_Auth}}error{{end}}">
@@ -30,15 +29,21 @@
3029

3130
<div class="inline field">
3231
<label>{{.i18n.Tr "repo.migrate_options"}}</label>
32+
{{if .DisableMirrors}}
3333
<div class="ui checkbox">
34-
{{if .DisableMirrors}}
3534
<input id="mirror" name="mirror" type="checkbox" readonly>
3635
<label>{{.i18n.Tr "repo.migrate_options_mirror_disabled"}}</label>
37-
{{else}}
38-
<input id="mirror" name="mirror" type="checkbox" {{if .mirror}}checked{{end}}>
36+
</div>
37+
{{else}}
38+
<div class="ui checkbox">
39+
<input id="mirror" name="mirror" type="checkbox" {{if .mirror}} checked{{end}}>
3940
<label>{{.i18n.Tr "repo.migrate_options_mirror_helper" | Safe}}</label>
40-
{{end}}
4141
</div>
42+
<div class="ui checkbox">
43+
<input id="lfs" name="lfs" type="checkbox" {{if .lfs}} checked{{end}}>
44+
<label>{{.i18n.Tr "repo.migrate_options_mirror_lfs" | Safe}}</label>
45+
</div>
46+
{{end}}
4247
</div>
4348

4449
<div class="ui divider"></div>

‎templates/repo/migrate/gitea.tmpl

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@
1515
<input id="clone_addr" name="clone_addr" value="{{.clone_addr}}" autofocus required>
1616
<span class="help">
1717
{{.i18n.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate.clone_local_path"}}{{end}}
18-
{{if .LFSActive}}<br />{{.i18n.Tr "repo.migrate.lfs_mirror_unsupported"}}{{end}}
1918
</span>
2019
</div>
2120

@@ -27,15 +26,21 @@
2726

2827
<div class="inline field">
2928
<label>{{.i18n.Tr "repo.migrate_options"}}</label>
29+
{{if .DisableMirrors}}
3030
<div class="ui checkbox">
31-
{{if .DisableMirrors}}
3231
<input id="mirror" name="mirror" type="checkbox" readonly>
3332
<label>{{.i18n.Tr "repo.migrate_options_mirror_disabled"}}</label>
34-
{{else}}
33+
</div>
34+
{{else}}
35+
<div class="ui checkbox">
3536
<input id="mirror" name="mirror" type="checkbox" {{if .mirror}} checked{{end}}>
3637
<label>{{.i18n.Tr "repo.migrate_options_mirror_helper" | Safe}}</label>
37-
{{end}}
3838
</div>
39+
<div class="ui checkbox">
40+
<input id="lfs" name="lfs" type="checkbox" {{if .lfs}} checked{{end}}>
41+
<label>{{.i18n.Tr "repo.migrate_options_mirror_lfs" | Safe}}</label>
42+
</div>
43+
{{end}}
3944
</div>
4045

4146
<span class="help">{{.i18n.Tr "repo.migrate.migrate_items_options"}}</span>

‎templates/repo/migrate/github.tmpl

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@
1515
<input id="clone_addr" name="clone_addr" value="{{.clone_addr}}" autofocus required>
1616
<span class="help">
1717
{{.i18n.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate.clone_local_path"}}{{end}}
18-
{{if .LFSActive}}<br/>{{.i18n.Tr "repo.migrate.lfs_mirror_unsupported"}}{{end}}
1918
</span>
2019
</div>
2120

@@ -27,15 +26,21 @@
2726

2827
<div class="inline field">
2928
<label>{{.i18n.Tr "repo.migrate_options"}}</label>
29+
{{if .DisableMirrors}}
3030
<div class="ui checkbox">
31-
{{if .DisableMirrors}}
3231
<input id="mirror" name="mirror" type="checkbox" readonly>
3332
<label>{{.i18n.Tr "repo.migrate_options_mirror_disabled"}}</label>
34-
{{else}}
35-
<input id="mirror" name="mirror" type="checkbox" {{if .mirror}}checked{{end}}>
33+
</div>
34+
{{else}}
35+
<div class="ui checkbox">
36+
<input id="mirror" name="mirror" type="checkbox" {{if .mirror}} checked{{end}}>
3637
<label>{{.i18n.Tr "repo.migrate_options_mirror_helper" | Safe}}</label>
37-
{{end}}
3838
</div>
39+
<div class="ui checkbox">
40+
<input id="lfs" name="lfs" type="checkbox" {{if .lfs}} checked{{end}}>
41+
<label>{{.i18n.Tr "repo.migrate_options_mirror_lfs" | Safe}}</label>
42+
</div>
43+
{{end}}
3944
</div>
4045

4146
<span class="help">{{.i18n.Tr "repo.migrate.migrate_items_options"}}</span>

‎templates/repo/migrate/gitlab.tmpl

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@
1515
<input id="clone_addr" name="clone_addr" value="{{.clone_addr}}" autofocus required>
1616
<span class="help">
1717
{{.i18n.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate.clone_local_path"}}{{end}}
18-
{{if .LFSActive}}<br/>{{.i18n.Tr "repo.migrate.lfs_mirror_unsupported"}}{{end}}
1918
</span>
2019
</div>
2120

@@ -27,15 +26,21 @@
2726

2827
<div class="inline field">
2928
<label>{{.i18n.Tr "repo.migrate_options"}}</label>
29+
{{if .DisableMirrors}}
3030
<div class="ui checkbox">
31-
{{if .DisableMirrors}}
3231
<input id="mirror" name="mirror" type="checkbox" readonly>
3332
<label>{{.i18n.Tr "repo.migrate_options_mirror_disabled"}}</label>
34-
{{else}}
35-
<input id="mirror" name="mirror" type="checkbox" {{if .mirror}}checked{{end}}>
33+
</div>
34+
{{else}}
35+
<div class="ui checkbox">
36+
<input id="mirror" name="mirror" type="checkbox" {{if .mirror}} checked{{end}}>
3637
<label>{{.i18n.Tr "repo.migrate_options_mirror_helper" | Safe}}</label>
37-
{{end}}
3838
</div>
39+
<div class="ui checkbox">
40+
<input id="lfs" name="lfs" type="checkbox" {{if .lfs}} checked{{end}}>
41+
<label>{{.i18n.Tr "repo.migrate_options_mirror_lfs" | Safe}}</label>
42+
</div>
43+
{{end}}
3944
</div>
4045

4146
<span class="help">{{.i18n.Tr "repo.migrate.migrate_items_options"}}</span>

‎templates/repo/migrate/gogs.tmpl

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@
1515
<input id="clone_addr" name="clone_addr" value="{{.clone_addr}}" autofocus required>
1616
<span class="help">
1717
{{.i18n.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate.clone_local_path"}}{{end}}
18-
{{if .LFSActive}}<br />{{.i18n.Tr "repo.migrate.lfs_mirror_unsupported"}}{{end}}
1918
</span>
2019
</div>
2120

@@ -27,15 +26,21 @@
2726

2827
<div class="inline field">
2928
<label>{{.i18n.Tr "repo.migrate_options"}}</label>
29+
{{if .DisableMirrors}}
3030
<div class="ui checkbox">
31-
{{if .DisableMirrors}}
3231
<input id="mirror" name="mirror" type="checkbox" readonly>
3332
<label>{{.i18n.Tr "repo.migrate_options_mirror_disabled"}}</label>
34-
{{else}}
33+
</div>
34+
{{else}}
35+
<div class="ui checkbox">
3536
<input id="mirror" name="mirror" type="checkbox" {{if .mirror}} checked{{end}}>
3637
<label>{{.i18n.Tr "repo.migrate_options_mirror_helper" | Safe}}</label>
37-
{{end}}
3838
</div>
39+
<div class="ui checkbox">
40+
<input id="lfs" name="lfs" type="checkbox" {{if .lfs}} checked{{end}}>
41+
<label>{{.i18n.Tr "repo.migrate_options_mirror_lfs" | Safe}}</label>
42+
</div>
43+
{{end}}
3944
</div>
4045

4146
<span class="help">{{.i18n.Tr "repo.migrate.migrate_items_options"}}</span>

‎templates/swagger/v1_json.tmpl

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14636,6 +14636,10 @@
1463614636
"type": "boolean",
1463714637
"x-go-name": "Labels"
1463814638
},
14639+
"lfs": {
14640+
"type": "boolean",
14641+
"x-go-name": "LFS"
14642+
},
1463914643
"milestones": {
1464014644
"type": "boolean",
1464114645
"x-go-name": "Milestones"
@@ -14715,6 +14719,10 @@
1471514719
"type": "boolean",
1471614720
"x-go-name": "Labels"
1471714721
},
14722+
"lfs": {
14723+
"type": "boolean",
14724+
"x-go-name": "LFS"
14725+
},
1471814726
"milestones": {
1471914727
"type": "boolean",
1472014728
"x-go-name": "Milestones"

0 commit comments

Comments
 (0)