Skip to content

Commit d92781b

Browse files
lunnyzeripath
lunny
authored and
zeripath
committed
Refactor repository check and sync functions (#9854)
Move more general repository functions out of models/repo.go
1 parent 27c6b8f commit d92781b

File tree

12 files changed

+313
-210
lines changed

12 files changed

+313
-210
lines changed

‎cmd/admin.go

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,10 @@ import (
1414
"code.gitea.io/gitea/models"
1515
"code.gitea.io/gitea/modules/auth/oauth2"
1616
"code.gitea.io/gitea/modules/git"
17+
"code.gitea.io/gitea/modules/graceful"
1718
"code.gitea.io/gitea/modules/log"
1819
pwd "code.gitea.io/gitea/modules/password"
19-
"code.gitea.io/gitea/modules/repository"
20+
repo_module "code.gitea.io/gitea/modules/repository"
2021
"code.gitea.io/gitea/modules/setting"
2122

2223
"github.com/urfave/cli"
@@ -375,7 +376,7 @@ func runRepoSyncReleases(c *cli.Context) error {
375376
}
376377
log.Trace(" currentNumReleases is %d, running SyncReleasesWithTags", oldnum)
377378

378-
if err = repository.SyncReleasesWithTags(repo, gitRepo); err != nil {
379+
if err = repo_module.SyncReleasesWithTags(repo, gitRepo); err != nil {
379380
log.Warn(" SyncReleasesWithTags: %v", err)
380381
gitRepo.Close()
381382
continue
@@ -410,7 +411,7 @@ func runRegenerateHooks(c *cli.Context) error {
410411
if err := initDB(); err != nil {
411412
return err
412413
}
413-
return models.SyncRepositoryHooks()
414+
return repo_module.SyncRepositoryHooks(graceful.GetManager().ShutdownContext())
414415
}
415416

416417
func runRegenerateKeys(c *cli.Context) error {

‎models/context.go

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,12 @@
44

55
package models
66

7+
import (
8+
"code.gitea.io/gitea/modules/setting"
9+
10+
"xorm.io/builder"
11+
)
12+
713
// DBContext represents a db context
814
type DBContext struct {
915
e Engine
@@ -53,3 +59,10 @@ func WithTx(f func(ctx DBContext) error) error {
5359
sess.Close()
5460
return err
5561
}
62+
63+
// Iterate iterates the databases and doing something
64+
func Iterate(ctx DBContext, tableBean interface{}, cond builder.Cond, fun func(idx int, bean interface{}) error) error {
65+
return ctx.e.Where(cond).
66+
BufferSize(setting.Database.IterateBufferSize).
67+
Iterate(tableBean, fun)
68+
}

‎models/migrations/v36.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,12 @@
55
package migrations
66

77
import (
8-
"code.gitea.io/gitea/models"
8+
"code.gitea.io/gitea/modules/graceful"
9+
repo_module "code.gitea.io/gitea/modules/repository"
910

1011
"xorm.io/xorm"
1112
)
1213

1314
func regenerateGitHooks36(x *xorm.Engine) (err error) {
14-
return models.SyncRepositoryHooks()
15+
return repo_module.SyncRepositoryHooks(graceful.GetManager().ShutdownContext())
1516
}

‎models/repo.go

Lines changed: 8 additions & 193 deletions
Original file line numberDiff line numberDiff line change
@@ -26,12 +26,10 @@ import (
2626
"time"
2727

2828
"code.gitea.io/gitea/modules/avatar"
29-
"code.gitea.io/gitea/modules/git"
3029
"code.gitea.io/gitea/modules/log"
3130
"code.gitea.io/gitea/modules/markup"
3231
"code.gitea.io/gitea/modules/options"
3332
"code.gitea.io/gitea/modules/setting"
34-
"code.gitea.io/gitea/modules/structs"
3533
api "code.gitea.io/gitea/modules/structs"
3634
"code.gitea.io/gitea/modules/timeutil"
3735
"code.gitea.io/gitea/modules/util"
@@ -147,13 +145,13 @@ type Repository struct {
147145
ID int64 `xorm:"pk autoincr"`
148146
OwnerID int64 `xorm:"UNIQUE(s) index"`
149147
OwnerName string
150-
Owner *User `xorm:"-"`
151-
LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"`
152-
Name string `xorm:"INDEX NOT NULL"`
153-
Description string `xorm:"TEXT"`
154-
Website string `xorm:"VARCHAR(2048)"`
155-
OriginalServiceType structs.GitServiceType `xorm:"index"`
156-
OriginalURL string `xorm:"VARCHAR(2048)"`
148+
Owner *User `xorm:"-"`
149+
LowerName string `xorm:"UNIQUE(s) INDEX NOT NULL"`
150+
Name string `xorm:"INDEX NOT NULL"`
151+
Description string `xorm:"TEXT"`
152+
Website string `xorm:"VARCHAR(2048)"`
153+
OriginalServiceType api.GitServiceType `xorm:"index"`
154+
OriginalURL string `xorm:"VARCHAR(2048)"`
157155
DefaultBranch string
158156

159157
NumWatches int
@@ -911,63 +909,12 @@ func CheckCreateRepository(doer, u *User, name string) error {
911909
return nil
912910
}
913911

914-
// CreateDelegateHooks creates all the hooks scripts for the repo
915-
func CreateDelegateHooks(repoPath string) error {
916-
return createDelegateHooks(repoPath)
917-
}
918-
919-
// createDelegateHooks creates all the hooks scripts for the repo
920-
func createDelegateHooks(repoPath string) (err error) {
921-
922-
var (
923-
hookNames = []string{"pre-receive", "update", "post-receive"}
924-
hookTpls = []string{
925-
fmt.Sprintf("#!/usr/bin/env %s\ndata=$(cat)\nexitcodes=\"\"\nhookname=$(basename $0)\nGIT_DIR=${GIT_DIR:-$(dirname $0)}\n\nfor hook in ${GIT_DIR}/hooks/${hookname}.d/*; do\ntest -x \"${hook}\" || continue\necho \"${data}\" | \"${hook}\"\nexitcodes=\"${exitcodes} $?\"\ndone\n\nfor i in ${exitcodes}; do\n[ ${i} -eq 0 ] || exit ${i}\ndone\n", setting.ScriptType),
926-
fmt.Sprintf("#!/usr/bin/env %s\nexitcodes=\"\"\nhookname=$(basename $0)\nGIT_DIR=${GIT_DIR:-$(dirname $0)}\n\nfor hook in ${GIT_DIR}/hooks/${hookname}.d/*; do\ntest -x \"${hook}\" || continue\n\"${hook}\" $1 $2 $3\nexitcodes=\"${exitcodes} $?\"\ndone\n\nfor i in ${exitcodes}; do\n[ ${i} -eq 0 ] || exit ${i}\ndone\n", setting.ScriptType),
927-
fmt.Sprintf("#!/usr/bin/env %s\ndata=$(cat)\nexitcodes=\"\"\nhookname=$(basename $0)\nGIT_DIR=${GIT_DIR:-$(dirname $0)}\n\nfor hook in ${GIT_DIR}/hooks/${hookname}.d/*; do\ntest -x \"${hook}\" || continue\necho \"${data}\" | \"${hook}\"\nexitcodes=\"${exitcodes} $?\"\ndone\n\nfor i in ${exitcodes}; do\n[ ${i} -eq 0 ] || exit ${i}\ndone\n", setting.ScriptType),
928-
}
929-
giteaHookTpls = []string{
930-
fmt.Sprintf("#!/usr/bin/env %s\n\"%s\" hook --config='%s' pre-receive\n", setting.ScriptType, setting.AppPath, setting.CustomConf),
931-
fmt.Sprintf("#!/usr/bin/env %s\n\"%s\" hook --config='%s' update $1 $2 $3\n", setting.ScriptType, setting.AppPath, setting.CustomConf),
932-
fmt.Sprintf("#!/usr/bin/env %s\n\"%s\" hook --config='%s' post-receive\n", setting.ScriptType, setting.AppPath, setting.CustomConf),
933-
}
934-
)
935-
936-
hookDir := filepath.Join(repoPath, "hooks")
937-
938-
for i, hookName := range hookNames {
939-
oldHookPath := filepath.Join(hookDir, hookName)
940-
newHookPath := filepath.Join(hookDir, hookName+".d", "gitea")
941-
942-
if err := os.MkdirAll(filepath.Join(hookDir, hookName+".d"), os.ModePerm); err != nil {
943-
return fmt.Errorf("create hooks dir '%s': %v", filepath.Join(hookDir, hookName+".d"), err)
944-
}
945-
946-
// WARNING: This will override all old server-side hooks
947-
if err = os.Remove(oldHookPath); err != nil && !os.IsNotExist(err) {
948-
return fmt.Errorf("unable to pre-remove old hook file '%s' prior to rewriting: %v ", oldHookPath, err)
949-
}
950-
if err = ioutil.WriteFile(oldHookPath, []byte(hookTpls[i]), 0777); err != nil {
951-
return fmt.Errorf("write old hook file '%s': %v", oldHookPath, err)
952-
}
953-
954-
if err = os.Remove(newHookPath); err != nil && !os.IsNotExist(err) {
955-
return fmt.Errorf("unable to pre-remove new hook file '%s' prior to rewriting: %v", newHookPath, err)
956-
}
957-
if err = ioutil.WriteFile(newHookPath, []byte(giteaHookTpls[i]), 0777); err != nil {
958-
return fmt.Errorf("write new hook file '%s': %v", newHookPath, err)
959-
}
960-
}
961-
962-
return nil
963-
}
964-
965912
// CreateRepoOptions contains the create repository options
966913
type CreateRepoOptions struct {
967914
Name string
968915
Description string
969916
OriginalURL string
970-
GitServiceType structs.GitServiceType
917+
GitServiceType api.GitServiceType
971918
Gitignores string
972919
IssueLabels string
973920
License string
@@ -1883,138 +1830,6 @@ func deleteOldRepositoryArchives(ctx context.Context, idx int, bean interface{})
18831830
return nil
18841831
}
18851832

1886-
func gatherMissingRepoRecords() ([]*Repository, error) {
1887-
repos := make([]*Repository, 0, 10)
1888-
if err := x.
1889-
Where("id > 0").
1890-
Iterate(new(Repository),
1891-
func(idx int, bean interface{}) error {
1892-
repo := bean.(*Repository)
1893-
if !com.IsDir(repo.RepoPath()) {
1894-
repos = append(repos, repo)
1895-
}
1896-
return nil
1897-
}); err != nil {
1898-
if err2 := CreateRepositoryNotice(fmt.Sprintf("gatherMissingRepoRecords: %v", err)); err2 != nil {
1899-
return nil, fmt.Errorf("CreateRepositoryNotice: %v", err)
1900-
}
1901-
}
1902-
return repos, nil
1903-
}
1904-
1905-
// DeleteMissingRepositories deletes all repository records that lost Git files.
1906-
func DeleteMissingRepositories(doer *User) error {
1907-
repos, err := gatherMissingRepoRecords()
1908-
if err != nil {
1909-
return fmt.Errorf("gatherMissingRepoRecords: %v", err)
1910-
}
1911-
1912-
if len(repos) == 0 {
1913-
return nil
1914-
}
1915-
1916-
for _, repo := range repos {
1917-
log.Trace("Deleting %d/%d...", repo.OwnerID, repo.ID)
1918-
if err := DeleteRepository(doer, repo.OwnerID, repo.ID); err != nil {
1919-
if err2 := CreateRepositoryNotice(fmt.Sprintf("DeleteRepository [%d]: %v", repo.ID, err)); err2 != nil {
1920-
return fmt.Errorf("CreateRepositoryNotice: %v", err)
1921-
}
1922-
}
1923-
}
1924-
return nil
1925-
}
1926-
1927-
// ReinitMissingRepositories reinitializes all repository records that lost Git files.
1928-
func ReinitMissingRepositories() error {
1929-
repos, err := gatherMissingRepoRecords()
1930-
if err != nil {
1931-
return fmt.Errorf("gatherMissingRepoRecords: %v", err)
1932-
}
1933-
1934-
if len(repos) == 0 {
1935-
return nil
1936-
}
1937-
1938-
for _, repo := range repos {
1939-
log.Trace("Initializing %d/%d...", repo.OwnerID, repo.ID)
1940-
if err := git.InitRepository(repo.RepoPath(), true); err != nil {
1941-
if err2 := CreateRepositoryNotice(fmt.Sprintf("InitRepository [%d]: %v", repo.ID, err)); err2 != nil {
1942-
return fmt.Errorf("CreateRepositoryNotice: %v", err)
1943-
}
1944-
}
1945-
}
1946-
return nil
1947-
}
1948-
1949-
// SyncRepositoryHooks rewrites all repositories' pre-receive, update and post-receive hooks
1950-
// to make sure the binary and custom conf path are up-to-date.
1951-
func SyncRepositoryHooks() error {
1952-
return x.Cols("owner_id", "name").Where("id > 0").Iterate(new(Repository),
1953-
func(idx int, bean interface{}) error {
1954-
if err := createDelegateHooks(bean.(*Repository).RepoPath()); err != nil {
1955-
return fmt.Errorf("SyncRepositoryHook: %v", err)
1956-
}
1957-
if bean.(*Repository).HasWiki() {
1958-
if err := createDelegateHooks(bean.(*Repository).WikiPath()); err != nil {
1959-
return fmt.Errorf("SyncRepositoryHook: %v", err)
1960-
}
1961-
}
1962-
return nil
1963-
})
1964-
}
1965-
1966-
// GitFsck calls 'git fsck' to check repository health.
1967-
func GitFsck(ctx context.Context) {
1968-
log.Trace("Doing: GitFsck")
1969-
if err := x.
1970-
Where("id>0 AND is_fsck_enabled=?", true).BufferSize(setting.Database.IterateBufferSize).
1971-
Iterate(new(Repository),
1972-
func(idx int, bean interface{}) error {
1973-
select {
1974-
case <-ctx.Done():
1975-
return fmt.Errorf("Aborted due to shutdown")
1976-
default:
1977-
}
1978-
repo := bean.(*Repository)
1979-
repoPath := repo.RepoPath()
1980-
log.Trace("Running health check on repository %s", repoPath)
1981-
if err := git.Fsck(repoPath, setting.Cron.RepoHealthCheck.Timeout, setting.Cron.RepoHealthCheck.Args...); err != nil {
1982-
desc := fmt.Sprintf("Failed to health check repository (%s): %v", repoPath, err)
1983-
log.Warn(desc)
1984-
if err = CreateRepositoryNotice(desc); err != nil {
1985-
log.Error("CreateRepositoryNotice: %v", err)
1986-
}
1987-
}
1988-
return nil
1989-
}); err != nil {
1990-
log.Error("GitFsck: %v", err)
1991-
}
1992-
log.Trace("Finished: GitFsck")
1993-
}
1994-
1995-
// GitGcRepos calls 'git gc' to remove unnecessary files and optimize the local repository
1996-
func GitGcRepos() error {
1997-
args := append([]string{"gc"}, setting.Git.GCArgs...)
1998-
return x.
1999-
Where("id > 0").BufferSize(setting.Database.IterateBufferSize).
2000-
Iterate(new(Repository),
2001-
func(idx int, bean interface{}) error {
2002-
repo := bean.(*Repository)
2003-
if err := repo.GetOwner(); err != nil {
2004-
return err
2005-
}
2006-
if stdout, err := git.NewCommand(args...).
2007-
SetDescription(fmt.Sprintf("Repository Garbage Collection: %s", repo.FullName())).
2008-
RunInDirTimeout(
2009-
time.Duration(setting.Git.Timeout.GC)*time.Second,
2010-
repo.RepoPath()); err != nil {
2011-
log.Error("Repository garbage collection failed for %v. Stdout: %s\nError: %v", repo, stdout, err)
2012-
return fmt.Errorf("Repository garbage collection failed: Error: %v", err)
2013-
}
2014-
return nil
2015-
})
2016-
}
2017-
20181833
type repoChecker struct {
20191834
querySQL, correctSQL string
20201835
desc string

‎modules/cron/cron.go

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
"code.gitea.io/gitea/modules/graceful"
1414
"code.gitea.io/gitea/modules/log"
1515
"code.gitea.io/gitea/modules/migrations"
16+
repo_module "code.gitea.io/gitea/modules/repository"
1617
"code.gitea.io/gitea/modules/setting"
1718
"code.gitea.io/gitea/modules/sync"
1819
mirror_service "code.gitea.io/gitea/services/mirror"
@@ -69,14 +70,22 @@ func NewContext() {
6970
}
7071
}
7172
if setting.Cron.RepoHealthCheck.Enabled {
72-
entry, err = c.AddFunc("Repository health check", setting.Cron.RepoHealthCheck.Schedule, WithUnique(gitFsck, models.GitFsck))
73+
entry, err = c.AddFunc("Repository health check", setting.Cron.RepoHealthCheck.Schedule, WithUnique(gitFsck, func(ctx context.Context) {
74+
if err := repo_module.GitFsck(ctx); err != nil {
75+
log.Error("GitFsck: %s", err)
76+
}
77+
}))
7378
if err != nil {
7479
log.Fatal("Cron[Repository health check]: %v", err)
7580
}
7681
if setting.Cron.RepoHealthCheck.RunAtStart {
7782
entry.Prev = time.Now()
7883
entry.ExecTimes++
79-
go WithUnique(gitFsck, models.GitFsck)()
84+
go WithUnique(gitFsck, func(ctx context.Context) {
85+
if err := repo_module.GitFsck(ctx); err != nil {
86+
log.Error("GitFsck: %s", err)
87+
}
88+
})()
8089
}
8190
}
8291
if setting.Cron.CheckRepoStats.Enabled {

0 commit comments

Comments
 (0)