From 326cb03e04512f65f2a85399e70b431287547150 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Thu, 10 Jun 2021 01:53:16 +0800 Subject: [PATCH 1/6] Defer last commit info Signed-off-by: Andrew Thornton --- integrations/repo_test.go | 11 +- modules/git/commit_info_gogit.go | 31 ++-- modules/git/commit_info_nogogit.go | 32 ++-- modules/git/last_commit_cache_gogit.go | 7 + modules/git/last_commit_cache_nogogit.go | 7 + modules/git/notes_nogogit.go | 5 +- modules/repository/cache.go | 5 +- modules/repository/last_commit_queue_gogit.go | 166 ++++++++++++++++++ .../repository/last_commit_queue_nogogit.go | 156 ++++++++++++++++ routers/init.go | 4 + routers/web/repo/commit.go | 6 +- routers/web/repo/view.go | 10 ++ templates/repo/commit_page.tmpl | 8 +- templates/repo/view_list.tmpl | 12 +- web_src/js/index.js | 4 + 15 files changed, 422 insertions(+), 42 deletions(-) create mode 100644 modules/repository/last_commit_queue_gogit.go create mode 100644 modules/repository/last_commit_queue_nogogit.go diff --git a/integrations/repo_test.go b/integrations/repo_test.go index 8c4cdf5a969fc..7df49c7d31f93 100644 --- a/integrations/repo_test.go +++ b/integrations/repo_test.go @@ -12,6 +12,7 @@ import ( "testing" "time" + "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" "github.com/PuerkitoBio/goquery" @@ -92,7 +93,15 @@ func TestViewRepo2(t *testing.T) { // enable last commit cache for all repositories oldCommitsCount := setting.CacheService.LastCommit.CommitsCount setting.CacheService.LastCommit.CommitsCount = 0 - // first view will not hit the cache + + ccr := &repository.CommitCacheRequest{ + Repo: "user3/repo3", + CommitID: "master", + TreePath: "", + } + err := ccr.Do() + assert.NoError(t, err) + // first view should hit the cache testViewRepo(t) // second view will hit the cache testViewRepo(t) diff --git a/modules/git/commit_info_gogit.go b/modules/git/commit_info_gogit.go index a8006dcef2e64..727a0b2e39cbd 100644 --- a/modules/git/commit_info_gogit.go +++ b/modules/git/commit_info_gogit.go @@ -37,24 +37,10 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath var revs map[string]*object.Commit if cache != nil { - var unHitPaths []string - revs, unHitPaths, err = getLastCommitForPathsByCache(commit.ID.String(), treePath, entryPaths, cache) + revs, _, err = getLastCommitForPathsByCache(commit.ID.String(), treePath, entryPaths, cache) if err != nil { return nil, nil, err } - if len(unHitPaths) > 0 { - revs2, err := GetLastCommitForPaths(ctx, c, treePath, unHitPaths) - if err != nil { - return nil, nil, err - } - - for k, v := range revs2 { - if err := cache.Put(commit.ID.String(), path.Join(treePath, k), v.ID().String()); err != nil { - return nil, nil, err - } - revs[k] = v - } - } } else { revs, err = GetLastCommitForPaths(ctx, c, treePath, entryPaths) } @@ -88,6 +74,21 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath subModuleFile := NewSubModuleFile(entryCommit, subModuleURL, entry.ID.String()) commitsInfo[i].SubModuleFile = subModuleFile } + } else if entry.IsSubModule() { + subModuleURL := "" + var fullPath string + if len(treePath) > 0 { + fullPath = treePath + "/" + entry.Name() + } else { + fullPath = entry.Name() + } + if subModule, err := commit.GetSubModule(fullPath); err != nil { + return nil, nil, err + } else if subModule != nil { + subModuleURL = subModule.URL + } + subModuleFile := NewSubModuleFile(nil, subModuleURL, entry.ID.String()) + commitsInfo[i].SubModuleFile = subModuleFile } } diff --git a/modules/git/commit_info_nogogit.go b/modules/git/commit_info_nogogit.go index f34bef9f018c8..4c3996c2a04a5 100644 --- a/modules/git/commit_info_nogogit.go +++ b/modules/git/commit_info_nogogit.go @@ -31,25 +31,10 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath var revs map[string]*Commit if cache != nil { - var unHitPaths []string - revs, unHitPaths, err = getLastCommitForPathsByCache(ctx, commit.ID.String(), treePath, entryPaths, cache) + revs, _, err = getLastCommitForPathsByCache(ctx, commit.ID.String(), treePath, entryPaths, cache) if err != nil { return nil, nil, err } - if len(unHitPaths) > 0 { - sort.Strings(unHitPaths) - commits, err := GetLastCommitForPaths(ctx, commit, treePath, unHitPaths) - if err != nil { - return nil, nil, err - } - - for i, found := range commits { - if err := cache.Put(commit.ID.String(), path.Join(treePath, unHitPaths[i]), found.ID.String()); err != nil { - return nil, nil, err - } - revs[unHitPaths[i]] = found - } - } } else { sort.Strings(entryPaths) revs = map[string]*Commit{} @@ -86,6 +71,21 @@ func (tes Entries) GetCommitsInfo(ctx context.Context, commit *Commit, treePath subModuleFile := NewSubModuleFile(entryCommit, subModuleURL, entry.ID.String()) commitsInfo[i].SubModuleFile = subModuleFile } + } else if entry.IsSubModule() { + subModuleURL := "" + var fullPath string + if len(treePath) > 0 { + fullPath = treePath + "/" + entry.Name() + } else { + fullPath = entry.Name() + } + if subModule, err := commit.GetSubModule(fullPath); err != nil { + return nil, nil, err + } else if subModule != nil { + subModuleURL = subModule.URL + } + subModuleFile := NewSubModuleFile(entryCommit, subModuleURL, entry.ID.String()) + commitsInfo[i].SubModuleFile = subModuleFile } } diff --git a/modules/git/last_commit_cache_gogit.go b/modules/git/last_commit_cache_gogit.go index 16fb1c988cca8..48d66e0ffc15a 100644 --- a/modules/git/last_commit_cache_gogit.go +++ b/modules/git/last_commit_cache_gogit.go @@ -37,6 +37,13 @@ func NewLastCommitCache(repoPath string, gitRepo *Repository, ttl func() int64, } } +// GetCachedCommitID returns if there is a cached commit for the ref and entryPath +func (c *LastCommitCache) GetCachedCommitID(ref, entryPath string) (string, bool) { + v := c.cache.Get(c.getCacheKey(c.repoPath, ref, entryPath)) + vs, ok := v.(string) + return vs, ok +} + // Get get the last commit information by commit id and entry path func (c *LastCommitCache) Get(ref, entryPath string) (interface{}, error) { v := c.cache.Get(c.getCacheKey(c.repoPath, ref, entryPath)) diff --git a/modules/git/last_commit_cache_nogogit.go b/modules/git/last_commit_cache_nogogit.go index 3cbb0cca32e05..69e74632bdbd6 100644 --- a/modules/git/last_commit_cache_nogogit.go +++ b/modules/git/last_commit_cache_nogogit.go @@ -35,6 +35,13 @@ func NewLastCommitCache(repoPath string, gitRepo *Repository, ttl func() int64, } } +// GetCachedCommitID returns if there is a cached commit for the ref and entryPath +func (c *LastCommitCache) GetCachedCommitID(ref, entryPath string) (string, bool) { + v := c.cache.Get(c.getCacheKey(c.repoPath, ref, entryPath)) + vs, ok := v.(string) + return vs, ok +} + // Get get the last commit information by commit id and entry path func (c *LastCommitCache) Get(ref, entryPath string, wr WriteCloserError, rd *bufio.Reader) (interface{}, error) { v := c.cache.Get(c.getCacheKey(c.repoPath, ref, entryPath)) diff --git a/modules/git/notes_nogogit.go b/modules/git/notes_nogogit.go index 2b927249954a7..a7291359c9e8f 100644 --- a/modules/git/notes_nogogit.go +++ b/modules/git/notes_nogogit.go @@ -68,7 +68,8 @@ func GetNote(ctx context.Context, repo *Repository, commitID string, note *Note) if err != nil { return err } - note.Commit = lastCommits[0] - + if len(lastCommits) > 0 { + note.Commit = lastCommits[0] + } return nil } diff --git a/modules/repository/cache.go b/modules/repository/cache.go index e574f1adb7f7c..72a4afba91397 100644 --- a/modules/repository/cache.go +++ b/modules/repository/cache.go @@ -42,7 +42,8 @@ func CacheRef(ctx context.Context, repo *models.Repository, gitRepo *git.Reposit return nil } - commitCache := git.NewLastCommitCache(repo.FullName(), gitRepo, setting.LastCommitCacheTTLSeconds, cache.GetCache()) + repoFullName := repo.FullName() + commitID := commit.ID.String() - return commitCache.CacheCommit(ctx, commit) + return UpdateCache(repoFullName, commitID, "", true) } diff --git a/modules/repository/last_commit_queue_gogit.go b/modules/repository/last_commit_queue_gogit.go new file mode 100644 index 0000000000000..32277a4ee684e --- /dev/null +++ b/modules/repository/last_commit_queue_gogit.go @@ -0,0 +1,166 @@ +// Copyright 2019 Gitea. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build gogit + +package repository + +import ( + "path" + "path/filepath" + "sync" + + "code.gitea.io/gitea/modules/cache" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/graceful" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/queue" + "code.gitea.io/gitea/modules/setting" +) + +var lock = sync.Mutex{} +var table = map[CommitCacheRequest]bool{} +var lastCommitQueue queue.UniqueQueue + +// CommitCacheRequest represents a cache request +type CommitCacheRequest struct { + Repo string + CommitID string + TreePath string + Recursive bool +} + +// Do runs the cache request uniquely ensuring that only one cache request is running for this request triple +func (req *CommitCacheRequest) Do() error { + recursive := req.Recursive + req.Recursive = false + + repo, err := git.OpenRepository(filepath.Join(setting.RepoRootPath, req.Repo+".git")) + if err != nil { + return err + } + commit, err := repo.GetCommit(req.CommitID) + if err != nil { + if git.IsErrNotExist(err) { + return nil + } + return err + } + + lccache := git.NewLastCommitCache(req.Repo, repo, setting.LastCommitCacheTTLSeconds, cache.GetCache()) + + directories := []string{req.TreePath} + for len(directories) > 0 { + req.TreePath = directories[len(directories)-1] + next, err := req.doTree(repo, commit, recursive, lccache) + if err != nil { + return err + } + directories = append(next, directories[:len(directories)-1]...) + } + return nil +} + +func (req *CommitCacheRequest) doTree(repo *git.Repository, commit *git.Commit, recursive bool, lccache *git.LastCommitCache) ([]string, error) { + + tree, err := commit.Tree.SubTree(req.TreePath) + if err != nil { + if git.IsErrNotExist(err) { + return nil, nil + } + return nil, err + } + entries, err := tree.ListEntries() + if err != nil { + if git.IsErrNotExist(err) { + return nil, nil + } + return nil, err + } + directories := make([]string, 0, len(entries)) + + commitNodeIndex, commitGraphFile := repo.CommitNodeIndex() + if commitGraphFile != nil { + defer commitGraphFile.Close() + } + + commitNode, err := commitNodeIndex.Get(commit.ID) + if err != nil { + return nil, err + } + + lock.Lock() + if has := table[*req]; has { + lock.Unlock() + if recursive { + for _, entry := range entries { + if entry.IsDir() { + directories = append(directories, path.Join(req.TreePath, entry.Name())) + } + } + } + return directories, nil + } + table[*req] = true + lock.Unlock() + defer func() { + lock.Lock() + delete(table, *req) + lock.Unlock() + }() + + entryPaths := make([]string, 0, len(entries)) + for _, entry := range entries { + if recursive && entry.IsDir() { + directories = append(directories, path.Join(req.TreePath, entry.Name())) + } + _, ok := lccache.GetCachedCommitID(req.CommitID, path.Join(req.TreePath, entry.Name())) + if !ok { + entryPaths = append(entryPaths, entry.Name()) + } + } + + if len(entryPaths) == 0 { + return directories, nil + } + + commits, err := git.GetLastCommitForPaths(graceful.GetManager().HammerContext(), commitNode, req.TreePath, entryPaths) + if err != nil { + return nil, err + } + + for entryPath, entryCommit := range commits { + if err := lccache.Put(commit.ID.String(), path.Join(req.TreePath, entryPath), entryCommit.ID().String()); err != nil { + return nil, err + } + } + + return directories, nil +} + +func handle(data ...queue.Data) { + for _, datum := range data { + req := datum.(*CommitCacheRequest) + if err := req.Do(); err != nil { + log.Error("Unable to process commit cache request for %s:%s:%s:%t: %v", req.Repo, req.CommitID, req.TreePath, req.Recursive, err) + } + } +} + +// Init initialises the queue +func Init() error { + lastCommitQueue = queue.CreateUniqueQueue("last_commit_queue", handle, &CommitCacheRequest{}).(queue.UniqueQueue) + + return nil +} + +// UpdateCache queues the the request +func UpdateCache(repo, commitID, treePath string, recursive bool) error { + return lastCommitQueue.Push(&CommitCacheRequest{ + Repo: repo, + CommitID: commitID, + TreePath: treePath, + Recursive: recursive, + }) +} diff --git a/modules/repository/last_commit_queue_nogogit.go b/modules/repository/last_commit_queue_nogogit.go new file mode 100644 index 0000000000000..24c360f7a31f1 --- /dev/null +++ b/modules/repository/last_commit_queue_nogogit.go @@ -0,0 +1,156 @@ +// Copyright 2019 Gitea. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build !gogit + +package repository + +import ( + "path" + "path/filepath" + "sync" + + "code.gitea.io/gitea/modules/cache" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/graceful" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/queue" + "code.gitea.io/gitea/modules/setting" +) + +var lock = sync.Mutex{} +var table = map[CommitCacheRequest]bool{} +var lastCommitQueue queue.UniqueQueue + +// CommitCacheRequest represents a cache request +type CommitCacheRequest struct { + Repo string + CommitID string + TreePath string + Recursive bool +} + +// Do runs the cache request uniquely ensuring that only one cache request is running for this request triple +func (req *CommitCacheRequest) Do() error { + recursive := req.Recursive + req.Recursive = false + + repo, err := git.OpenRepository(filepath.Join(setting.RepoRootPath, req.Repo+".git")) + if err != nil { + return err + } + commit, err := repo.GetCommit(req.CommitID) + if err != nil { + if git.IsErrNotExist(err) { + return nil + } + return err + } + + lccache := git.NewLastCommitCache(req.Repo, repo, setting.LastCommitCacheTTLSeconds, cache.GetCache()) + + directories := []string{req.TreePath} + for len(directories) > 0 { + req.TreePath = directories[len(directories)-1] + next, err := req.doTree(repo, commit, recursive, lccache) + if err != nil { + return err + } + directories = append(next, directories[:len(directories)-1]...) + } + return nil +} + +func (req *CommitCacheRequest) doTree(repo *git.Repository, commit *git.Commit, recursive bool, lccache *git.LastCommitCache) ([]string, error) { + + tree, err := commit.Tree.SubTree(req.TreePath) + if err != nil { + if git.IsErrNotExist(err) { + return nil, nil + } + return nil, err + } + entries, err := tree.ListEntries() + if err != nil { + if git.IsErrNotExist(err) { + return nil, nil + } + return nil, err + } + directories := make([]string, 0, len(entries)) + + lock.Lock() + if has := table[*req]; has { + lock.Unlock() + if recursive { + for _, entry := range entries { + if entry.IsDir() { + directories = append(directories, path.Join(req.TreePath, entry.Name())) + } + } + } + return directories, nil + } + table[*req] = true + lock.Unlock() + defer func() { + lock.Lock() + delete(table, *req) + lock.Unlock() + }() + + entryPaths := make([]string, 0, len(entries)) + for _, entry := range entries { + if recursive && entry.IsDir() { + directories = append(directories, path.Join(req.TreePath, entry.Name())) + } + _, ok := lccache.GetCachedCommitID(req.CommitID, path.Join(req.TreePath, entry.Name())) + if !ok { + entryPaths = append(entryPaths, entry.Name()) + } + } + + if len(entryPaths) == 0 { + return directories, nil + } + + commits, err := git.GetLastCommitForPaths(graceful.GetManager().HammerContext(), commit, req.TreePath, entryPaths) + if err != nil { + return nil, err + } + + for i, entryCommit := range commits { + if err := lccache.Put(commit.ID.String(), path.Join(req.TreePath, entryPaths[i]), entryCommit.ID.String()); err != nil { + return nil, err + } + } + + return directories, nil +} + +func handle(data ...queue.Data) { + for _, datum := range data { + req := datum.(*CommitCacheRequest) + if err := req.Do(); err != nil { + log.Error("Unable to process commit cache request for %s:%s:%s:%t: %v", req.Repo, req.CommitID, req.TreePath, req.Recursive, err) + } + } +} + +// Init initialises the queue +func Init() error { + lastCommitQueue = queue.CreateUniqueQueue("last_commit_queue", handle, &CommitCacheRequest{}).(queue.UniqueQueue) + + return nil +} + +// UpdateCache queues the the request +func UpdateCache(repo, commitID, treePath string, recursive bool) error { + return lastCommitQueue.Push(&CommitCacheRequest{ + Repo: repo, + CommitID: commitID, + TreePath: treePath, + Recursive: recursive, + }) +} diff --git a/routers/init.go b/routers/init.go index 4c28a953955ba..19db43fe54e5b 100644 --- a/routers/init.go +++ b/routers/init.go @@ -22,6 +22,7 @@ import ( "code.gitea.io/gitea/modules/markup/external" repo_migrations "code.gitea.io/gitea/modules/migrations" "code.gitea.io/gitea/modules/notification" + repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/ssh" "code.gitea.io/gitea/modules/storage" @@ -123,6 +124,9 @@ func GlobalInit(ctx context.Context) { if err := task.Init(); err != nil { log.Fatal("Failed to initialize task scheduler: %v", err) } + if err := repo_module.Init(); err != nil { + log.Fatal("Failed to initialize last commit scheduler: %v", err) + } if err := repo_migrations.Init(); err != nil { log.Fatal("Failed to initialize repository migrations: %v", err) } diff --git a/routers/web/repo/commit.go b/routers/web/repo/commit.go index 3e6148bcbbc76..65c172ca8f59c 100644 --- a/routers/web/repo/commit.go +++ b/routers/web/repo/commit.go @@ -358,8 +358,10 @@ func Diff(ctx *context.Context) { err = git.GetNote(ctx, ctx.Repo.GitRepo, commitID, note) if err == nil { ctx.Data["Note"] = string(charset.ToUTF8WithFallback(note.Message)) - ctx.Data["NoteCommit"] = note.Commit - ctx.Data["NoteAuthor"] = models.ValidateCommitWithEmail(note.Commit) + if note.Commit != nil { + ctx.Data["NoteCommit"] = note.Commit + ctx.Data["NoteAuthor"] = models.ValidateCommitWithEmail(note.Commit) + } } ctx.Data["BranchName"], err = commit.GetBranchName() diff --git a/routers/web/repo/view.go b/routers/web/repo/view.go index cd5b0f43edbc7..7ba68a9fb27c7 100644 --- a/routers/web/repo/view.go +++ b/routers/web/repo/view.go @@ -28,6 +28,7 @@ import ( "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markup" + "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/typesniffer" ) @@ -143,6 +144,11 @@ func renderDirectory(ctx *context.Context, treeLink string) { var c *git.LastCommitCache if setting.CacheService.LastCommit.Enabled && ctx.Repo.CommitsCount >= setting.CacheService.LastCommit.CommitsCount { c = git.NewLastCommitCache(ctx.Repo.Repository.FullName(), ctx.Repo.GitRepo, setting.LastCommitCacheTTLSeconds, cache.GetCache()) + err := repository.UpdateCache(ctx.Repo.Repository.FullName(), ctx.Repo.Commit.ID.String(), ctx.Repo.TreePath, false) + if err != nil { + ctx.ServerError("UpdateCache", err) + return + } } var latestCommit *git.Commit @@ -152,6 +158,10 @@ func renderDirectory(ctx *context.Context, treeLink string) { return } + if latestCommit == nil { + latestCommit = ctx.Repo.Commit + } + // 3 for the extensions in exts[] in order // the last one is for a readme that doesn't // strictly match an extension diff --git a/templates/repo/commit_page.tmpl b/templates/repo/commit_page.tmpl index 01cbd5182dc66..b9080296219fa 100644 --- a/templates/repo/commit_page.tmpl +++ b/templates/repo/commit_page.tmpl @@ -127,7 +127,9 @@
{{.i18n.Tr "repo.diff.git-notes"}}: - {{if .NoteAuthor}} + {{if not .NoteCommit}} +
+ {{else if .NoteAuthor}} {{if .NoteAuthor.FullName}} {{.NoteAuthor.FullName}} @@ -138,7 +140,9 @@ {{else}} {{.NoteCommit.Author.Name}} {{end}} - {{TimeSince .NoteCommit.Author.When $.Lang}} + {{if .NoteCommit}} + {{TimeSince .NoteCommit.Author.When $.Lang}} + {{end}} 
{{RenderNote .Note $.RepoLink $.Repository.ComposeMetas}}
diff --git a/templates/repo/view_list.tmpl b/templates/repo/view_list.tmpl index fc66fb6b2d5b7..6098e7022ea04 100644 --- a/templates/repo/view_list.tmpl +++ b/templates/repo/view_list.tmpl @@ -75,10 +75,18 @@ - {{$commit.Summary | RenderEmoji}} + {{if $commit}} + {{$commit.Summary | RenderEmoji}} + {{else}} +
+ {{end}} - {{TimeSince $commit.Committer.When $.Lang}} + + {{if $commit}} + {{TimeSince $commit.Committer.When $.Lang}} + {{end}} + {{end}} diff --git a/web_src/js/index.js b/web_src/js/index.js index 8818511e32cf2..d6023a127a0e9 100644 --- a/web_src/js/index.js +++ b/web_src/js/index.js @@ -2745,6 +2745,10 @@ $(document).ready(async () => { } }); + if ($('.reload.loader').length > 0) { + setTimeout(reload, 500); + } + buttonsClickOnEnter(); searchUsers(); searchTeams(); From 10b56e1aea7039065ef3e9afafcafd01e3b1f7cc Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Wed, 9 Jun 2021 19:54:33 +0100 Subject: [PATCH 2/6] Make Queue requests cancelable Signed-off-by: Andrew Thornton --- modules/process/manager.go | 11 +++++++++++ modules/repository/last_commit_queue_gogit.go | 19 +++++++++++++++---- .../repository/last_commit_queue_nogogit.go | 18 +++++++++++++++--- 3 files changed, 41 insertions(+), 7 deletions(-) diff --git a/modules/process/manager.go b/modules/process/manager.go index e42e38a0f02a4..127387bd58756 100644 --- a/modules/process/manager.go +++ b/modules/process/manager.go @@ -70,6 +70,17 @@ func (pm *Manager) Add(description string, cancel context.CancelFunc) int64 { return pid } +// AddContext adds a process to the ProcessManager using a base context and returns a context, cancel func and its PID +func (pm *Manager) AddContext(baseCtx context.Context, description string) (context.Context, context.CancelFunc, int64) { + ctx, cancel := context.WithCancel(baseCtx) + pid := pm.Add(description, cancel) + + return ctx, func() { + defer pm.Remove(pid) + cancel() + }, pid +} + // Remove a process from the ProcessManager. func (pm *Manager) Remove(pid int64) { pm.mutex.Lock() diff --git a/modules/repository/last_commit_queue_gogit.go b/modules/repository/last_commit_queue_gogit.go index 32277a4ee684e..3cf1a88334eac 100644 --- a/modules/repository/last_commit_queue_gogit.go +++ b/modules/repository/last_commit_queue_gogit.go @@ -7,6 +7,8 @@ package repository import ( + "context" + "fmt" "path" "path/filepath" "sync" @@ -15,6 +17,7 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/queue" "code.gitea.io/gitea/modules/setting" ) @@ -33,6 +36,9 @@ type CommitCacheRequest struct { // Do runs the cache request uniquely ensuring that only one cache request is running for this request triple func (req *CommitCacheRequest) Do() error { + ctx, cancel, _ := process.GetManager().AddContext(graceful.GetManager().HammerContext(), fmt.Sprintf("Cache: %s:%s:%s:%t", req.Repo, req.CommitID, req.TreePath, req.Recursive)) + defer cancel() + recursive := req.Recursive req.Recursive = false @@ -52,8 +58,14 @@ func (req *CommitCacheRequest) Do() error { directories := []string{req.TreePath} for len(directories) > 0 { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + req.TreePath = directories[len(directories)-1] - next, err := req.doTree(repo, commit, recursive, lccache) + next, err := req.doTree(ctx, repo, commit, recursive, lccache) if err != nil { return err } @@ -62,8 +74,7 @@ func (req *CommitCacheRequest) Do() error { return nil } -func (req *CommitCacheRequest) doTree(repo *git.Repository, commit *git.Commit, recursive bool, lccache *git.LastCommitCache) ([]string, error) { - +func (req *CommitCacheRequest) doTree(ctx context.Context, repo *git.Repository, commit *git.Commit, recursive bool, lccache *git.LastCommitCache) ([]string, error) { tree, err := commit.Tree.SubTree(req.TreePath) if err != nil { if git.IsErrNotExist(err) { @@ -125,7 +136,7 @@ func (req *CommitCacheRequest) doTree(repo *git.Repository, commit *git.Commit, return directories, nil } - commits, err := git.GetLastCommitForPaths(graceful.GetManager().HammerContext(), commitNode, req.TreePath, entryPaths) + commits, err := git.GetLastCommitForPaths(ctx, commitNode, req.TreePath, entryPaths) if err != nil { return nil, err } diff --git a/modules/repository/last_commit_queue_nogogit.go b/modules/repository/last_commit_queue_nogogit.go index 24c360f7a31f1..3dad81d0cc4f0 100644 --- a/modules/repository/last_commit_queue_nogogit.go +++ b/modules/repository/last_commit_queue_nogogit.go @@ -7,6 +7,8 @@ package repository import ( + "context" + "fmt" "path" "path/filepath" "sync" @@ -15,6 +17,7 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/queue" "code.gitea.io/gitea/modules/setting" ) @@ -33,6 +36,9 @@ type CommitCacheRequest struct { // Do runs the cache request uniquely ensuring that only one cache request is running for this request triple func (req *CommitCacheRequest) Do() error { + ctx, cancel, _ := process.GetManager().AddContext(graceful.GetManager().HammerContext(), fmt.Sprintf("Cache: %s:%s:%s:%t", req.Repo, req.CommitID, req.TreePath, req.Recursive)) + defer cancel() + recursive := req.Recursive req.Recursive = false @@ -52,8 +58,14 @@ func (req *CommitCacheRequest) Do() error { directories := []string{req.TreePath} for len(directories) > 0 { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + req.TreePath = directories[len(directories)-1] - next, err := req.doTree(repo, commit, recursive, lccache) + next, err := req.doTree(ctx, repo, commit, recursive, lccache) if err != nil { return err } @@ -62,7 +74,7 @@ func (req *CommitCacheRequest) Do() error { return nil } -func (req *CommitCacheRequest) doTree(repo *git.Repository, commit *git.Commit, recursive bool, lccache *git.LastCommitCache) ([]string, error) { +func (req *CommitCacheRequest) doTree(ctx context.Context, repo *git.Repository, commit *git.Commit, recursive bool, lccache *git.LastCommitCache) ([]string, error) { tree, err := commit.Tree.SubTree(req.TreePath) if err != nil { @@ -115,7 +127,7 @@ func (req *CommitCacheRequest) doTree(repo *git.Repository, commit *git.Commit, return directories, nil } - commits, err := git.GetLastCommitForPaths(graceful.GetManager().HammerContext(), commit, req.TreePath, entryPaths) + commits, err := git.GetLastCommitForPaths(ctx, commit, req.TreePath, entryPaths) if err != nil { return nil, err } From 67b817e9f52a1e5390f0591ac7e9bd21fc846fee Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Wed, 9 Jun 2021 21:44:32 +0100 Subject: [PATCH 3/6] reload in two seconds instead Signed-off-by: Andrew Thornton --- web_src/js/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web_src/js/index.js b/web_src/js/index.js index d6023a127a0e9..b29cc511b7741 100644 --- a/web_src/js/index.js +++ b/web_src/js/index.js @@ -2746,7 +2746,7 @@ $(document).ready(async () => { }); if ($('.reload.loader').length > 0) { - setTimeout(reload, 500); + setTimeout(reload, 2000); } buttonsClickOnEnter(); From 44b42543ecd07b41b5b0af9d2db2fe5134265d42 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Mon, 14 Jun 2021 17:13:40 +0100 Subject: [PATCH 4/6] fix copyright notice Signed-off-by: Andrew Thornton --- modules/repository/last_commit_queue_gogit.go | 2 +- modules/repository/last_commit_queue_nogogit.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/repository/last_commit_queue_gogit.go b/modules/repository/last_commit_queue_gogit.go index 3cf1a88334eac..a73c01f616753 100644 --- a/modules/repository/last_commit_queue_gogit.go +++ b/modules/repository/last_commit_queue_gogit.go @@ -1,4 +1,4 @@ -// Copyright 2019 Gitea. All rights reserved. +// Copyright 2021 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. diff --git a/modules/repository/last_commit_queue_nogogit.go b/modules/repository/last_commit_queue_nogogit.go index 3dad81d0cc4f0..0af4a948e6c0b 100644 --- a/modules/repository/last_commit_queue_nogogit.go +++ b/modules/repository/last_commit_queue_nogogit.go @@ -1,4 +1,4 @@ -// Copyright 2019 Gitea. All rights reserved. +// Copyright 2021 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. From dcd739784124b5d241b8049d0557eaa834f08c3c Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Wed, 16 Jun 2021 22:11:47 +0100 Subject: [PATCH 5/6] default to use a level queue Signed-off-by: Andrew Thornton --- modules/setting/queue.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/modules/setting/queue.go b/modules/setting/queue.go index 41e95fbe5a5e7..d243ed1e80ea4 100644 --- a/modules/setting/queue.go +++ b/modules/setting/queue.go @@ -98,6 +98,7 @@ func NewQueueService() { Queue.QueueLength = sec.Key("LENGTH").MustInt(20) Queue.BatchLength = sec.Key("BATCH_LENGTH").MustInt(20) Queue.ConnectionString = sec.Key("CONN_STR").MustString("") + defaultType := sec.Key("TYPE").String() Queue.Type = sec.Key("TYPE").MustString("persistable-channel") Queue.Network, Queue.Addresses, Queue.Password, Queue.DBIndex, _ = ParseQueueConnStr(Queue.ConnectionString) Queue.WrapIfNecessary = sec.Key("WRAP_IF_NECESSARY").MustBool(true) @@ -143,6 +144,16 @@ func NewQueueService() { _, _ = section.NewKey("CONN_STR", Indexer.IssueQueueConnStr) } + // Default the last_commit_queue to use the level queue + section = Cfg.Section("queue.issue_indexer") + sectionMap = map[string]bool{} + for _, key := range section.Keys() { + sectionMap[key.Name()] = true + } + if _, ok := sectionMap["TYPE"]; !ok && defaultType == "" { + _, _ = section.NewKey("TYPE", "level") + } + // Handle the old mailer configuration section = Cfg.Section("queue.mailer") sectionMap = map[string]bool{} From bcded8a80b402d01bbaf5799bdfef139918bd329 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Mon, 21 Jun 2021 17:41:30 +0100 Subject: [PATCH 6/6] fixup! Merge remote-tracking branch 'origin/main' into defer-last-commit-info --- modules/repository/last_commit_queue_nogogit.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/repository/last_commit_queue_nogogit.go b/modules/repository/last_commit_queue_nogogit.go index 0af4a948e6c0b..d1b7bb695a0e6 100644 --- a/modules/repository/last_commit_queue_nogogit.go +++ b/modules/repository/last_commit_queue_nogogit.go @@ -132,8 +132,8 @@ func (req *CommitCacheRequest) doTree(ctx context.Context, repo *git.Repository, return nil, err } - for i, entryCommit := range commits { - if err := lccache.Put(commit.ID.String(), path.Join(req.TreePath, entryPaths[i]), entryCommit.ID.String()); err != nil { + for pth, entryCommit := range commits { + if err := lccache.Put(commit.ID.String(), path.Join(req.TreePath, pth), entryCommit.ID.String()); err != nil { return nil, err } }