Skip to content

Issue #1076: load commit cache in a background thread #1140

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Oct 27, 2016
Merged

Issue #1076: load commit cache in a background thread #1140

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 35 additions & 27 deletions src/main/java/com/gitblit/manager/RepositoryManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1952,39 +1952,47 @@ protected void configureJGit() {
}

protected void configureCommitCache() {
int daysToCache = settings.getInteger(Keys.web.activityCacheDays, 14);
final int daysToCache = settings.getInteger(Keys.web.activityCacheDays, 14);
if (daysToCache <= 0) {
logger.info("Commit cache is disabled");
} else {
long start = System.nanoTime();
long repoCount = 0;
long commitCount = 0;
logger.info(MessageFormat.format("Preparing {0} day commit cache. please wait...", daysToCache));
CommitCache.instance().setCacheDays(daysToCache);
Date cutoff = CommitCache.instance().getCutoffDate();
for (String repositoryName : getRepositoryList()) {
RepositoryModel model = getRepositoryModel(repositoryName);
if (model != null && model.hasCommits && model.lastChange.after(cutoff)) {
repoCount++;
Repository repository = getRepository(repositoryName);
for (RefModel ref : JGitUtils.getLocalBranches(repository, true, -1)) {
if (!ref.getDate().after(cutoff)) {
// branch not recently updated
continue;
}
List<?> commits = CommitCache.instance().getCommits(repositoryName, repository, ref.getName());
if (commits.size() > 0) {
logger.info(MessageFormat.format(" cached {0} commits for {1}:{2}",
commits.size(), repositoryName, ref.getName()));
commitCount += commits.size();
return;
}
logger.info(MessageFormat.format("Preparing {0} day commit cache...", daysToCache));
CommitCache.instance().setCacheDays(daysToCache);
Thread loader = new Thread() {
@Override
public void run() {
long start = System.nanoTime();
long repoCount = 0;
long commitCount = 0;
Date cutoff = CommitCache.instance().getCutoffDate();
for (String repositoryName : getRepositoryList()) {
RepositoryModel model = getRepositoryModel(repositoryName);
if (model != null && model.hasCommits && model.lastChange.after(cutoff)) {
repoCount++;
Repository repository = getRepository(repositoryName);
for (RefModel ref : JGitUtils.getLocalBranches(repository, true, -1)) {
if (!ref.getDate().after(cutoff)) {
// branch not recently updated
continue;
}
List<?> commits = CommitCache.instance().getCommits(repositoryName, repository, ref.getName());
if (commits.size() > 0) {
logger.info(MessageFormat.format(" cached {0} commits for {1}:{2}",
commits.size(), repositoryName, ref.getName()));
commitCount += commits.size();
}
}
repository.close();
}
repository.close();
}
logger.info(MessageFormat.format("built {0} day commit cache of {1} commits across {2} repositories in {3} msecs",
daysToCache, commitCount, repoCount, TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start)));
}
logger.info(MessageFormat.format("built {0} day commit cache of {1} commits across {2} repositories in {3} msecs",
daysToCache, commitCount, repoCount, TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start)));
}
};
loader.setName("CommitCacheLoader");
loader.setDaemon(true);
loader.start();
}

protected void confirmWriteAccess() {
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/com/gitblit/utils/ArrayUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ public static boolean isEmpty(Object [] array) {
}

public static boolean isEmpty(Collection<?> collection) {
return collection == null || collection.size() == 0;
return collection == null || collection.isEmpty();
}

public static String toString(Collection<?> collection) {
Expand Down
117 changes: 67 additions & 50 deletions src/main/java/com/gitblit/utils/CommitCache.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,9 @@
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;

import org.eclipse.jgit.lib.ObjectId;
Expand Down Expand Up @@ -58,7 +58,7 @@ public static CommitCache instance() {
}

protected CommitCache() {
cache = new ConcurrentHashMap<String, ObjectCache<List<RepositoryCommit>>>();
cache = new HashMap<>();
}

/**
Expand Down Expand Up @@ -93,7 +93,9 @@ public synchronized void setCacheDays(int days) {
*
*/
public void clear() {
cache.clear();
synchronized (cache) {
cache.clear();
}
}

/**
Expand All @@ -103,8 +105,11 @@ public void clear() {
*/
public void clear(String repositoryName) {
String repoKey = repositoryName.toLowerCase();
ObjectCache<List<RepositoryCommit>> repoCache = cache.remove(repoKey);
if (repoCache != null) {
boolean hadEntries = false;
synchronized (cache) {
hadEntries = cache.remove(repoKey) != null;
}
if (hadEntries) {
logger.info(MessageFormat.format("{0} commit cache cleared", repositoryName));
}
}
Expand All @@ -117,13 +122,17 @@ public void clear(String repositoryName) {
*/
public void clear(String repositoryName, String branch) {
String repoKey = repositoryName.toLowerCase();
ObjectCache<List<RepositoryCommit>> repoCache = cache.get(repoKey);
if (repoCache != null) {
List<RepositoryCommit> commits = repoCache.remove(branch.toLowerCase());
if (!ArrayUtils.isEmpty(commits)) {
logger.info(MessageFormat.format("{0}:{1} commit cache cleared", repositoryName, branch));
boolean hadEntries = false;
synchronized (cache) {
ObjectCache<List<RepositoryCommit>> repoCache = cache.get(repoKey);
if (repoCache != null) {
List<RepositoryCommit> commits = repoCache.remove(branch.toLowerCase());
hadEntries = !ArrayUtils.isEmpty(commits);
}
}
if (hadEntries) {
logger.info(MessageFormat.format("{0}:{1} commit cache cleared", repositoryName, branch));
}
}

/**
Expand Down Expand Up @@ -156,49 +165,55 @@ public List<RepositoryCommit> getCommits(String repositoryName, Repository repos
if (cacheDays > 0 && (sinceDate.getTime() >= cacheCutoffDate.getTime())) {
// request fits within the cache window
String repoKey = repositoryName.toLowerCase();
if (!cache.containsKey(repoKey)) {
cache.put(repoKey, new ObjectCache<List<RepositoryCommit>>());
}

ObjectCache<List<RepositoryCommit>> repoCache = cache.get(repoKey);
String branchKey = branch.toLowerCase();

RevCommit tip = JGitUtils.getCommit(repository, branch);
Date tipDate = JGitUtils.getCommitDate(tip);

List<RepositoryCommit> commits;
if (!repoCache.hasCurrent(branchKey, tipDate)) {
commits = repoCache.getObject(branchKey);
if (ArrayUtils.isEmpty(commits)) {
// we don't have any cached commits for this branch, reload
commits = get(repositoryName, repository, branch, cacheCutoffDate);
repoCache.updateObject(branchKey, tipDate, commits);
logger.debug(MessageFormat.format("parsed {0} commits from {1}:{2} since {3,date,yyyy-MM-dd} in {4} msecs",
commits.size(), repositoryName, branch, cacheCutoffDate, TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start)));
} else {
// incrementally update cache since the last cached commit
ObjectId sinceCommit = commits.get(0).getId();
List<RepositoryCommit> incremental = get(repositoryName, repository, branch, sinceCommit);
logger.info(MessageFormat.format("incrementally added {0} commits to cache for {1}:{2} in {3} msecs",
incremental.size(), repositoryName, branch, TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start)));
incremental.addAll(commits);
repoCache.updateObject(branchKey, tipDate, incremental);
commits = incremental;
ObjectCache<List<RepositoryCommit>> repoCache;
synchronized (cache) {
repoCache = cache.get(repoKey);
if (repoCache == null) {
repoCache = new ObjectCache<>();
cache.put(repoKey, repoCache);
}
} else {
// cache is current
commits = repoCache.getObject(branchKey);
// evict older commits outside the cache window
commits = reduce(commits, cacheCutoffDate);
// update cache
repoCache.updateObject(branchKey, tipDate, commits);
}
synchronized (repoCache) {
List<RepositoryCommit> commits;
if (!repoCache.hasCurrent(branchKey, tipDate)) {
commits = repoCache.getObject(branchKey);
if (ArrayUtils.isEmpty(commits)) {
// we don't have any cached commits for this branch, reload
commits = get(repositoryName, repository, branch, cacheCutoffDate);
repoCache.updateObject(branchKey, tipDate, commits);
logger.debug(MessageFormat.format("parsed {0} commits from {1}:{2} since {3,date,yyyy-MM-dd} in {4} msecs",
commits.size(), repositoryName, branch, cacheCutoffDate, TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start)));
} else {
// incrementally update cache since the last cached commit
ObjectId sinceCommit = commits.get(0).getId();
List<RepositoryCommit> incremental = get(repositoryName, repository, branch, sinceCommit);
logger.info(MessageFormat.format("incrementally added {0} commits to cache for {1}:{2} in {3} msecs",
incremental.size(), repositoryName, branch, TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start)));
incremental.addAll(commits);
repoCache.updateObject(branchKey, tipDate, incremental);
commits = incremental;
}
} else {
// cache is current
commits = repoCache.getObject(branchKey);
// evict older commits outside the cache window
commits = reduce(commits, cacheCutoffDate);
// update cache
repoCache.updateObject(branchKey, tipDate, commits);
}

if (sinceDate.equals(cacheCutoffDate)) {
list = commits;
} else {
// reduce the commits to those since the specified date
list = reduce(commits, sinceDate);
if (sinceDate.equals(cacheCutoffDate)) {
// Mustn't hand out the cached list; that's not thread-safe
list = new ArrayList<>(commits);
} else {
// reduce the commits to those since the specified date
list = reduce(commits, sinceDate);
}
}
logger.debug(MessageFormat.format("retrieved {0} commits from cache of {1}:{2} since {3,date,yyyy-MM-dd} in {4} msecs",
list.size(), repositoryName, branch, sinceDate, TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start)));
Expand All @@ -222,8 +237,9 @@ public List<RepositoryCommit> getCommits(String repositoryName, Repository repos
*/
protected List<RepositoryCommit> get(String repositoryName, Repository repository, String branch, Date sinceDate) {
Map<ObjectId, List<RefModel>> allRefs = JGitUtils.getAllRefs(repository, false);
List<RepositoryCommit> commits = new ArrayList<RepositoryCommit>();
for (RevCommit commit : JGitUtils.getRevLog(repository, branch, sinceDate)) {
List<RevCommit> revLog = JGitUtils.getRevLog(repository, branch, sinceDate);
List<RepositoryCommit> commits = new ArrayList<RepositoryCommit>(revLog.size());
for (RevCommit commit : revLog) {
RepositoryCommit commitModel = new RepositoryCommit(repositoryName, branch, commit);
List<RefModel> commitRefs = allRefs.get(commitModel.getId());
commitModel.setRefs(commitRefs);
Expand All @@ -243,8 +259,9 @@ protected List<RepositoryCommit> get(String repositoryName, Repository repositor
*/
protected List<RepositoryCommit> get(String repositoryName, Repository repository, String branch, ObjectId sinceCommit) {
Map<ObjectId, List<RefModel>> allRefs = JGitUtils.getAllRefs(repository, false);
List<RepositoryCommit> commits = new ArrayList<RepositoryCommit>();
for (RevCommit commit : JGitUtils.getRevLog(repository, sinceCommit.getName(), branch)) {
List<RevCommit> revLog = JGitUtils.getRevLog(repository, sinceCommit.getName(), branch);
List<RepositoryCommit> commits = new ArrayList<RepositoryCommit>(revLog.size());
for (RevCommit commit : revLog) {
RepositoryCommit commitModel = new RepositoryCommit(repositoryName, branch, commit);
List<RefModel> commitRefs = allRefs.get(commitModel.getId());
commitModel.setRefs(commitRefs);
Expand All @@ -261,7 +278,7 @@ protected List<RepositoryCommit> get(String repositoryName, Repository repositor
* @return a list of commits
*/
protected List<RepositoryCommit> reduce(List<RepositoryCommit> commits, Date sinceDate) {
List<RepositoryCommit> filtered = new ArrayList<RepositoryCommit>();
List<RepositoryCommit> filtered = new ArrayList<RepositoryCommit>(commits.size());
for (RepositoryCommit commit : commits) {
if (commit.getCommitDate().compareTo(sinceDate) >= 0) {
filtered.add(commit);
Expand Down