Skip to content

Commit bac1241

Browse files
gitblitgitblit
authored
Merge pull request #1140 from tomaswolf/issue_1076
Issue #1076: load commit cache in a background thread
2 parents a3665a6 + de3f435 commit bac1241

File tree

3 files changed

+103
-78
lines changed

3 files changed

+103
-78
lines changed

‎src/main/java/com/gitblit/manager/RepositoryManager.java

Lines changed: 35 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1952,39 +1952,47 @@ protected void configureJGit() {
19521952
}
19531953

19541954
protected void configureCommitCache() {
1955-
int daysToCache = settings.getInteger(Keys.web.activityCacheDays, 14);
1955+
final int daysToCache = settings.getInteger(Keys.web.activityCacheDays, 14);
19561956
if (daysToCache <= 0) {
19571957
logger.info("Commit cache is disabled");
1958-
} else {
1959-
long start = System.nanoTime();
1960-
long repoCount = 0;
1961-
long commitCount = 0;
1962-
logger.info(MessageFormat.format("Preparing {0} day commit cache. please wait...", daysToCache));
1963-
CommitCache.instance().setCacheDays(daysToCache);
1964-
Date cutoff = CommitCache.instance().getCutoffDate();
1965-
for (String repositoryName : getRepositoryList()) {
1966-
RepositoryModel model = getRepositoryModel(repositoryName);
1967-
if (model != null && model.hasCommits && model.lastChange.after(cutoff)) {
1968-
repoCount++;
1969-
Repository repository = getRepository(repositoryName);
1970-
for (RefModel ref : JGitUtils.getLocalBranches(repository, true, -1)) {
1971-
if (!ref.getDate().after(cutoff)) {
1972-
// branch not recently updated
1973-
continue;
1974-
}
1975-
List<?> commits = CommitCache.instance().getCommits(repositoryName, repository, ref.getName());
1976-
if (commits.size() > 0) {
1977-
logger.info(MessageFormat.format(" cached {0} commits for {1}:{2}",
1978-
commits.size(), repositoryName, ref.getName()));
1979-
commitCount += commits.size();
1958+
return;
1959+
}
1960+
logger.info(MessageFormat.format("Preparing {0} day commit cache...", daysToCache));
1961+
CommitCache.instance().setCacheDays(daysToCache);
1962+
Thread loader = new Thread() {
1963+
@Override
1964+
public void run() {
1965+
long start = System.nanoTime();
1966+
long repoCount = 0;
1967+
long commitCount = 0;
1968+
Date cutoff = CommitCache.instance().getCutoffDate();
1969+
for (String repositoryName : getRepositoryList()) {
1970+
RepositoryModel model = getRepositoryModel(repositoryName);
1971+
if (model != null && model.hasCommits && model.lastChange.after(cutoff)) {
1972+
repoCount++;
1973+
Repository repository = getRepository(repositoryName);
1974+
for (RefModel ref : JGitUtils.getLocalBranches(repository, true, -1)) {
1975+
if (!ref.getDate().after(cutoff)) {
1976+
// branch not recently updated
1977+
continue;
1978+
}
1979+
List<?> commits = CommitCache.instance().getCommits(repositoryName, repository, ref.getName());
1980+
if (commits.size() > 0) {
1981+
logger.info(MessageFormat.format(" cached {0} commits for {1}:{2}",
1982+
commits.size(), repositoryName, ref.getName()));
1983+
commitCount += commits.size();
1984+
}
19801985
}
1986+
repository.close();
19811987
}
1982-
repository.close();
19831988
}
1989+
logger.info(MessageFormat.format("built {0} day commit cache of {1} commits across {2} repositories in {3} msecs",
1990+
daysToCache, commitCount, repoCount, TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start)));
19841991
}
1985-
logger.info(MessageFormat.format("built {0} day commit cache of {1} commits across {2} repositories in {3} msecs",
1986-
daysToCache, commitCount, repoCount, TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start)));
1987-
}
1992+
};
1993+
loader.setName("CommitCacheLoader");
1994+
loader.setDaemon(true);
1995+
loader.start();
19881996
}
19891997

19901998
protected void confirmWriteAccess() {

‎src/main/java/com/gitblit/utils/ArrayUtils.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ public static boolean isEmpty(Object [] array) {
4242
}
4343

4444
public static boolean isEmpty(Collection<?> collection) {
45-
return collection == null || collection.size() == 0;
45+
return collection == null || collection.isEmpty();
4646
}
4747

4848
public static String toString(Collection<?> collection) {

‎src/main/java/com/gitblit/utils/CommitCache.java

Lines changed: 67 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,9 @@
1919
import java.util.ArrayList;
2020
import java.util.Calendar;
2121
import java.util.Date;
22+
import java.util.HashMap;
2223
import java.util.List;
2324
import java.util.Map;
24-
import java.util.concurrent.ConcurrentHashMap;
2525
import java.util.concurrent.TimeUnit;
2626

2727
import org.eclipse.jgit.lib.ObjectId;
@@ -58,7 +58,7 @@ public static CommitCache instance() {
5858
}
5959

6060
protected CommitCache() {
61-
cache = new ConcurrentHashMap<String, ObjectCache<List<RepositoryCommit>>>();
61+
cache = new HashMap<>();
6262
}
6363

6464
/**
@@ -93,7 +93,9 @@ public synchronized void setCacheDays(int days) {
9393
*
9494
*/
9595
public void clear() {
96-
cache.clear();
96+
synchronized (cache) {
97+
cache.clear();
98+
}
9799
}
98100

99101
/**
@@ -103,8 +105,11 @@ public void clear() {
103105
*/
104106
public void clear(String repositoryName) {
105107
String repoKey = repositoryName.toLowerCase();
106-
ObjectCache<List<RepositoryCommit>> repoCache = cache.remove(repoKey);
107-
if (repoCache != null) {
108+
boolean hadEntries = false;
109+
synchronized (cache) {
110+
hadEntries = cache.remove(repoKey) != null;
111+
}
112+
if (hadEntries) {
108113
logger.info(MessageFormat.format("{0} commit cache cleared", repositoryName));
109114
}
110115
}
@@ -117,13 +122,17 @@ public void clear(String repositoryName) {
117122
*/
118123
public void clear(String repositoryName, String branch) {
119124
String repoKey = repositoryName.toLowerCase();
120-
ObjectCache<List<RepositoryCommit>> repoCache = cache.get(repoKey);
121-
if (repoCache != null) {
122-
List<RepositoryCommit> commits = repoCache.remove(branch.toLowerCase());
123-
if (!ArrayUtils.isEmpty(commits)) {
124-
logger.info(MessageFormat.format("{0}:{1} commit cache cleared", repositoryName, branch));
125+
boolean hadEntries = false;
126+
synchronized (cache) {
127+
ObjectCache<List<RepositoryCommit>> repoCache = cache.get(repoKey);
128+
if (repoCache != null) {
129+
List<RepositoryCommit> commits = repoCache.remove(branch.toLowerCase());
130+
hadEntries = !ArrayUtils.isEmpty(commits);
125131
}
126132
}
133+
if (hadEntries) {
134+
logger.info(MessageFormat.format("{0}:{1} commit cache cleared", repositoryName, branch));
135+
}
127136
}
128137

129138
/**
@@ -156,49 +165,55 @@ public List<RepositoryCommit> getCommits(String repositoryName, Repository repos
156165
if (cacheDays > 0 && (sinceDate.getTime() >= cacheCutoffDate.getTime())) {
157166
// request fits within the cache window
158167
String repoKey = repositoryName.toLowerCase();
159-
if (!cache.containsKey(repoKey)) {
160-
cache.put(repoKey, new ObjectCache<List<RepositoryCommit>>());
161-
}
162-
163-
ObjectCache<List<RepositoryCommit>> repoCache = cache.get(repoKey);
164168
String branchKey = branch.toLowerCase();
165169

166170
RevCommit tip = JGitUtils.getCommit(repository, branch);
167171
Date tipDate = JGitUtils.getCommitDate(tip);
168172

169-
List<RepositoryCommit> commits;
170-
if (!repoCache.hasCurrent(branchKey, tipDate)) {
171-
commits = repoCache.getObject(branchKey);
172-
if (ArrayUtils.isEmpty(commits)) {
173-
// we don't have any cached commits for this branch, reload
174-
commits = get(repositoryName, repository, branch, cacheCutoffDate);
175-
repoCache.updateObject(branchKey, tipDate, commits);
176-
logger.debug(MessageFormat.format("parsed {0} commits from {1}:{2} since {3,date,yyyy-MM-dd} in {4} msecs",
177-
commits.size(), repositoryName, branch, cacheCutoffDate, TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start)));
178-
} else {
179-
// incrementally update cache since the last cached commit
180-
ObjectId sinceCommit = commits.get(0).getId();
181-
List<RepositoryCommit> incremental = get(repositoryName, repository, branch, sinceCommit);
182-
logger.info(MessageFormat.format("incrementally added {0} commits to cache for {1}:{2} in {3} msecs",
183-
incremental.size(), repositoryName, branch, TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start)));
184-
incremental.addAll(commits);
185-
repoCache.updateObject(branchKey, tipDate, incremental);
186-
commits = incremental;
173+
ObjectCache<List<RepositoryCommit>> repoCache;
174+
synchronized (cache) {
175+
repoCache = cache.get(repoKey);
176+
if (repoCache == null) {
177+
repoCache = new ObjectCache<>();
178+
cache.put(repoKey, repoCache);
187179
}
188-
} else {
189-
// cache is current
190-
commits = repoCache.getObject(branchKey);
191-
// evict older commits outside the cache window
192-
commits = reduce(commits, cacheCutoffDate);
193-
// update cache
194-
repoCache.updateObject(branchKey, tipDate, commits);
195180
}
181+
synchronized (repoCache) {
182+
List<RepositoryCommit> commits;
183+
if (!repoCache.hasCurrent(branchKey, tipDate)) {
184+
commits = repoCache.getObject(branchKey);
185+
if (ArrayUtils.isEmpty(commits)) {
186+
// we don't have any cached commits for this branch, reload
187+
commits = get(repositoryName, repository, branch, cacheCutoffDate);
188+
repoCache.updateObject(branchKey, tipDate, commits);
189+
logger.debug(MessageFormat.format("parsed {0} commits from {1}:{2} since {3,date,yyyy-MM-dd} in {4} msecs",
190+
commits.size(), repositoryName, branch, cacheCutoffDate, TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start)));
191+
} else {
192+
// incrementally update cache since the last cached commit
193+
ObjectId sinceCommit = commits.get(0).getId();
194+
List<RepositoryCommit> incremental = get(repositoryName, repository, branch, sinceCommit);
195+
logger.info(MessageFormat.format("incrementally added {0} commits to cache for {1}:{2} in {3} msecs",
196+
incremental.size(), repositoryName, branch, TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start)));
197+
incremental.addAll(commits);
198+
repoCache.updateObject(branchKey, tipDate, incremental);
199+
commits = incremental;
200+
}
201+
} else {
202+
// cache is current
203+
commits = repoCache.getObject(branchKey);
204+
// evict older commits outside the cache window
205+
commits = reduce(commits, cacheCutoffDate);
206+
// update cache
207+
repoCache.updateObject(branchKey, tipDate, commits);
208+
}
196209

197-
if (sinceDate.equals(cacheCutoffDate)) {
198-
list = commits;
199-
} else {
200-
// reduce the commits to those since the specified date
201-
list = reduce(commits, sinceDate);
210+
if (sinceDate.equals(cacheCutoffDate)) {
211+
// Mustn't hand out the cached list; that's not thread-safe
212+
list = new ArrayList<>(commits);
213+
} else {
214+
// reduce the commits to those since the specified date
215+
list = reduce(commits, sinceDate);
216+
}
202217
}
203218
logger.debug(MessageFormat.format("retrieved {0} commits from cache of {1}:{2} since {3,date,yyyy-MM-dd} in {4} msecs",
204219
list.size(), repositoryName, branch, sinceDate, TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start)));
@@ -222,8 +237,9 @@ public List<RepositoryCommit> getCommits(String repositoryName, Repository repos
222237
*/
223238
protected List<RepositoryCommit> get(String repositoryName, Repository repository, String branch, Date sinceDate) {
224239
Map<ObjectId, List<RefModel>> allRefs = JGitUtils.getAllRefs(repository, false);
225-
List<RepositoryCommit> commits = new ArrayList<RepositoryCommit>();
226-
for (RevCommit commit : JGitUtils.getRevLog(repository, branch, sinceDate)) {
240+
List<RevCommit> revLog = JGitUtils.getRevLog(repository, branch, sinceDate);
241+
List<RepositoryCommit> commits = new ArrayList<RepositoryCommit>(revLog.size());
242+
for (RevCommit commit : revLog) {
227243
RepositoryCommit commitModel = new RepositoryCommit(repositoryName, branch, commit);
228244
List<RefModel> commitRefs = allRefs.get(commitModel.getId());
229245
commitModel.setRefs(commitRefs);
@@ -243,8 +259,9 @@ protected List<RepositoryCommit> get(String repositoryName, Repository repositor
243259
*/
244260
protected List<RepositoryCommit> get(String repositoryName, Repository repository, String branch, ObjectId sinceCommit) {
245261
Map<ObjectId, List<RefModel>> allRefs = JGitUtils.getAllRefs(repository, false);
246-
List<RepositoryCommit> commits = new ArrayList<RepositoryCommit>();
247-
for (RevCommit commit : JGitUtils.getRevLog(repository, sinceCommit.getName(), branch)) {
262+
List<RevCommit> revLog = JGitUtils.getRevLog(repository, sinceCommit.getName(), branch);
263+
List<RepositoryCommit> commits = new ArrayList<RepositoryCommit>(revLog.size());
264+
for (RevCommit commit : revLog) {
248265
RepositoryCommit commitModel = new RepositoryCommit(repositoryName, branch, commit);
249266
List<RefModel> commitRefs = allRefs.get(commitModel.getId());
250267
commitModel.setRefs(commitRefs);
@@ -261,7 +278,7 @@ protected List<RepositoryCommit> get(String repositoryName, Repository repositor
261278
* @return a list of commits
262279
*/
263280
protected List<RepositoryCommit> reduce(List<RepositoryCommit> commits, Date sinceDate) {
264-
List<RepositoryCommit> filtered = new ArrayList<RepositoryCommit>();
281+
List<RepositoryCommit> filtered = new ArrayList<RepositoryCommit>(commits.size());
265282
for (RepositoryCommit commit : commits) {
266283
if (commit.getCommitDate().compareTo(sinceDate) >= 0) {
267284
filtered.add(commit);

0 commit comments

Comments
 (0)