aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRoberto Tyley2012-12-25 04:10:51 (EST)
committerGerrit Code Review @ Eclipse.org2013-01-13 18:59:59 (EST)
commit5dcc8693d7d1ab81e3d6e6583a3918a3b6003bad (patch)
tree3ead984ff904d1c0d6a9c26a51b9c4b19f2edfd4
parent912ef3da19c7bc1975805e1e3e9746baf479c2be (diff)
downloadjgit-5dcc8693d7d1ab81e3d6e6583a3918a3b6003bad.zip
jgit-5dcc8693d7d1ab81e3d6e6583a3918a3b6003bad.tar.gz
jgit-5dcc8693d7d1ab81e3d6e6583a3918a3b6003bad.tar.bz2
Fix concurrent creation of fan-out object directoriesrefs/changes/72/9372/2
If multiple threads attempted to insert loose objects into the same new fan-out directory, the creation of that directory was subject to a race condition that could lead to an unnecessary IOException being thrown - because an inserter could not 'create' a directory that had just been generated by a different thread. All we require is that the directory does indeed *exist*, so not being able to _create_ it is not actually a fatal problem. Setting 'skipExisting' to 'true' on the call to mkdir() fixes the issue. I found this issue as a real world occurrence while working on The BFG Repo Cleaner (https://github.com/rtyley/bfg-repo-cleaner), a tool which concurrently performs a lot of object creation. In order to demonstrate the problem here I've added a small test case which reliably reproduces the issue on the few different hardware systems I've tried. The error thrown when the race-condition arises is this: java.io.IOException: Creating directory /home/roberto/repo.git/objects/e6 failed at org.eclipse.jgit.util.FileUtils.mkdir(FileUtils.java:182) at org.eclipse.jgit.storage.file.ObjectDirectory.insertUnpackedObject(ObjectDirectory.java:590) at org.eclipse.jgit.storage.file.ObjectDirectoryInserter.insertOneObject(ObjectDirectoryInserter.java:113) at org.eclipse.jgit.storage.file.ObjectDirectoryInserter.insert(ObjectDirectoryInserter.java:91) at org.eclipse.jgit.lib.ObjectInserter.insert(ObjectInserter.java:329) Change-Id: I88eac49bc600c56ba9ad290e6133d8a7113125ab
-rw-r--r--org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/ObjectDirectoryTest.java77
-rw-r--r--org.eclipse.jgit/src/org/eclipse/jgit/storage/file/ObjectDirectory.java2
2 files changed, 78 insertions, 1 deletions
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/ObjectDirectoryTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/ObjectDirectoryTest.java
new file mode 100644
index 0000000..52c9ec7
--- /dev/null
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/storage/file/ObjectDirectoryTest.java
@@ -0,0 +1,77 @@
+/*
+ * Copyright (C) 2012, Roberto Tyley <roberto.tyley@gmail.com>
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials provided
+ * with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ * names of its contributors may be used to endorse or promote
+ * products derived from this software without specific prior
+ * written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.eclipse.jgit.storage.file;
+
+import org.eclipse.jgit.lib.Constants;
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.RepositoryTestCase;
+import org.junit.Test;
+
+import java.util.*;
+import java.util.concurrent.*;
+
+public class ObjectDirectoryTest extends RepositoryTestCase {
+
+ @Test
+ public void testConcurrentInsertionOfBlobsToTheSameNewFanOutDirectory()
+ throws Exception {
+ ExecutorService e = Executors.newCachedThreadPool();
+ for (int i=0; i < 100; ++i) {
+ ObjectDirectory db = createBareRepository().getObjectDatabase();
+ for (Future f : e.invokeAll(blobInsertersForTheSameFanOutDir(db))) {
+ f.get();
+ }
+ }
+ }
+
+ private Collection<Callable<ObjectId>> blobInsertersForTheSameFanOutDir(
+ final ObjectDirectory db) {
+ Callable<ObjectId> callable = new Callable<ObjectId>() {
+ public ObjectId call() throws Exception {
+ return db.newInserter().insert(Constants.OBJ_BLOB, new byte[0]);
+ }
+ };
+ return Collections.nCopies(4, callable);
+ }
+
+}
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/ObjectDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/ObjectDirectory.java
index 4d196fb..22ef45d 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/ObjectDirectory.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/ObjectDirectory.java
@@ -577,7 +577,7 @@ public class ObjectDirectory extends FileObjectDatabase {
// directories are always lazily created. Note that we
// try the rename first as the directory likely does exist.
//
- FileUtils.mkdir(dst.getParentFile());
+ FileUtils.mkdir(dst.getParentFile(), true);
if (tmp.renameTo(dst)) {
dst.setReadOnly();
unpackedObjectCache.add(id);