Merge branch 'stable-5.9' into stable-5.10

* stable-5.9:
  LockFile: create OutputStream only when needed
  Remove ReftableNumbersNotIncreasingException
  Fix stamping to produce stable file timestamps

Change-Id: I056382d1d93f3e0a95838bdd1f0be89711c8a722

6 files changed, 249 insertions, 73 deletions

diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/LockFileTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/LockFileTest.java
index 0f93749d9b..509935dfb9 100644
--- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/LockFileTest.java
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/LockFileTest.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2012, GitHub Inc. and others
+ * Copyright (C) 2012, 2021 GitHub Inc. and others
  *
  * This program and the accompanying materials are made available under the
  * terms of the Eclipse Distribution License v. 1.0 which is available at
@@ -9,10 +9,16 @@
  */
 package org.eclipse.jgit.internal.storage.file;
 
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThrows;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
+import java.io.File;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+
 import org.eclipse.jgit.api.Git;
 import org.eclipse.jgit.api.errors.JGitInternalException;
 import org.eclipse.jgit.errors.LockFailedException;
@@ -49,4 +55,149 @@ public class LockFileTest extends RepositoryTestCase {
 			}
 		}
 	}
+
+	@Test
+	public void testLockTwice() throws Exception {
+		File f = writeTrashFile("somefile", "content");
+		LockFile lock = new LockFile(f);
+		assertTrue(lock.lock());
+		lock.write("other".getBytes(StandardCharsets.US_ASCII));
+		lock.commit();
+		assertFalse(lock.isLocked());
+		checkFile(f, "other");
+		assertTrue(lock.lock());
+		assertTrue(lock.isLocked());
+		try (OutputStream out = lock.getOutputStream()) {
+			out.write("second".getBytes(StandardCharsets.US_ASCII));
+		}
+		lock.commit();
+		assertFalse(lock.isLocked());
+		checkFile(f, "second");
+	}
+
+	@Test
+	public void testLockTwiceUnlock() throws Exception {
+		File f = writeTrashFile("somefile", "content");
+		LockFile lock = new LockFile(f);
+		assertTrue(lock.lock());
+		assertTrue(lock.isLocked());
+		lock.write("other".getBytes(StandardCharsets.US_ASCII));
+		lock.unlock();
+		assertFalse(lock.isLocked());
+		checkFile(f, "content");
+		assertTrue(lock.lock());
+		assertTrue(lock.isLocked());
+		try (OutputStream out = lock.getOutputStream()) {
+			out.write("second".getBytes(StandardCharsets.US_ASCII));
+		}
+		lock.commit();
+		assertFalse(lock.isLocked());
+		checkFile(f, "second");
+	}
+
+	@Test
+	public void testLockWriteTwiceThrows1() throws Exception {
+		File f = writeTrashFile("somefile", "content");
+		LockFile lock = new LockFile(f);
+		assertTrue(lock.lock());
+		assertTrue(lock.isLocked());
+		lock.write("other".getBytes(StandardCharsets.US_ASCII));
+		assertThrows(Exception.class,
+				() -> lock.write("second".getBytes(StandardCharsets.US_ASCII)));
+		lock.unlock();
+	}
+
+	@Test
+	public void testLockWriteTwiceThrows2() throws Exception {
+		File f = writeTrashFile("somefile", "content");
+		LockFile lock = new LockFile(f);
+		assertTrue(lock.lock());
+		assertTrue(lock.isLocked());
+		try (OutputStream out = lock.getOutputStream()) {
+			out.write("other".getBytes(StandardCharsets.US_ASCII));
+		}
+		assertThrows(Exception.class,
+				() -> lock.write("second".getBytes(StandardCharsets.US_ASCII)));
+		lock.unlock();
+	}
+
+	@Test
+	public void testLockWriteTwiceThrows3() throws Exception {
+		File f = writeTrashFile("somefile", "content");
+		LockFile lock = new LockFile(f);
+		assertTrue(lock.lock());
+		assertTrue(lock.isLocked());
+		lock.write("other".getBytes(StandardCharsets.US_ASCII));
+		assertThrows(Exception.class, () -> {
+			try (OutputStream out = lock.getOutputStream()) {
+				out.write("second".getBytes(StandardCharsets.US_ASCII));
+			}
+		});
+		lock.unlock();
+	}
+
+	@Test
+	public void testLockWriteTwiceThrows4() throws Exception {
+		File f = writeTrashFile("somefile", "content");
+		LockFile lock = new LockFile(f);
+		assertTrue(lock.lock());
+		assertTrue(lock.isLocked());
+		try (OutputStream out = lock.getOutputStream()) {
+			out.write("other".getBytes(StandardCharsets.US_ASCII));
+		}
+		assertThrows(Exception.class, () -> {
+			try (OutputStream out = lock.getOutputStream()) {
+				out.write("second".getBytes(StandardCharsets.US_ASCII));
+			}
+		});
+		lock.unlock();
+	}
+
+	@Test
+	public void testLockUnclosedCommitThrows() throws Exception {
+		File f = writeTrashFile("somefile", "content");
+		LockFile lock = new LockFile(f);
+		assertTrue(lock.lock());
+		assertTrue(lock.isLocked());
+		try (OutputStream out = lock.getOutputStream()) {
+			out.write("other".getBytes(StandardCharsets.US_ASCII));
+			assertThrows(Exception.class, () -> lock.commit());
+		}
+	}
+
+	@Test
+	public void testLockNested() throws Exception {
+		File f = writeTrashFile("somefile", "content");
+		LockFile lock = new LockFile(f);
+		assertTrue(lock.lock());
+		assertTrue(lock.isLocked());
+		assertThrows(IllegalStateException.class, () -> lock.lock());
+		assertTrue(lock.isLocked());
+		lock.unlock();
+	}
+
+	@Test
+	public void testLockHeld() throws Exception {
+		File f = writeTrashFile("somefile", "content");
+		LockFile lock = new LockFile(f);
+		assertTrue(lock.lock());
+		assertTrue(lock.isLocked());
+		LockFile lock2 = new LockFile(f);
+		assertFalse(lock2.lock());
+		assertFalse(lock2.isLocked());
+		assertTrue(lock.isLocked());
+		lock.unlock();
+	}
+
+	@Test
+	public void testLockForAppend() throws Exception {
+		File f = writeTrashFile("somefile", "content");
+		LockFile lock = new LockFile(f);
+		assertTrue(lock.lockForAppend());
+		assertTrue(lock.isLocked());
+		lock.write("other".getBytes(StandardCharsets.US_ASCII));
+		lock.commit();
+		assertFalse(lock.isLocked());
+		checkFile(f, "contentother");
+	}
 }
diff --git a/org.eclipse.jgit/BUILD b/org.eclipse.jgit/BUILD
index 2083372248..04873b0c72 100644
--- a/org.eclipse.jgit/BUILD
+++ b/org.eclipse.jgit/BUILD
@@ -38,7 +38,7 @@ genrule(
         "cd $$TMP",
         "unzip -q $$ROOT/$<",
         "echo \"Implementation-Version: $$GEN_VERSION\n$$(cat META-INF/MANIFEST.MF)\" > META-INF/MANIFEST.MF",
-        "find . -exec touch '{}' ';'",
+        "find . -exec touch -t 198001010000 '{}' ';'",
         "zip -Xqr $$ROOT/$@ .",
         "rm -rf $$TMP",
     ]),
diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties
index 12902b9004..c52e307814 100644
--- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties
+++ b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties
@@ -407,11 +407,14 @@ listingPacks=Listing packs
 localObjectsIncomplete=Local objects incomplete.
 localRefIsMissingObjects=Local ref {0} is missing object(s).
 localRepository=local repository
+lockAlreadyHeld=Lock on {0} already held
 lockCountMustBeGreaterOrEqual1=lockCount must be >= 1
 lockError=lock error: {0}
 lockFailedRetry=locking {0} failed after {1} retries
 lockOnNotClosed=Lock on {0} not closed.
 lockOnNotHeld=Lock on {0} not held.
+lockStreamClosed=Output to lock on {0} already closed
+lockStreamMultiple=Output to lock on {0} already opened
 logInconsistentFiletimeDiff={}: inconsistent duration from file timestamps on {}, {}: {} > {}, but diff = {}. Aborting measurement at resolution {}.
 logLargerFiletimeDiff={}: inconsistent duration from file timestamps on {}, {}: diff = {} > {} (last good value). Aborting measurement.
 logSmallerFiletime={}: got smaller file timestamp on {}, {}: {} < {}. Aborting measurement at resolution {}.
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java
index 892657d5d3..d12a243a8e 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java
@@ -1,6 +1,6 @@
 /*
  * Copyright (C) 2010, 2013 Sasa Zivkov <sasa.zivkov@sap.com>
- * Copyright (C) 2012, Research In Motion Limited and others
+ * Copyright (C) 2012, 2021 Research In Motion Limited and others
  *
  * This program and the accompanying materials are made available under the
  * terms of the Eclipse Distribution License v. 1.0 which is available at
@@ -436,10 +436,13 @@ public class JGitText extends TranslationBundle {
 	/***/ public String localRefIsMissingObjects;
 	/***/ public String localRepository;
 	/***/ public String lockCountMustBeGreaterOrEqual1;
+	/***/ public String lockAlreadyHeld;
 	/***/ public String lockError;
 	/***/ public String lockFailedRetry;
 	/***/ public String lockOnNotClosed;
 	/***/ public String lockOnNotHeld;
+	/***/ public String lockStreamClosed;
+	/***/ public String lockStreamMultiple;
 	/***/ public String logInconsistentFiletimeDiff;
 	/***/ public String logLargerFiletimeDiff;
 	/***/ public String logSmallerFiletime;
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableStack.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableStack.java
index bc2039c56b..71130f0f3b 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableStack.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableStack.java
@@ -128,33 +128,6 @@ public class FileReftableStack implements AutoCloseable {
 		return stats;
 	}
 
-	/** Thrown if the update indices in the stack are not monotonic */
-	public static class ReftableNumbersNotIncreasingException
-			extends RuntimeException {
-		private static final long serialVersionUID = 1L;
-
-		String name;
-
-		long lastMax;
-
-		long min;
-
-		ReftableNumbersNotIncreasingException(String name, long lastMax,
-				long min) {
-			this.name = name;
-			this.lastMax = lastMax;
-			this.min = min;
-		}
-
-		@SuppressWarnings({ "nls", "boxing" })
-		@Override
-		public String toString() {
-			return String.format(
-					"ReftableNumbersNotIncreasingException %s: min %d, lastMax %d",
-					name, min, lastMax);
-		}
-	}
-
 	/**
 	 * Reloads the stack, potentially reusing opened reftableReaders.
 	 *
@@ -173,7 +146,6 @@ public class FileReftableStack implements AutoCloseable {
 		List<ReftableReader> newTables = new ArrayList<>();
 		List<StackEntry> newStack = new ArrayList<>(stack.size() + 1);
 		try {
-			ReftableReader last = null;
 			for (String name : names) {
 				StackEntry entry = new StackEntry();
 				entry.name = name;
@@ -191,15 +163,6 @@ public class FileReftableStack implements AutoCloseable {
 					newTables.add(t);
 				}
 
-				if (last != null) {
-					// TODO: move this to MergedReftable
-					if (last.maxUpdateIndex() >= t.minUpdateIndex()) {
-						throw new ReftableNumbersNotIncreasingException(name,
-								last.maxUpdateIndex(), t.minUpdateIndex());
-					}
-				}
-				last = t;
-
 				entry.reftableReader = t;
 				newStack.add(entry);
 			}
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LockFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LockFile.java
index 2e0a6da3a4..f57581a299 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LockFile.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LockFile.java
@@ -1,6 +1,6 @@
 /*
  * Copyright (C) 2007, Robin Rosenberg <robin.rosenberg@dewire.com>
- * Copyright (C) 2006-2008, Shawn O. Pearce <spearce@spearce.org> and others
+ * Copyright (C) 2006-2021, Shawn O. Pearce <spearce@spearce.org> and others
  *
  * This program and the accompanying materials are made available under the
  * terms of the Eclipse Distribution License v. 1.0 which is available at
@@ -96,11 +96,15 @@ public class LockFile {
 
 	private boolean haveLck;
 
-	FileOutputStream os;
+	private FileOutputStream os;
 
 	private boolean needSnapshot;
 
-	boolean fsync;
+	private boolean fsync;
+
+	private boolean isAppend;
+
+	private boolean written;
 
 	private FileSnapshot commitSnapshot;
 
@@ -127,6 +131,10 @@ public class LockFile {
 	 *             does not hold the lock.
 	 */
 	public boolean lock() throws IOException {
+		if (haveLck) {
+			throw new IllegalStateException(
+					MessageFormat.format(JGitText.get().lockAlreadyHeld, ref));
+		}
 		FileUtils.mkdirs(lck.getParentFile(), true);
 		try {
 			token = FS.DETECTED.createNewFileAtomic(lck);
@@ -134,18 +142,15 @@ public class LockFile {
 			LOG.error(JGitText.get().failedCreateLockFile, lck, e);
 			throw e;
 		}
-		if (token.isCreated()) {
+		boolean obtainedLock = token.isCreated();
+		if (obtainedLock) {
 			haveLck = true;
-			try {
-				os = new FileOutputStream(lck);
-			} catch (IOException ioe) {
-				unlock();
-				throw ioe;
-			}
+			isAppend = false;
+			written = false;
 		} else {
 			closeToken();
 		}
-		return haveLck;
+		return obtainedLock;
 	}
 
 	/**
@@ -158,12 +163,24 @@ public class LockFile {
 	 *             does not hold the lock.
 	 */
 	public boolean lockForAppend() throws IOException {
-		if (!lock())
+		if (!lock()) {
 			return false;
+		}
 		copyCurrentContent();
+		isAppend = true;
+		written = false;
 		return true;
 	}
 
+	// For tests only
+	boolean isLocked() {
+		return haveLck;
+	}
+
+	private FileOutputStream getStream() throws IOException {
+		return new FileOutputStream(lck, isAppend);
+	}
+
 	/**
 	 * Copy the current file content into the temporary file.
 	 * <p>
@@ -185,32 +202,31 @@ public class LockFile {
 	 */
 	public void copyCurrentContent() throws IOException {
 		requireLock();
-		try {
+		try (FileOutputStream out = getStream()) {
 			try (FileInputStream fis = new FileInputStream(ref)) {
 				if (fsync) {
 					FileChannel in = fis.getChannel();
 					long pos = 0;
 					long cnt = in.size();
 					while (0 < cnt) {
-						long r = os.getChannel().transferFrom(in, pos, cnt);
+						long r = out.getChannel().transferFrom(in, pos, cnt);
 						pos += r;
 						cnt -= r;
 					}
 				} else {
 					final byte[] buf = new byte[2048];
 					int r;
-					while ((r = fis.read(buf)) >= 0)
-						os.write(buf, 0, r);
+					while ((r = fis.read(buf)) >= 0) {
+						out.write(buf, 0, r);
 				}
 			}
 		} catch (FileNotFoundException fnfe) {
 			if (ref.exists()) {
-				unlock();
 				throw fnfe;
 			}
 			// Don't worry about a file that doesn't exist yet, it
 			// conceptually has no current content to copy.
-			//
+			}
 		} catch (IOException | RuntimeException | Error ioe) {
 			unlock();
 			throw ioe;
@@ -253,18 +269,22 @@ public class LockFile {
 	 */
 	public void write(byte[] content) throws IOException {
 		requireLock();
-		try {
+		try (FileOutputStream out = getStream()) {
+			if (written) {
+				throw new IOException(MessageFormat
+						.format(JGitText.get().lockStreamClosed, ref));
+			}
 			if (fsync) {
-				FileChannel fc = os.getChannel();
+				FileChannel fc = out.getChannel();
 				ByteBuffer buf = ByteBuffer.wrap(content);
-				while (0 < buf.remaining())
+				while (0 < buf.remaining()) {
 					fc.write(buf);
+				}
 				fc.force(true);
 			} else {
-				os.write(content);
+				out.write(content);
 			}
-			os.close();
-			os = null;
+			written = true;
 		} catch (IOException | RuntimeException | Error ioe) {
 			unlock();
 			throw ioe;
@@ -283,36 +303,68 @@ public class LockFile {
 	public OutputStream getOutputStream() {
 		requireLock();
 
-		final OutputStream out;
-		if (fsync)
+		if (written || os != null) {
+			throw new IllegalStateException(MessageFormat
+					.format(JGitText.get().lockStreamMultiple, ref));
+		}
+
+		return new OutputStream() {
+
+			private OutputStream out;
+
+			private boolean closed;
+
+			private OutputStream get() throws IOException {
+				if (written) {
+					throw new IOException(MessageFormat
+							.format(JGitText.get().lockStreamMultiple, ref));
+				}
+				if (out == null) {
+					os = getStream();
+					if (fsync) {
 			out = Channels.newOutputStream(os.getChannel());
-		else
+					} else {
 			out = os;
+					}
+				}
+				return out;
+			}
 
-		return new OutputStream() {
 			@Override
 			public void write(byte[] b, int o, int n)
 					throws IOException {
-				out.write(b, o, n);
+				get().write(b, o, n);
 			}
 
 			@Override
 			public void write(byte[] b) throws IOException {
-				out.write(b);
+				get().write(b);
 			}
 
 			@Override
 			public void write(int b) throws IOException {
-				out.write(b);
+				get().write(b);
 			}
 
 			@Override
 			public void close() throws IOException {
+				if (closed) {
+					return;
+				}
+				closed = true;
 				try {
-					if (fsync)
+					if (written) {
+						throw new IOException(MessageFormat
+								.format(JGitText.get().lockStreamClosed, ref));
+					}
+					if (out != null) {
+						if (fsync) {
 						os.getChannel().force(true);
+						}
 					out.close();
 					os = null;
+					}
+					written = true;
 				} catch (IOException | RuntimeException | Error ioe) {
 					unlock();
 					throw ioe;
@@ -322,7 +374,7 @@ public class LockFile {
 	}
 
 	void requireLock() {
-		if (os == null) {
+		if (!haveLck) {
 			unlock();
 			throw new IllegalStateException(MessageFormat.format(JGitText.get().lockOnNotHeld, ref));
 		}
@@ -411,6 +463,8 @@ public class LockFile {
 		try {
 			FileUtils.rename(lck, ref, StandardCopyOption.ATOMIC_MOVE);
 			haveLck = false;
+			isAppend = false;
+			written = false;
 			closeToken();
 			return true;
 		} catch (IOException e) {
@@ -497,6 +551,8 @@ public class LockFile {
 				closeToken();
 			}
 		}
+		isAppend = false;
+		written = false;
 	}
 
 	/** {@inheritDoc} */