Extend FileSnapshot for packfiles to also use checksum to detect changes

If the attributes of FileSnapshot don't detect modification of a
packfile read the packfile's checksum and compare it against the
checksum cached in the loaded packfile.

Since reading the checksum needs less IO than reloading the complete
packfile this may help to reduce the overhead to detect modficiation
when a gc completes while ObjectDirectory scans for packfiles in another
thread.

Bug: 546891
Change-Id: I9811b497eb11b8a85ae689081dc5d949ca8c4be5
Signed-off-by: Matthias Sohn <matthias.sohn@sap.com>

4 files changed, 312 insertions, 22 deletions

diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileSnapshotTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileSnapshotTest.java
new file mode 100644
index 0000000000..0e25c494e5
--- /dev/null
+++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileSnapshotTest.java
@@ -0,0 +1,143 @@
+/*
+ * Copyright (C) 2019, Matthias Sohn <matthias.sohn@sap.com>
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.eclipse.jgit.internal.storage.file;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.Writer;
+import java.nio.file.Files;
+import java.nio.file.StandardOpenOption;
+import java.text.ParseException;
+import java.util.Collection;
+import java.util.Random;
+import java.util.zip.Deflater;
+
+import org.eclipse.jgit.api.GarbageCollectCommand;
+import org.eclipse.jgit.api.Git;
+import org.eclipse.jgit.junit.RepositoryTestCase;
+import org.eclipse.jgit.lib.ConfigConstants;
+import org.eclipse.jgit.storage.file.FileBasedConfig;
+import org.eclipse.jgit.storage.pack.PackConfig;
+import org.junit.Test;
+
+public class PackFileSnapshotTest extends RepositoryTestCase {
+
+	@Test
+	public void testSamePackDifferentCompressionDetectChecksumChanged()
+			throws Exception {
+		Git git = Git.wrap(db);
+		File f = writeTrashFile("file", "foobar ");
+		for (int i = 0; i < 10; i++) {
+			appendRandomLine(f);
+			git.add().addFilepattern("file").call();
+			git.commit().setMessage("message" + i).call();
+		}
+
+		FileBasedConfig c = db.getConfig();
+		c.setInt(ConfigConstants.CONFIG_GC_SECTION, null,
+				ConfigConstants.CONFIG_KEY_AUTOPACKLIMIT, 1);
+		c.save();
+		Collection<PackFile> packs = gc(Deflater.NO_COMPRESSION);
+		assertEquals("expected 1 packfile after gc", 1, packs.size());
+		PackFile p1 = packs.iterator().next();
+		PackFileSnapshot snapshot = p1.getFileSnapshot();
+
+		packs = gc(Deflater.BEST_COMPRESSION);
+		assertEquals("expected 1 packfile after gc", 1, packs.size());
+		PackFile p2 = packs.iterator().next();
+		File pf = p2.getPackFile();
+
+		// changing compression level with aggressive gc may change size,
+		// fileKey (on *nix) and checksum. Hence FileSnapshot.isModified can
+		// return true already based on size or fileKey.
+		// So the only thing we can test here is that we ensure that checksum
+		// also changed when we read it here in this test
+		assertTrue("expected snapshot to detect modified pack",
+				snapshot.isModified(pf));
+		assertTrue("expected checksum changed", snapshot.isChecksumChanged(pf));
+	}
+
+	private void appendRandomLine(File f) throws IOException {
+		try (Writer w = Files.newBufferedWriter(f.toPath(),
+				StandardOpenOption.APPEND)) {
+			w.append(randomLine(20));
+		}
+	}
+
+	private String randomLine(int len) {
+		final int a = 97; // 'a'
+		int z = 122; // 'z'
+		Random random = new Random();
+		StringBuilder buffer = new StringBuilder(len);
+		for (int i = 0; i < len; i++) {
+			int rnd = a + (int) (random.nextFloat() * (z - a + 1));
+			buffer.append((char) rnd);
+		}
+		buffer.append('\n');
+		return buffer.toString();
+	}
+
+	private Collection<PackFile> gc(int compressionLevel)
+			throws IOException, ParseException {
+		GC gc = new GC(db);
+		PackConfig pc = new PackConfig(db.getConfig());
+		pc.setCompressionLevel(compressionLevel);
+
+		pc.setSinglePack(true);
+
+		// --aggressive
+		pc.setDeltaSearchWindowSize(
+				GarbageCollectCommand.DEFAULT_GC_AGGRESSIVE_WINDOW);
+		pc.setMaxDeltaDepth(GarbageCollectCommand.DEFAULT_GC_AGGRESSIVE_DEPTH);
+		pc.setReuseObjects(false);
+
+		gc.setPackConfig(pc);
+		gc.setExpireAgeMillis(0);
+		gc.setPackExpireAgeMillis(0);
+		return gc.gc();
+	}
+
+}
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java
index b6837d2861..1de3135001 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java
@@ -119,24 +119,7 @@ public class FileSnapshot {
 	 * @return the snapshot.
 	 */
 	public static FileSnapshot save(File path) {
-		long read = System.currentTimeMillis();
-		long modified;
-		long size;
-		Object fileKey = null;
-		Duration fsTimerResolution = FS
-				.getFsTimerResolution(path.toPath().getParent());
-		try {
-			BasicFileAttributes fileAttributes = FS.DETECTED.fileAttributes(path);
-			modified = fileAttributes.lastModifiedTime().toMillis();
-			size = fileAttributes.size();
-			fileKey = getFileKey(fileAttributes);
-		} catch (IOException e) {
-			modified = path.lastModified();
-			size = path.length();
-			fileKey = MISSING_FILEKEY;
-		}
-		return new FileSnapshot(read, modified, size, fsTimerResolution,
-				fileKey);
+		return new FileSnapshot(path);
 	}
 
 	private static Object getFileKey(BasicFileAttributes fileAttributes) {
@@ -186,7 +169,34 @@ public class FileSnapshot {
 	 * Object that uniquely identifies the given file, or {@code
 	 * null} if a file key is not available
 	 */
-	private Object fileKey;
+	private final Object fileKey;
+
+	/**
+	 * Record a snapshot for a specific file path.
+	 * <p>
+	 * This method should be invoked before the file is accessed.
+	 *
+	 * @param path
+	 *            the path to later remember. The path's current status
+	 *            information is saved.
+	 */
+	protected FileSnapshot(File path) {
+		this.lastRead = System.currentTimeMillis();
+		this.fsTimestampResolution = FS
+				.getFsTimerResolution(path.toPath().getParent());
+		BasicFileAttributes fileAttributes = null;
+		try {
+			fileAttributes = FS.DETECTED.fileAttributes(path);
+		} catch (IOException e) {
+			this.lastModified = path.lastModified();
+			this.size = path.length();
+			this.fileKey = MISSING_FILEKEY;
+			return;
+		}
+		this.lastModified = fileAttributes.lastModifiedTime().toMillis();
+		this.size = fileAttributes.size();
+		this.fileKey = getFileKey(fileAttributes);
+	}
 
 	private boolean sizeChanged;
 
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java
index d834b1a729..7313305c9e 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java
@@ -131,7 +131,7 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> {
 
 	int packLastModified;
 
-	private FileSnapshot fileSnapshot;
+	private PackFileSnapshot fileSnapshot;
 
 	private volatile boolean invalid;
 
@@ -168,7 +168,7 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> {
 	 */
 	public PackFile(File packFile, int extensions) {
 		this.packFile = packFile;
-		this.fileSnapshot = FileSnapshot.save(packFile);
+		this.fileSnapshot = PackFileSnapshot.save(packFile);
 		this.packLastModified = (int) (fileSnapshot.lastModified() >> 10);
 		this.extensions = extensions;
 
@@ -193,6 +193,8 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> {
 
 						if (packChecksum == null) {
 							packChecksum = idx.packChecksum;
+							fileSnapshot.setChecksum(
+									ObjectId.fromRaw(packChecksum));
 						} else if (!Arrays.equals(packChecksum,
 								idx.packChecksum)) {
 							throw new PackMismatchException(MessageFormat
@@ -371,10 +373,14 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> {
 	 *
 	 * @return the packfile @{@link FileSnapshot} that the object is loaded from.
 	 */
-	FileSnapshot getFileSnapshot() {
+	PackFileSnapshot getFileSnapshot() {
 		return fileSnapshot;
 	}
 
+	AnyObjectId getPackChecksum() {
+		return ObjectId.fromRaw(packChecksum);
+	}
+
 	private final byte[] decompress(final long position, final int sz,
 			final WindowCursor curs) throws IOException, DataFormatException {
 		byte[] dstbuf;
@@ -1209,4 +1215,12 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> {
 	private boolean hasExt(PackExt ext) {
 		return (extensions & ext.getBit()) != 0;
 	}
+
+	@SuppressWarnings("nls")
+	@Override
+	public String toString() {
+		return "PackFile [packFileName=" + packFile.getName() + ", length="
+				+ packFile.length() + ", packChecksum="
+				+ ObjectId.fromRaw(packChecksum).name() + "]";
+	}
 }
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFileSnapshot.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFileSnapshot.java
new file mode 100644
index 0000000000..19ec3af493
--- /dev/null
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFileSnapshot.java
@@ -0,0 +1,123 @@
+/*
+ * Copyright (C) 2019, Matthias Sohn <matthias.sohn@sap.com>
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.eclipse.jgit.internal.storage.file;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+
+import org.eclipse.jgit.lib.AnyObjectId;
+import org.eclipse.jgit.lib.ObjectId;
+
+class PackFileSnapshot extends FileSnapshot {
+
+	private static final ObjectId MISSING_CHECKSUM = ObjectId.zeroId();
+
+	/**
+	 * Record a snapshot for a specific packfile path.
+	 * <p>
+	 * This method should be invoked before the packfile is accessed.
+	 *
+	 * @param path
+	 *            the path to later remember. The path's current status
+	 *            information is saved.
+	 * @return the snapshot.
+	 */
+	public static PackFileSnapshot save(File path) {
+		return new PackFileSnapshot(path);
+	}
+
+	private AnyObjectId checksum = MISSING_CHECKSUM;
+
+	private boolean wasChecksumChanged;
+
+
+	PackFileSnapshot(File packFile) {
+		super(packFile);
+	}
+
+	void setChecksum(AnyObjectId checksum) {
+		this.checksum = checksum;
+	}
+
+	/** {@inheritDoc} */
+	@Override
+	public boolean isModified(File packFile) {
+		if (!super.isModified(packFile)) {
+			return false;
+		}
+		if (wasSizeChanged() || wasFileKeyChanged()
+				|| !wasLastModifiedRacilyClean()) {
+			return true;
+		}
+		return isChecksumChanged(packFile);
+	}
+
+	boolean isChecksumChanged(File packFile) {
+		return wasChecksumChanged = checksum != MISSING_CHECKSUM
+				&& !checksum.equals(readChecksum(packFile));
+	}
+
+	private AnyObjectId readChecksum(File packFile) {
+		try (RandomAccessFile fd = new RandomAccessFile(packFile, "r")) { //$NON-NLS-1$
+			fd.seek(fd.length() - 20);
+			final byte[] buf = new byte[20];
+			fd.readFully(buf, 0, 20);
+			return ObjectId.fromRaw(buf);
+		} catch (IOException e) {
+			return MISSING_CHECKSUM;
+		}
+	}
+
+	boolean wasChecksumChanged() {
+		return wasChecksumChanged;
+	}
+
+	@SuppressWarnings("nls")
+	@Override
+	public String toString() {
+		return "PackFileSnapshot [checksum=" + checksum + ", "
+				+ super.toString() + "]";
+	}
+
+}