diff options
4 files changed, 312 insertions, 22 deletions
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileSnapshotTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileSnapshotTest.java new file mode 100644 index 0000000000..0e25c494e5 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackFileSnapshotTest.java @@ -0,0 +1,143 @@ +/* + * Copyright (C) 2019, Matthias Sohn <matthias.sohn@sap.com> + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.eclipse.jgit.internal.storage.file; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.io.Writer; +import java.nio.file.Files; +import java.nio.file.StandardOpenOption; +import java.text.ParseException; +import java.util.Collection; +import java.util.Random; +import java.util.zip.Deflater; + +import org.eclipse.jgit.api.GarbageCollectCommand; +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.junit.RepositoryTestCase; +import org.eclipse.jgit.lib.ConfigConstants; +import org.eclipse.jgit.storage.file.FileBasedConfig; +import org.eclipse.jgit.storage.pack.PackConfig; +import org.junit.Test; + +public class PackFileSnapshotTest extends RepositoryTestCase { + + @Test + public void testSamePackDifferentCompressionDetectChecksumChanged() + throws Exception { + Git git = Git.wrap(db); + File f = writeTrashFile("file", "foobar "); + for (int i = 0; i < 10; i++) { + appendRandomLine(f); + git.add().addFilepattern("file").call(); + git.commit().setMessage("message" + i).call(); + } + + FileBasedConfig c = db.getConfig(); + c.setInt(ConfigConstants.CONFIG_GC_SECTION, null, + ConfigConstants.CONFIG_KEY_AUTOPACKLIMIT, 1); + c.save(); + Collection<PackFile> packs = gc(Deflater.NO_COMPRESSION); + assertEquals("expected 1 packfile after gc", 1, packs.size()); + PackFile p1 = packs.iterator().next(); + PackFileSnapshot snapshot = p1.getFileSnapshot(); + + packs = gc(Deflater.BEST_COMPRESSION); + assertEquals("expected 1 packfile after gc", 1, packs.size()); + PackFile p2 = packs.iterator().next(); + File pf = p2.getPackFile(); + + // changing compression level with aggressive gc may change size, + // fileKey (on *nix) and checksum. Hence FileSnapshot.isModified can + // return true already based on size or fileKey. + // So the only thing we can test here is that we ensure that checksum + // also changed when we read it here in this test + assertTrue("expected snapshot to detect modified pack", + snapshot.isModified(pf)); + assertTrue("expected checksum changed", snapshot.isChecksumChanged(pf)); + } + + private void appendRandomLine(File f) throws IOException { + try (Writer w = Files.newBufferedWriter(f.toPath(), + StandardOpenOption.APPEND)) { + w.append(randomLine(20)); + } + } + + private String randomLine(int len) { + final int a = 97; // 'a' + int z = 122; // 'z' + Random random = new Random(); + StringBuilder buffer = new StringBuilder(len); + for (int i = 0; i < len; i++) { + int rnd = a + (int) (random.nextFloat() * (z - a + 1)); + buffer.append((char) rnd); + } + buffer.append('\n'); + return buffer.toString(); + } + + private Collection<PackFile> gc(int compressionLevel) + throws IOException, ParseException { + GC gc = new GC(db); + PackConfig pc = new PackConfig(db.getConfig()); + pc.setCompressionLevel(compressionLevel); + + pc.setSinglePack(true); + + // --aggressive + pc.setDeltaSearchWindowSize( + GarbageCollectCommand.DEFAULT_GC_AGGRESSIVE_WINDOW); + pc.setMaxDeltaDepth(GarbageCollectCommand.DEFAULT_GC_AGGRESSIVE_DEPTH); + pc.setReuseObjects(false); + + gc.setPackConfig(pc); + gc.setExpireAgeMillis(0); + gc.setPackExpireAgeMillis(0); + return gc.gc(); + } + +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java index b6837d2861..1de3135001 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java @@ -119,24 +119,7 @@ public class FileSnapshot { * @return the snapshot. */ public static FileSnapshot save(File path) { - long read = System.currentTimeMillis(); - long modified; - long size; - Object fileKey = null; - Duration fsTimerResolution = FS - .getFsTimerResolution(path.toPath().getParent()); - try { - BasicFileAttributes fileAttributes = FS.DETECTED.fileAttributes(path); - modified = fileAttributes.lastModifiedTime().toMillis(); - size = fileAttributes.size(); - fileKey = getFileKey(fileAttributes); - } catch (IOException e) { - modified = path.lastModified(); - size = path.length(); - fileKey = MISSING_FILEKEY; - } - return new FileSnapshot(read, modified, size, fsTimerResolution, - fileKey); + return new FileSnapshot(path); } private static Object getFileKey(BasicFileAttributes fileAttributes) { @@ -186,7 +169,34 @@ public class FileSnapshot { * Object that uniquely identifies the given file, or {@code * null} if a file key is not available */ - private Object fileKey; + private final Object fileKey; + + /** + * Record a snapshot for a specific file path. + * <p> + * This method should be invoked before the file is accessed. + * + * @param path + * the path to later remember. The path's current status + * information is saved. + */ + protected FileSnapshot(File path) { + this.lastRead = System.currentTimeMillis(); + this.fsTimestampResolution = FS + .getFsTimerResolution(path.toPath().getParent()); + BasicFileAttributes fileAttributes = null; + try { + fileAttributes = FS.DETECTED.fileAttributes(path); + } catch (IOException e) { + this.lastModified = path.lastModified(); + this.size = path.length(); + this.fileKey = MISSING_FILEKEY; + return; + } + this.lastModified = fileAttributes.lastModifiedTime().toMillis(); + this.size = fileAttributes.size(); + this.fileKey = getFileKey(fileAttributes); + } private boolean sizeChanged; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java index d834b1a729..7313305c9e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java @@ -131,7 +131,7 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> { int packLastModified; - private FileSnapshot fileSnapshot; + private PackFileSnapshot fileSnapshot; private volatile boolean invalid; @@ -168,7 +168,7 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> { */ public PackFile(File packFile, int extensions) { this.packFile = packFile; - this.fileSnapshot = FileSnapshot.save(packFile); + this.fileSnapshot = PackFileSnapshot.save(packFile); this.packLastModified = (int) (fileSnapshot.lastModified() >> 10); this.extensions = extensions; @@ -193,6 +193,8 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> { if (packChecksum == null) { packChecksum = idx.packChecksum; + fileSnapshot.setChecksum( + ObjectId.fromRaw(packChecksum)); } else if (!Arrays.equals(packChecksum, idx.packChecksum)) { throw new PackMismatchException(MessageFormat @@ -371,10 +373,14 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> { * * @return the packfile @{@link FileSnapshot} that the object is loaded from. */ - FileSnapshot getFileSnapshot() { + PackFileSnapshot getFileSnapshot() { return fileSnapshot; } + AnyObjectId getPackChecksum() { + return ObjectId.fromRaw(packChecksum); + } + private final byte[] decompress(final long position, final int sz, final WindowCursor curs) throws IOException, DataFormatException { byte[] dstbuf; @@ -1209,4 +1215,12 @@ public class PackFile implements Iterable<PackIndex.MutableEntry> { private boolean hasExt(PackExt ext) { return (extensions & ext.getBit()) != 0; } + + @SuppressWarnings("nls") + @Override + public String toString() { + return "PackFile [packFileName=" + packFile.getName() + ", length=" + + packFile.length() + ", packChecksum=" + + ObjectId.fromRaw(packChecksum).name() + "]"; + } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFileSnapshot.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFileSnapshot.java new file mode 100644 index 0000000000..19ec3af493 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFileSnapshot.java @@ -0,0 +1,123 @@ +/* + * Copyright (C) 2019, Matthias Sohn <matthias.sohn@sap.com> + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.eclipse.jgit.internal.storage.file; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; + +import org.eclipse.jgit.lib.AnyObjectId; +import org.eclipse.jgit.lib.ObjectId; + +class PackFileSnapshot extends FileSnapshot { + + private static final ObjectId MISSING_CHECKSUM = ObjectId.zeroId(); + + /** + * Record a snapshot for a specific packfile path. + * <p> + * This method should be invoked before the packfile is accessed. + * + * @param path + * the path to later remember. The path's current status + * information is saved. + * @return the snapshot. + */ + public static PackFileSnapshot save(File path) { + return new PackFileSnapshot(path); + } + + private AnyObjectId checksum = MISSING_CHECKSUM; + + private boolean wasChecksumChanged; + + + PackFileSnapshot(File packFile) { + super(packFile); + } + + void setChecksum(AnyObjectId checksum) { + this.checksum = checksum; + } + + /** {@inheritDoc} */ + @Override + public boolean isModified(File packFile) { + if (!super.isModified(packFile)) { + return false; + } + if (wasSizeChanged() || wasFileKeyChanged() + || !wasLastModifiedRacilyClean()) { + return true; + } + return isChecksumChanged(packFile); + } + + boolean isChecksumChanged(File packFile) { + return wasChecksumChanged = checksum != MISSING_CHECKSUM + && !checksum.equals(readChecksum(packFile)); + } + + private AnyObjectId readChecksum(File packFile) { + try (RandomAccessFile fd = new RandomAccessFile(packFile, "r")) { //$NON-NLS-1$ + fd.seek(fd.length() - 20); + final byte[] buf = new byte[20]; + fd.readFully(buf, 0, 20); + return ObjectId.fromRaw(buf); + } catch (IOException e) { + return MISSING_CHECKSUM; + } + } + + boolean wasChecksumChanged() { + return wasChecksumChanged; + } + + @SuppressWarnings("nls") + @Override + public String toString() { + return "PackFileSnapshot [checksum=" + checksum + ", " + + super.toString() + "]"; + } + +} |