From 13212d7ec32e52d51698b5bf97f34b4ffd2e0439 Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Thu, 28 Mar 2024 14:55:09 -0700 Subject: PackIndex: move checksum to the subclasses PackIndex is almost an interface, and making it so simplifies writing implementations over other storages. Checksum and its getter is the only functionality that is class specific. Make getChecksum abstract and implement it in the subclasses. Change-Id: I3746515d816abab075210505f1bb31b1936962e9 --- .../src/org/eclipse/jgit/internal/storage/file/Pack.java | 8 ++++---- .../src/org/eclipse/jgit/internal/storage/file/PackIndex.java | 7 +------ .../src/org/eclipse/jgit/internal/storage/file/PackIndexV1.java | 8 ++++++++ .../src/org/eclipse/jgit/internal/storage/file/PackIndexV2.java | 8 ++++++++ 4 files changed, 21 insertions(+), 10 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java index f87329ccc2..be457644d9 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java @@ -187,18 +187,18 @@ public class Pack implements Iterable { } if (packChecksum == null) { - packChecksum = idx.packChecksum; + packChecksum = idx.getChecksum(); fileSnapshot.setChecksum( ObjectId.fromRaw(packChecksum)); } else if (!Arrays.equals(packChecksum, - idx.packChecksum)) { + idx.getChecksum())) { throw new PackMismatchException(MessageFormat .format(JGitText.get().packChecksumMismatch, packFile.getPath(), PackExt.PACK.getExtension(), Hex.toHexString(packChecksum), PackExt.INDEX.getExtension(), - Hex.toHexString(idx.packChecksum))); + Hex.toHexString(idx.getChecksum()))); } loadedIdx = optionally(idx); return idx; @@ -791,7 +791,7 @@ public class Pack implements Iterable { MessageFormat.format(JGitText.get().packChecksumMismatch, getPackFile(), PackExt.PACK.getExtension(), Hex.toHexString(buf), PackExt.INDEX.getExtension(), - Hex.toHexString(idx.packChecksum))); + Hex.toHexString(idx.getChecksum()))); } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java index c2c3775d67..8a64c3247a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java @@ -116,9 +116,6 @@ public abstract class PackIndex return true; } - /** Footer checksum applied on the bottom of the pack file. */ - protected byte[] packChecksum; - /** * Determine if an object is contained within the pack file. * @@ -297,9 +294,7 @@ public abstract class PackIndex * @return the checksum of the pack; caller must not modify it * @since 5.5 */ - public byte[] getChecksum() { - return packChecksum; - } + public abstract byte[] getChecksum(); /** * Represent mutable entry of pack index consisting of object id and offset diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV1.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV1.java index 5180df46bf..21f7314f33 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV1.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV1.java @@ -36,6 +36,9 @@ class PackIndexV1 extends PackIndex { private final long[] idxHeader; + /** Footer checksum applied on the bottom of the pack file. */ + protected byte[] packChecksum; + byte[][] idxdata; private long objectCnt; @@ -238,6 +241,11 @@ class PackIndexV1 extends PackIndex { return (RECORD_SIZE * mid) + 4; } + @Override + public byte[] getChecksum() { + return packChecksum; + } + private class IndexV1Iterator extends EntriesIterator { int levelOne; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV2.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV2.java index 751b62dc40..9a6f4a421a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV2.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV2.java @@ -37,6 +37,9 @@ class PackIndexV2 extends PackIndex { private static final byte[] NO_BYTES = {}; + /** Footer checksum applied on the bottom of the pack file. */ + protected byte[] packChecksum; + private long objectCnt; private final long[] fanoutTable; @@ -281,6 +284,11 @@ class PackIndexV2 extends PackIndex { return -1; } + @Override + public byte[] getChecksum() { + return packChecksum; + } + private class EntriesIteratorV2 extends EntriesIterator { int levelOne; -- cgit v1.2.3 From 0647785eb22c2547d2ea2a5afe17cdb7082a2952 Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Thu, 28 Mar 2024 15:07:34 -0700 Subject: PackIndex: Make iterator static (pass object count in ctor) The iterator uses an implicit reference to the external class to call #getObjectCount(). This implicit reference prevents PackIndex to become an interface. The object count is immutable and known at index (and iterator) construction time. Pass the object count in the constructor of the iterator and make it static. Change-Id: I4949ff943de2a88380fb7f4c9b65b089734854b9 --- .../src/org/eclipse/jgit/internal/storage/file/PackIndex.java | 10 ++++++++-- .../org/eclipse/jgit/internal/storage/file/PackIndexV1.java | 6 +++++- .../org/eclipse/jgit/internal/storage/file/PackIndexV2.java | 6 +++++- 3 files changed, 18 insertions(+), 4 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java index 8a64c3247a..a23bc4f0a5 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java @@ -353,16 +353,22 @@ public abstract class PackIndex } } - abstract class EntriesIterator implements Iterator { + static abstract class EntriesIterator implements Iterator { protected final MutableEntry entry = initEntry(); + private final long objectCount; + + protected EntriesIterator(long objectCount) { + this.objectCount = objectCount; + } + protected long returnedNumber = 0; protected abstract MutableEntry initEntry(); @Override public boolean hasNext() { - return returnedNumber < getObjectCount(); + return returnedNumber < objectCount; } /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV1.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV1.java index 21f7314f33..3496a2a20c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV1.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV1.java @@ -203,7 +203,7 @@ class PackIndexV1 extends PackIndex { @Override public Iterator iterator() { - return new IndexV1Iterator(); + return new IndexV1Iterator(objectCnt); } @Override @@ -251,6 +251,10 @@ class PackIndexV1 extends PackIndex { int levelTwo; + IndexV1Iterator(long objectCount) { + super(objectCount); + } + @Override protected MutableEntry initEntry() { return new MutableEntry() { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV2.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV2.java index 9a6f4a421a..12dfeb6695 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV2.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV2.java @@ -224,7 +224,7 @@ class PackIndexV2 extends PackIndex { @Override public Iterator iterator() { - return new EntriesIteratorV2(); + return new EntriesIteratorV2(objectCnt); } @Override @@ -294,6 +294,10 @@ class PackIndexV2 extends PackIndex { int levelTwo; + EntriesIteratorV2(long objectCount){ + super(objectCount); + } + @Override protected MutableEntry initEntry() { return new MutableEntry() { -- cgit v1.2.3 From 9d5555d3c269a4b537e47c43e4f7ea51b5375f86 Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Thu, 28 Mar 2024 15:11:05 -0700 Subject: PackIndex: Make it an interface As we work on different backends for PackIndex, it is much easier to extend an interface than an abstract class. Note that PackIndex is alredy acting as an interface, offering barely any functionality. Change-Id: Icd0db0e96a097631c2b9c3b05e5700f601f606d5 --- .../jgit/internal/storage/file/PackIndex.java | 43 ++++++++++++---------- .../jgit/internal/storage/file/PackIndexV1.java | 4 +- .../jgit/internal/storage/file/PackIndexV2.java | 2 +- 3 files changed, 26 insertions(+), 23 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java index a23bc4f0a5..c0540d5a4c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java @@ -42,8 +42,8 @@ import org.eclipse.jgit.util.io.SilentFileInputStream; * by ObjectId. *

*/ -public abstract class PackIndex - implements Iterable, ObjectIdSet { +public interface PackIndex + extends Iterable, ObjectIdSet { /** * Open an existing pack .idx file for reading. *

@@ -61,7 +61,7 @@ public abstract class PackIndex * the file exists but could not be read due to security errors, * unrecognized data version, or unexpected data corruption. */ - public static PackIndex open(File idxFile) throws IOException { + static PackIndex open(File idxFile) throws IOException { try (SilentFileInputStream fd = new SilentFileInputStream( idxFile)) { return read(fd); @@ -92,7 +92,7 @@ public abstract class PackIndex * @throws org.eclipse.jgit.errors.CorruptObjectException * the stream does not contain a valid pack index. */ - public static PackIndex read(InputStream fd) throws IOException, + static PackIndex read(InputStream fd) throws IOException, CorruptObjectException { final byte[] hdr = new byte[8]; IO.readFully(fd, hdr, 0, hdr.length); @@ -123,12 +123,12 @@ public abstract class PackIndex * the object to look for. Must not be null. * @return true if the object is listed in this index; false otherwise. */ - public boolean hasObject(AnyObjectId id) { + default boolean hasObject(AnyObjectId id) { return findOffset(id) != -1; } @Override - public boolean contains(AnyObjectId id) { + default boolean contains(AnyObjectId id) { return findOffset(id) != -1; } @@ -144,7 +144,7 @@ public abstract class PackIndex *

*/ @Override - public abstract Iterator iterator(); + Iterator iterator(); /** * Obtain the total number of objects described by this index. @@ -152,7 +152,7 @@ public abstract class PackIndex * @return number of objects in this index, and likewise in the associated * pack that this index was generated from. */ - public abstract long getObjectCount(); + long getObjectCount(); /** * Obtain the total number of objects needing 64 bit offsets. @@ -160,7 +160,7 @@ public abstract class PackIndex * @return number of objects in this index using a 64 bit offset; that is an * object positioned after the 2 GB position within the file. */ - public abstract long getOffset64Count(); + long getOffset64Count(); /** * Get ObjectId for the n-th object entry returned by {@link #iterator()}. @@ -182,7 +182,7 @@ public abstract class PackIndex * is 0, the second is 1, etc. * @return the ObjectId for the corresponding entry. */ - public abstract ObjectId getObjectId(long nthPosition); + ObjectId getObjectId(long nthPosition); /** * Get ObjectId for the n-th object entry returned by {@link #iterator()}. @@ -206,7 +206,7 @@ public abstract class PackIndex * negative, but still valid. * @return the ObjectId for the corresponding entry. */ - public final ObjectId getObjectId(int nthPosition) { + default ObjectId getObjectId(int nthPosition) { if (nthPosition >= 0) return getObjectId((long) nthPosition); final int u31 = nthPosition >>> 1; @@ -225,7 +225,7 @@ public abstract class PackIndex * etc. Positions past 2**31-1 are negative, but still valid. * @return the offset in a pack for the corresponding entry. */ - protected abstract long getOffset(long nthPosition); + long getOffset(long nthPosition); /** * Locate the file offset position for the requested object. @@ -236,7 +236,7 @@ public abstract class PackIndex * object does not exist in this index and is thus not stored in the * associated pack. */ - public abstract long findOffset(AnyObjectId objId); + long findOffset(AnyObjectId objId); /** * Locate the position of this id in the list of object-ids in the index @@ -247,7 +247,7 @@ public abstract class PackIndex * of ids stored in this index; -1 if the object does not exist in * this index and is thus not stored in the associated pack. */ - public abstract int findPosition(AnyObjectId objId); + int findPosition(AnyObjectId objId); /** * Retrieve stored CRC32 checksum of the requested object raw-data @@ -261,7 +261,7 @@ public abstract class PackIndex * @throws java.lang.UnsupportedOperationException * when this index doesn't support CRC32 checksum */ - public abstract long findCRC32(AnyObjectId objId) + long findCRC32(AnyObjectId objId) throws MissingObjectException, UnsupportedOperationException; /** @@ -269,7 +269,7 @@ public abstract class PackIndex * * @return true if CRC32 is stored, false otherwise */ - public abstract boolean hasCRC32Support(); + boolean hasCRC32Support(); /** * Find objects matching the prefix abbreviation. @@ -285,7 +285,7 @@ public abstract class PackIndex * @throws java.io.IOException * the index cannot be read. */ - public abstract void resolve(Set matches, AbbreviatedObjectId id, + void resolve(Set matches, AbbreviatedObjectId id, int matchLimit) throws IOException; /** @@ -294,14 +294,14 @@ public abstract class PackIndex * @return the checksum of the pack; caller must not modify it * @since 5.5 */ - public abstract byte[] getChecksum(); + byte[] getChecksum(); /** * Represent mutable entry of pack index consisting of object id and offset * in pack (both mutable). * */ - public static class MutableEntry { + class MutableEntry { final MutableObjectId idBuffer = new MutableObjectId(); long offset; @@ -353,7 +353,10 @@ public abstract class PackIndex } } - static abstract class EntriesIterator implements Iterator { + /** + * Base implementation of the iterator over index entries. + */ + abstract class EntriesIterator implements Iterator { protected final MutableEntry entry = initEntry(); private final long objectCount; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV1.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV1.java index 3496a2a20c..d7c83785d8 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV1.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV1.java @@ -29,7 +29,7 @@ import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.util.IO; import org.eclipse.jgit.util.NB; -class PackIndexV1 extends PackIndex { +class PackIndexV1 implements PackIndex { private static final int IDX_HDR_LEN = 256 * 4; private static final int RECORD_SIZE = 4 + Constants.OBJECT_ID_LENGTH; @@ -121,7 +121,7 @@ class PackIndexV1 extends PackIndex { } @Override - protected long getOffset(long nthPosition) { + public long getOffset(long nthPosition) { final int levelOne = findLevelOne(nthPosition); final int levelTwo = getLevelTwo(nthPosition, levelOne); final int p = (4 + Constants.OBJECT_ID_LENGTH) * levelTwo; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV2.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV2.java index 12dfeb6695..caf8b71180 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV2.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV2.java @@ -28,7 +28,7 @@ import org.eclipse.jgit.util.IO; import org.eclipse.jgit.util.NB; /** Support for the pack index v2 format. */ -class PackIndexV2 extends PackIndex { +class PackIndexV2 implements PackIndex { private static final long IS_O64 = 1L << 31; private static final int FANOUT = 256; -- cgit v1.2.3 From c1f95130c554580afc1d6090d30b3946b578714e Mon Sep 17 00:00:00 2001 From: Yury Molchan Date: Wed, 6 Dec 2023 04:29:39 +0200 Subject: Check an execution bit by reading it from the file system Files.isExecutable() checks possibility to execute a file from the JVM, not POSIX attributes. So it fails when Java Security Manager does not allow to execute a file, but file has X-bit. Change-Id: I85b96fb6cedbaff1b3c063728c25b6ce4d04e9b9 --- org.eclipse.jgit/src/org/eclipse/jgit/util/FS_POSIX.java | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_POSIX.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_POSIX.java index ee907f2ee6..e73095f5a8 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_POSIX.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_POSIX.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010, Robin Rosenberg and others + * Copyright (C) 2010, 2024, Robin Rosenberg and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -203,7 +203,16 @@ public class FS_POSIX extends FS { /** {@inheritDoc} */ @Override public boolean canExecute(File f) { - return FileUtils.canExecute(f); + if (!isFile(f)) { + return false; + } + try { + Path path = FileUtils.toPath(f); + Set pset = Files.getPosixFilePermissions(path); + return pset.contains(PosixFilePermission.OWNER_EXECUTE); + } catch (IOException ex) { + return false; + } } /** {@inheritDoc} */ -- cgit v1.2.3 From 6e9a170364efc727f4368cf5304115c7b278a649 Mon Sep 17 00:00:00 2001 From: Patrick Hiesel Date: Fri, 10 Mar 2023 16:50:37 +0100 Subject: Allow applying a patch with conflicts In some settings, we want to let users apply a patch that does not cleanly apply and add conflict markers. In Gerrit, this is useful when cherry picking (via Git patches) from one host to another. This commit takes a simple approach: If a hunk doesn't apply, go to the pre-image line, treat all lines in pre-image length as left side of the conflict and all context and newly added lines as right side of the conflict. Change-Id: I01411d7a32b3f3207097b26231909aae6b835650 --- .../eclipse/jgit/diff/ConflictOutOfBounds.patch | 10 ++ .../jgit/diff/ConflictOutOfBounds_PostImage | 15 +++ .../eclipse/jgit/diff/ConflictOutOfBounds_PreImage | 8 ++ .../org/eclipse/jgit/diff/allowconflict.patch | 10 ++ .../org/eclipse/jgit/diff/allowconflict_PostImage | 15 +++ .../org/eclipse/jgit/diff/allowconflict_PreImage | 8 ++ .../org/eclipse/jgit/patch/PatchApplierTest.java | 55 ++++++++- .../src/org/eclipse/jgit/patch/PatchApplier.java | 134 ++++++++++++++++++--- 8 files changed, 234 insertions(+), 21 deletions(-) create mode 100644 org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/ConflictOutOfBounds.patch create mode 100644 org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/ConflictOutOfBounds_PostImage create mode 100644 org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/ConflictOutOfBounds_PreImage create mode 100644 org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/allowconflict.patch create mode 100644 org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/allowconflict_PostImage create mode 100644 org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/allowconflict_PreImage (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/ConflictOutOfBounds.patch b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/ConflictOutOfBounds.patch new file mode 100644 index 0000000000..6e7448b95c --- /dev/null +++ b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/ConflictOutOfBounds.patch @@ -0,0 +1,10 @@ +diff --git a/ConflictOutOfBounds b/ConflictOutOfBounds +index 0000000..de98044 +--- a/ConflictOutOfBounds ++++ b/ConflictOutOfBounds +@@ -25,4 +25,4 @@ + line3 +-lineA ++lineB + line5 + line6 diff --git a/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/ConflictOutOfBounds_PostImage b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/ConflictOutOfBounds_PostImage new file mode 100644 index 0000000000..4e5d5b2d88 --- /dev/null +++ b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/ConflictOutOfBounds_PostImage @@ -0,0 +1,15 @@ +line1 +line2 +line3 +line4 +line5 +line6 +line7 +line8 +<<<<<<< HEAD +======= +line3 +lineB +line5 +line6 +>>>>>>> PATCH diff --git a/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/ConflictOutOfBounds_PreImage b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/ConflictOutOfBounds_PreImage new file mode 100644 index 0000000000..f62562a216 --- /dev/null +++ b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/ConflictOutOfBounds_PreImage @@ -0,0 +1,8 @@ +line1 +line2 +line3 +line4 +line5 +line6 +line7 +line8 diff --git a/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/allowconflict.patch b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/allowconflict.patch new file mode 100644 index 0000000000..a99e636382 --- /dev/null +++ b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/allowconflict.patch @@ -0,0 +1,10 @@ +diff --git a/allowconflict b/allowconflict +index 0000000..de98044 +--- a/allowconflict ++++ b/allowconflict +@@ -3,4 +3,4 @@ + line3 +-lineA ++lineB + line5 + line6 diff --git a/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/allowconflict_PostImage b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/allowconflict_PostImage new file mode 100644 index 0000000000..a963b40dfe --- /dev/null +++ b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/allowconflict_PostImage @@ -0,0 +1,15 @@ +line1 +line2 +<<<<<<< HEAD +line3 +line4 +line5 +line6 +======= +line3 +lineB +line5 +line6 +>>>>>>> PATCH +line7 +line8 diff --git a/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/allowconflict_PreImage b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/allowconflict_PreImage new file mode 100644 index 0000000000..f62562a216 --- /dev/null +++ b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/diff/allowconflict_PreImage @@ -0,0 +1,8 @@ +line1 +line2 +line3 +line4 +line5 +line6 +line7 +line8 diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/patch/PatchApplierTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/patch/PatchApplierTest.java index 2aac15bbb6..e4b0ffbc21 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/patch/PatchApplierTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/patch/PatchApplierTest.java @@ -27,6 +27,7 @@ import java.nio.file.Files; import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.errors.PatchFormatException; import org.eclipse.jgit.attributes.FilterCommand; import org.eclipse.jgit.attributes.FilterCommandFactory; import org.eclipse.jgit.attributes.FilterCommandRegistry; @@ -48,8 +49,7 @@ import org.junit.runner.RunWith; import org.junit.runners.Suite; @RunWith(Suite.class) -@Suite.SuiteClasses({ - PatchApplierTest.WithWorktree. class, // +@Suite.SuiteClasses({ PatchApplierTest.WithWorktree.class, // PatchApplierTest.InCore.class, // }) public class PatchApplierTest { @@ -128,6 +128,20 @@ public class PatchApplierTest { } } + protected Result applyPatchAllowConflicts() throws IOException { + InputStream patchStream = getTestResource(name + ".patch"); + Patch patch = new Patch(); + patch.parse(patchStream); + if (inCore) { + try (ObjectInserter oi = db.newObjectInserter()) { + return new PatchApplier(db, baseTip, oi).allowConflicts() + .applyPatch(patch); + } + } + return new PatchApplier(db).allowConflicts() + .applyPatch(patch); + } + protected static InputStream getTestResource(String patchFile) { return PatchApplierTest.class.getClassLoader() .getResourceAsStream("org/eclipse/jgit/diff/" + patchFile); @@ -169,6 +183,13 @@ public class PatchApplierTest { verifyContent(result, aName, exists); } + void verifyChange(Result result, String aName, boolean exists, + int numConflicts) throws Exception { + assertEquals(numConflicts, result.getErrors().size()); + assertEquals(1, result.getPaths().size()); + verifyContent(result, aName, exists); + } + protected byte[] readBlob(ObjectId treeish, String path) throws Exception { try (TestRepository tr = new TestRepository<>(db); @@ -345,6 +366,36 @@ public class PatchApplierTest { verifyChange(result, "CopyResult", true); } + @Test + public void testConflictMarkers() throws Exception { + init("allowconflict", true, true); + + Result result = applyPatchAllowConflicts(); + + assertEquals(result.getErrors().size(), 1); + PatchApplier.Result.Error error = result.getErrors().get(0); + error.hh = null; // We don't assert the hunk header as it is a + // complex object with lots of internal state. + assertEquals(error, new PatchApplier.Result.Error( + "cannot apply hunk", "allowconflict", null)); + verifyChange(result, "allowconflict", true, 1); + } + + @Test + public void testConflictMarkersOutOfBounds() throws Exception { + init("ConflictOutOfBounds", true, true); + + Result result = applyPatchAllowConflicts(); + + assertEquals(result.getErrors().size(), 1); + PatchApplier.Result.Error error = result.getErrors().get(0); + error.hh = null; // We don't assert the hunk header as it is a + // complex object with lots of internal state. + assertEquals(error, new PatchApplier.Result.Error( + "cannot apply hunk", "ConflictOutOfBounds", null)); + verifyChange(result, "ConflictOutOfBounds", true, 1); + } + @Test public void testShiftUp() throws Exception { init("ShiftUp"); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/patch/PatchApplier.java b/org.eclipse.jgit/src/org/eclipse/jgit/patch/PatchApplier.java index a327095c81..84c2ec43dc 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/patch/PatchApplier.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/patch/PatchApplier.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteBuffer; +import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.StandardCopyOption; @@ -33,9 +34,11 @@ import java.util.Arrays; import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; import java.util.zip.InflaterInputStream; + import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.api.errors.FilterFailedException; import org.eclipse.jgit.api.errors.PatchFormatException; @@ -101,11 +104,12 @@ import org.eclipse.jgit.util.sha1.SHA1; * @since 6.4 */ public class PatchApplier { - private static final byte[] NO_EOL = "\\ No newline at end of file" //$NON-NLS-1$ .getBytes(StandardCharsets.US_ASCII); - /** The tree before applying the patch. Only non-null for inCore operation. */ + /** + * The tree before applying the patch. Only non-null for inCore operation. + */ @Nullable private final RevTree beforeTree; @@ -115,10 +119,14 @@ public class PatchApplier { private final ObjectReader reader; + private final Charset charset; + private WorkingTreeOptions workingTreeOptions; private int inCoreSizeLimit; + private boolean allowConflicts; + /** * @param repo * repository to apply the patch in @@ -128,7 +136,8 @@ public class PatchApplier { inserter = repo.newObjectInserter(); reader = inserter.newReader(); beforeTree = null; - + allowConflicts = false; + charset = StandardCharsets.UTF_8; Config config = repo.getConfig(); workingTreeOptions = config.get(WorkingTreeOptions.KEY); inCoreSizeLimit = config.getInt(ConfigConstants.CONFIG_MERGE_SECTION, @@ -143,11 +152,14 @@ public class PatchApplier { * @param oi * to be used for modifying objects */ - public PatchApplier(Repository repo, RevTree beforeTree, ObjectInserter oi) { + public PatchApplier(Repository repo, RevTree beforeTree, + ObjectInserter oi) { this.repo = repo; this.beforeTree = beforeTree; inserter = oi; reader = oi.newReader(); + allowConflicts = false; + charset = StandardCharsets.UTF_8; } /** @@ -157,7 +169,6 @@ public class PatchApplier { * @since 6.3 */ public static class Result { - /** * A wrapper for a patch applying error that affects a given file. * @@ -166,13 +177,14 @@ public class PatchApplier { // TODO(ms): rename this class in next major release @SuppressWarnings("JavaLangClash") public static class Error { + String msg; + + String oldFileName; - private String msg; - private String oldFileName; - private @Nullable HunkHeader hh; + @Nullable + HunkHeader hh; - private Error(String msg, String oldFileName, - @Nullable HunkHeader hh) { + Error(String msg, String oldFileName, @Nullable HunkHeader hh) { this.msg = msg; this.oldFileName = oldFileName; this.hh = hh; @@ -181,13 +193,33 @@ public class PatchApplier { @Override public String toString() { if (hh != null) { - return MessageFormat.format(JGitText.get().patchApplyErrorWithHunk, - oldFileName, hh, msg); + return MessageFormat.format( + JGitText.get().patchApplyErrorWithHunk, oldFileName, + hh, msg); } - return MessageFormat.format(JGitText.get().patchApplyErrorWithoutHunk, - oldFileName, msg); + return MessageFormat.format( + JGitText.get().patchApplyErrorWithoutHunk, oldFileName, + msg); } + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || !(o instanceof Error)) { + return false; + } + Error error = (Error) o; + return Objects.equals(msg, error.msg) + && Objects.equals(oldFileName, error.oldFileName) + && Objects.equals(hh, error.hh); + } + + @Override + public int hashCode() { + return Objects.hash(msg, oldFileName, hh); + } } private ObjectId treeId; @@ -357,6 +389,17 @@ public class PatchApplier { return result; } + /** + * Sets up the {@link PatchApplier} to apply patches even if they conflict. + * + * @return the {@link PatchApplier} to apply any patches + * @since 6.10 + */ + public PatchApplier allowConflicts() { + allowConflicts = true; + return this; + } + private File getFile(String path) { return inCore() ? null : new File(repo.getWorkTree(), path); } @@ -439,6 +482,7 @@ public class PatchApplier { return false; } } + private static final int FILE_TREE_INDEX = 1; /** @@ -539,7 +583,9 @@ public class PatchApplier { convertCrLf); resultStreamLoader = applyText(raw, fh, result); } - if (resultStreamLoader == null || !result.getErrors().isEmpty()) { + if (resultStreamLoader == null + || (!result.getErrors().isEmpty() && result.getErrors().stream() + .anyMatch(e -> !e.msg.equals("cannot apply hunk")))) { //$NON-NLS-1$ return; } @@ -961,9 +1007,51 @@ public class PatchApplier { } } if (!applies) { - result.addError(JGitText.get().applyTextPatchCannotApplyHunk, - fh.getOldPath(), hh); - return null; + if (!allowConflicts) { + result.addError( + JGitText.get().applyTextPatchCannotApplyHunk, + fh.getOldPath(), hh); + return null; + } + // Insert conflict markers. This is best-guess because the + // file might have changed completely. But at least we give + // the user a graceful state that they can resolve manually. + // An alternative to this is using the 3-way merger. This + // only works if the pre-image SHA is contained in the repo. + // If that was the case, cherry-picking the original commit + // should be preferred to apply a patch. + result.addError("cannot apply hunk", fh.getOldPath(), hh); //$NON-NLS-1$ + newLines.add(Math.min(applyAt++, newLines.size()), + asBytes("<<<<<<< HEAD")); //$NON-NLS-1$ + applyAt += hh.getOldImage().lineCount; + newLines.add(Math.min(applyAt++, newLines.size()), + asBytes("=======")); //$NON-NLS-1$ + + int sz = hunkLines.size(); + for (int j = 1; j < sz; j++) { + ByteBuffer hunkLine = hunkLines.get(j); + if (!hunkLine.hasRemaining()) { + // Completely empty line; accept as empty context + // line + applyAt++; + lastWasRemoval = false; + continue; + } + switch (hunkLine.array()[hunkLine.position()]) { + case ' ': + case '+': + newLines.add(Math.min(applyAt++, newLines.size()), + slice(hunkLine, 1)); + break; + case '-': + case '\\': + default: + break; + } + } + newLines.add(Math.min(applyAt++, newLines.size()), + asBytes(">>>>>>> PATCH")); //$NON-NLS-1$ + continue; } // Hunk applies at applyAt. Apply it, and update afterLastHunk and // lineNumberShift @@ -1010,7 +1098,11 @@ public class PatchApplier { } else if (!rt.isMissingNewlineAtEnd()) { newLines.add(null); } + return toContentStreamLoader(newLines); + } + private static ContentStreamLoader toContentStreamLoader( + List newLines) throws IOException { // We could check if old == new, but the short-circuiting complicates // logic for inCore patching, so just write the new thing regardless. TemporaryBuffer buffer = new TemporaryBuffer.LocalFile(null); @@ -1034,6 +1126,10 @@ public class PatchApplier { } } + private ByteBuffer asBytes(String str) { + return ByteBuffer.wrap(str.getBytes(charset)); + } + @SuppressWarnings("ByteBufferBackingArray") private boolean canApplyAt(List hunkLines, List newLines, int line) { @@ -1123,4 +1219,4 @@ public class PatchApplier { in.close(); } } -} +} \ No newline at end of file -- cgit v1.2.3 From e6217e2fe60c3c36bd14feeb2aae14e9019170c6 Mon Sep 17 00:00:00 2001 From: "jackdt@google.com" Date: Wed, 15 May 2024 14:41:57 -0700 Subject: Do not use ArrayList when there will be deletions In https://gerrithub.io/c/eclipse-jgit/jgit/+/1194015, LinkedList was replaced with ArrayList in DfsReader and WalkFetchConnection. In this case, the Iterator.remove() method of List is called, which is an O(n) operation for ArrayList. This results in an O(n^2) algorithm. Instead of reverting to LinkedList, use a HashSet and LinkedHashmap instead. This maintains O(1) removal, and is less likely to be treated as an antipattern than LinkedList. A likely innocuous usage of Iterator.remove() in UnionInputStream was also fixed. Change-Id: I57d884440c726b2fc80c1b8b2bec9bd7e2e6e3fe --- .../jgit/internal/storage/dfs/DfsReader.java | 14 ++++----- .../jgit/transport/WalkFetchConnection.java | 36 ++++++++++++---------- .../org/eclipse/jgit/util/io/UnionInputStream.java | 7 ++--- 3 files changed, 30 insertions(+), 27 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java index c939114f5f..9cfcbaa5f7 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java @@ -307,7 +307,7 @@ public class DfsReader extends ObjectReader implements ObjectReuseAsIs { private Iterable> findAll( Iterable objectIds) throws IOException { - Collection pending = new ArrayList<>(); + HashSet pending = new HashSet<>(); for (T id : objectIds) { pending.add(id); } @@ -327,22 +327,21 @@ public class DfsReader extends ObjectReader implements ObjectReuseAsIs { } private void findAllImpl(PackList packList, - Collection pending, List> r) { + HashSet pending, List> r) { DfsPackFile[] packs = packList.packs; if (packs.length == 0) { return; } int lastIdx = 0; DfsPackFile lastPack = packs[lastIdx]; - - OBJECT_SCAN: for (Iterator it = pending.iterator(); it.hasNext();) { - T t = it.next(); + HashSet toRemove = new HashSet<>(); + OBJECT_SCAN: for (T t : pending) { if (!skipGarbagePack(lastPack)) { try { long p = lastPack.findOffset(this, t); if (0 < p) { r.add(new FoundObject<>(t, lastIdx, lastPack, p)); - it.remove(); + toRemove.add(t); continue; } } catch (IOException e) { @@ -360,7 +359,7 @@ public class DfsReader extends ObjectReader implements ObjectReuseAsIs { long p = pack.findOffset(this, t); if (0 < p) { r.add(new FoundObject<>(t, i, pack, p)); - it.remove(); + toRemove.add(t); lastIdx = i; lastPack = pack; continue OBJECT_SCAN; @@ -370,6 +369,7 @@ public class DfsReader extends ObjectReader implements ObjectReuseAsIs { } } } + pending.removeAll(toRemove); last = lastPack; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkFetchConnection.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkFetchConnection.java index 3da76f38fa..ea789b1c09 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkFetchConnection.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkFetchConnection.java @@ -22,8 +22,10 @@ import java.util.Collection; import java.util.Deque; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; +import java.util.Map.Entry; import java.util.Set; import org.eclipse.jgit.errors.CompoundException; @@ -122,7 +124,7 @@ class WalkFetchConnection extends BaseFetchConnection { private final Deque noAlternatesYet; /** Packs we have discovered, but have not yet fetched locally. */ - private final Deque unfetchedPacks; + private final Map unfetchedPacks; /** * Packs whose indexes we have looked at in {@link #unfetchedPacks}. @@ -164,7 +166,7 @@ class WalkFetchConnection extends BaseFetchConnection { remotes = new ArrayList<>(); remotes.add(w); - unfetchedPacks = new ArrayDeque<>(); + unfetchedPacks = new LinkedHashMap<>(); packsConsidered = new HashSet<>(); noPacksYet = new ArrayDeque<>(); @@ -227,7 +229,7 @@ class WalkFetchConnection extends BaseFetchConnection { public void close() { inserter.close(); reader.close(); - for (RemotePack p : unfetchedPacks) { + for (RemotePack p : unfetchedPacks.values()) { if (p.tmpIdx != null) p.tmpIdx.delete(); } @@ -423,7 +425,7 @@ class WalkFetchConnection extends BaseFetchConnection { continue; for (String packName : packNameList) { if (packsConsidered.add(packName)) - unfetchedPacks.add(new RemotePack(wrr, packName)); + unfetchedPacks.put(packName, new RemotePack(wrr, packName)); } if (downloadPackedObject(pm, id)) return; @@ -472,9 +474,12 @@ class WalkFetchConnection extends BaseFetchConnection { // Search for the object in a remote pack whose index we have, // but whose pack we do not yet have. // - final Iterator packItr = unfetchedPacks.iterator(); - while (packItr.hasNext() && !monitor.isCancelled()) { - final RemotePack pack = packItr.next(); + Set toRemove = new HashSet<>(); + for (Entry entry : unfetchedPacks.entrySet()) { + if (monitor.isCancelled()) { + break; + } + final RemotePack pack = entry.getValue(); try { pack.openIndex(monitor); } catch (IOException err) { @@ -484,7 +489,7 @@ class WalkFetchConnection extends BaseFetchConnection { // another source, so don't consider it a failure. // recordError(id, err); - packItr.remove(); + toRemove.add(entry.getKey()); continue; } @@ -535,7 +540,7 @@ class WalkFetchConnection extends BaseFetchConnection { } throw new TransportException(e.getMessage(), e); } - packItr.remove(); + toRemove.add(entry.getKey()); } if (!alreadyHave(id)) { @@ -550,11 +555,9 @@ class WalkFetchConnection extends BaseFetchConnection { // Complete any other objects that we can. // - final Iterator pending = swapFetchQueue(); - while (pending.hasNext()) { - final ObjectId p = pending.next(); + final Deque pending = swapFetchQueue(); + for (ObjectId p : pending) { if (pack.index.hasObject(p)) { - pending.remove(); process(p); } else { workQueue.add(p); @@ -563,11 +566,12 @@ class WalkFetchConnection extends BaseFetchConnection { return true; } + toRemove.forEach(unfetchedPacks::remove); return false; } - private Iterator swapFetchQueue() { - final Iterator r = workQueue.iterator(); + private Deque swapFetchQueue() { + final Deque r = workQueue; workQueue = new ArrayDeque<>(); return r; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/UnionInputStream.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/UnionInputStream.java index c3a1c4e3bb..7e950f6529 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/UnionInputStream.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/UnionInputStream.java @@ -14,7 +14,6 @@ import java.io.IOException; import java.io.InputStream; import java.util.ArrayDeque; import java.util.Deque; -import java.util.Iterator; /** * An InputStream which reads from one or more InputStreams. @@ -164,14 +163,14 @@ public class UnionInputStream extends InputStream { public void close() throws IOException { IOException err = null; - for (Iterator i = streams.iterator(); i.hasNext();) { + for (InputStream stream : streams) { try { - i.next().close(); + stream.close(); } catch (IOException closeError) { err = closeError; } - i.remove(); } + streams.clear(); if (err != null) throw err; -- cgit v1.2.3 From cac835835d2e7789f2a01c03da83b44185668269 Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Thu, 16 May 2024 12:28:53 -0700 Subject: WalkFetchConnection: Remove marked packs on all function exits [1] replaces Iterator.remove() with a list of "toRemove" that gets processed when returning at the end. There are two others returns in the function where the list is not processed. Let the method report the broken packages and wrap it so the caller can clean them up in any case. [1] https://review.gerrithub.io/c/eclipse-jgit/jgit/+/1194812 Change-Id: I1ffb7829039f644df03b0b3ea1e5d10408ce19b7 --- .../eclipse/jgit/transport/WalkFetchConnection.java | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkFetchConnection.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkFetchConnection.java index ea789b1c09..b7bb0cbce3 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkFetchConnection.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkFetchConnection.java @@ -424,8 +424,9 @@ class WalkFetchConnection extends BaseFetchConnection { if (packNameList == null || packNameList.isEmpty()) continue; for (String packName : packNameList) { - if (packsConsidered.add(packName)) + if (packsConsidered.add(packName)) { unfetchedPacks.put(packName, new RemotePack(wrr, packName)); + } } if (downloadPackedObject(pm, id)) return; @@ -468,13 +469,22 @@ class WalkFetchConnection extends BaseFetchConnection { } } + private boolean downloadPackedObject(ProgressMonitor monitor, + AnyObjectId id) throws TransportException { + Set brokenPacks = new HashSet<>(); + try { + return downloadPackedObject(monitor, id, brokenPacks); + } finally { + brokenPacks.forEach(unfetchedPacks::remove); + } + } + @SuppressWarnings("Finally") private boolean downloadPackedObject(final ProgressMonitor monitor, - final AnyObjectId id) throws TransportException { + final AnyObjectId id, Set brokenPacks) throws TransportException { // Search for the object in a remote pack whose index we have, // but whose pack we do not yet have. // - Set toRemove = new HashSet<>(); for (Entry entry : unfetchedPacks.entrySet()) { if (monitor.isCancelled()) { break; @@ -489,7 +499,7 @@ class WalkFetchConnection extends BaseFetchConnection { // another source, so don't consider it a failure. // recordError(id, err); - toRemove.add(entry.getKey()); + brokenPacks.add(entry.getKey()); continue; } @@ -540,7 +550,7 @@ class WalkFetchConnection extends BaseFetchConnection { } throw new TransportException(e.getMessage(), e); } - toRemove.add(entry.getKey()); + brokenPacks.add(entry.getKey()); } if (!alreadyHave(id)) { @@ -566,7 +576,6 @@ class WalkFetchConnection extends BaseFetchConnection { return true; } - toRemove.forEach(unfetchedPacks::remove); return false; } -- cgit v1.2.3 From c0c59ccf2d320c1d50f3ad949e5369ae167b8fb5 Mon Sep 17 00:00:00 2001 From: Patrick Hiesel Date: Mon, 27 May 2024 16:11:40 +0200 Subject: PatchApplier: Set a boolean on the result if conflict markers were added This will let callers show a different error message or mark the state as conflicting. Change-Id: Id8eea614b6b8d54c62b49ffbac90599e6f4c5efa --- .../org/eclipse/jgit/patch/PatchApplierTest.java | 4 +-- .../src/org/eclipse/jgit/patch/PatchApplier.java | 37 ++++++++++++++++++---- 2 files changed, 33 insertions(+), 8 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/patch/PatchApplierTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/patch/PatchApplierTest.java index 8f3478db18..eec403a976 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/patch/PatchApplierTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/patch/PatchApplierTest.java @@ -376,7 +376,7 @@ public class PatchApplierTest { error.hh = null; // We don't assert the hunk header as it is a // complex object with lots of internal state. assertEquals(error, new PatchApplier.Result.Error( - "cannot apply hunk", "allowconflict", null)); + "cannot apply hunk", "allowconflict", null, true)); verifyChange(result, "allowconflict", true, 1); } @@ -391,7 +391,7 @@ public class PatchApplierTest { error.hh = null; // We don't assert the hunk header as it is a // complex object with lots of internal state. assertEquals(error, new PatchApplier.Result.Error( - "cannot apply hunk", "ConflictOutOfBounds", null)); + "cannot apply hunk", "ConflictOutOfBounds", null, true)); verifyChange(result, "ConflictOutOfBounds", true, 1); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/patch/PatchApplier.java b/org.eclipse.jgit/src/org/eclipse/jgit/patch/PatchApplier.java index 84c2ec43dc..1a98d79f99 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/patch/PatchApplier.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/patch/PatchApplier.java @@ -184,10 +184,27 @@ public class PatchApplier { @Nullable HunkHeader hh; - Error(String msg, String oldFileName, @Nullable HunkHeader hh) { + final boolean isGitConflict; + + Error(String msg, String oldFileName, @Nullable HunkHeader hh, + boolean isGitConflict) { this.msg = msg; this.oldFileName = oldFileName; this.hh = hh; + this.isGitConflict = isGitConflict; + } + + /** + * Signals if as part of encountering this error, conflict markers + * were added to the file. + * + * @return {@code true} if conflict markers were added for this + * error. + * + * @since 6.10 + */ + public boolean isGitConflict() { + return isGitConflict; } @Override @@ -213,12 +230,14 @@ public class PatchApplier { Error error = (Error) o; return Objects.equals(msg, error.msg) && Objects.equals(oldFileName, error.oldFileName) - && Objects.equals(hh, error.hh); + && Objects.equals(hh, error.hh) + && isGitConflict == error.isGitConflict; } @Override public int hashCode() { - return Objects.hash(msg, oldFileName, hh); + return Objects.hash(msg, oldFileName, hh, + Boolean.valueOf(isGitConflict)); } } @@ -257,8 +276,14 @@ public class PatchApplier { return errors; } - private void addError(String msg,String oldFileName, @Nullable HunkHeader hh) { - errors.add(new Error(msg, oldFileName, hh)); + private void addError(String msg, String oldFileName, + @Nullable HunkHeader hh) { + errors.add(new Error(msg, oldFileName, hh, false)); + } + + private void addErrorWithGitConflict(String msg, String oldFileName, + @Nullable HunkHeader hh) { + errors.add(new Error(msg, oldFileName, hh, true)); } } @@ -1020,7 +1045,7 @@ public class PatchApplier { // only works if the pre-image SHA is contained in the repo. // If that was the case, cherry-picking the original commit // should be preferred to apply a patch. - result.addError("cannot apply hunk", fh.getOldPath(), hh); //$NON-NLS-1$ + result.addErrorWithGitConflict("cannot apply hunk", fh.getOldPath(), hh); //$NON-NLS-1$ newLines.add(Math.min(applyAt++, newLines.size()), asBytes("<<<<<<< HEAD")); //$NON-NLS-1$ applyAt += hh.getOldImage().lineCount; -- cgit v1.2.3 From 9b2fe85c0279f4d5ac69f07ddcd48566c3555405 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Mon, 27 May 2024 18:22:29 +0200 Subject: PatchApplier.Result.Error: mark fields final Fields of an Error instance shouldn't be modifiable after its creation. Adapt tests which were setting hh to null to skip asserting it. Change-Id: I0f55c1d5cd529aa510029054e6f05bd2637d1bca --- .../tst/org/eclipse/jgit/patch/PatchApplierTest.java | 14 ++++++-------- .../src/org/eclipse/jgit/patch/PatchApplier.java | 6 +++--- 2 files changed, 9 insertions(+), 11 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/patch/PatchApplierTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/patch/PatchApplierTest.java index eec403a976..5507f8572d 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/patch/PatchApplierTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/patch/PatchApplierTest.java @@ -373,10 +373,9 @@ public class PatchApplierTest { assertEquals(result.getErrors().size(), 1); PatchApplier.Result.Error error = result.getErrors().get(0); - error.hh = null; // We don't assert the hunk header as it is a - // complex object with lots of internal state. - assertEquals(error, new PatchApplier.Result.Error( - "cannot apply hunk", "allowconflict", null, true)); + assertEquals("cannot apply hunk", error.msg); + assertEquals("allowconflict", error.oldFileName); + assertTrue(error.isGitConflict()); verifyChange(result, "allowconflict", true, 1); } @@ -388,10 +387,9 @@ public class PatchApplierTest { assertEquals(result.getErrors().size(), 1); PatchApplier.Result.Error error = result.getErrors().get(0); - error.hh = null; // We don't assert the hunk header as it is a - // complex object with lots of internal state. - assertEquals(error, new PatchApplier.Result.Error( - "cannot apply hunk", "ConflictOutOfBounds", null, true)); + assertEquals("cannot apply hunk", error.msg); + assertEquals("ConflictOutOfBounds", error.oldFileName); + assertTrue(error.isGitConflict()); verifyChange(result, "ConflictOutOfBounds", true, 1); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/patch/PatchApplier.java b/org.eclipse.jgit/src/org/eclipse/jgit/patch/PatchApplier.java index 1a98d79f99..cb6cc6efa7 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/patch/PatchApplier.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/patch/PatchApplier.java @@ -177,12 +177,12 @@ public class PatchApplier { // TODO(ms): rename this class in next major release @SuppressWarnings("JavaLangClash") public static class Error { - String msg; + final String msg; - String oldFileName; + final String oldFileName; @Nullable - HunkHeader hh; + final HunkHeader hh; final boolean isGitConflict; -- cgit v1.2.3 From 1dd6324d4b4d9596813b18a44e315295f559ea12 Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Thu, 30 May 2024 10:56:20 -0700 Subject: RepoProject: read the "upstream" attribute of a project The manifest spec [1] defines the "upstream" attribute: "name of the git ref in which a sha1 can be found", when the revision is a sha1. The parser is ignoring it, but RepoCommand could use it to populate the "ref=" field of pinned submodules. Parse the value and store it in the RepoProject. RepoProject is public API and the current constructors are not telescopic, so we cannot just add a new constructor with an extra argument. Use plain getter/setters.j [1] https://gerrit.googlesource.com/git-repo/+/master/docs/manifest-format.md#Element-project Change-Id: Ia50b85b95bfd3710f9fbda2050be5950dd686941 --- .../eclipse/jgit/gitrepo/ManifestParserTest.java | 39 ++++++++++++++++++++++ .../org/eclipse/jgit/gitrepo/ManifestParser.java | 2 ++ .../src/org/eclipse/jgit/gitrepo/RepoProject.java | 26 +++++++++++++++ 3 files changed, 67 insertions(+) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/gitrepo/ManifestParserTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/gitrepo/ManifestParserTest.java index 20958a812c..76176fe347 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/gitrepo/ManifestParserTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/gitrepo/ManifestParserTest.java @@ -11,6 +11,7 @@ package org.eclipse.jgit.gitrepo; import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -18,7 +19,9 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.net.URI; import java.util.HashSet; +import java.util.Map; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -138,6 +141,42 @@ public class ManifestParserTest { .collect(Collectors.toSet())); } + @Test + public void testPinProjectWithUpstream() throws Exception { + StringBuilder xmlContent = new StringBuilder(); + xmlContent.append("\n") + .append("") + .append("") + .append("") + .append("") + .append("") + .append(""); + + ManifestParser parser = new ManifestParser(null, null, "master", + "https://git.google.com/", null, null); + parser.read(new ByteArrayInputStream( + xmlContent.toString().getBytes(UTF_8))); + + Map repos = parser.getProjects().stream().collect( + Collectors.toMap(RepoProject::getName, Function.identity())); + assertEquals(2, repos.size()); + + RepoProject foo = repos.get("pin-with-upstream"); + assertEquals("pin-with-upstream", foo.getName()); + assertEquals("9b2fe85c0279f4d5ac69f07ddcd48566c3555405", + foo.getRevision()); + assertEquals("branchX", foo.getUpstream()); + + RepoProject bar = repos.get("pin-without-upstream"); + assertEquals("pin-without-upstream", bar.getName()); + assertEquals("76ce6d91a2e07fdfcbfc8df6970c9e98a98e36a0", + bar.getRevision()); + assertNull(bar.getUpstream()); + } + void testNormalize(String in, String want) { URI got = ManifestParser.normalizeEmptyPath(URI.create(in)); if (!got.toString().equals(want)) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/ManifestParser.java b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/ManifestParser.java index 957b3869f2..7402c760c3 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/ManifestParser.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/ManifestParser.java @@ -176,6 +176,8 @@ public class ManifestParser extends DefaultHandler { attributes.getValue("groups")); currentProject .setRecommendShallow(attributes.getValue("clone-depth")); + currentProject + .setUpstream(attributes.getValue("upstream")); break; case "remote": String alias = attributes.getValue("alias"); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoProject.java b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoProject.java index 8deb7386a6..aa1af21d23 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoProject.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoProject.java @@ -38,6 +38,7 @@ public class RepoProject implements Comparable { private final Set groups; private final List copyfiles; private final List linkfiles; + private String upstream; private String recommendShallow; private String url; private String defaultRevision; @@ -389,6 +390,31 @@ public class RepoProject implements Comparable { this.linkfiles.clear(); } + /** + * Return the upstream attribute of the project + * + * @return the upstream value if present, null otherwise. + * + * @since 6.10 + */ + public String getUpstream() { + return this.upstream; + } + + /** + * Set the upstream attribute of the project + * + * Name of the git ref in which a sha1 can be found, when the revision is a + * sha1. + * + * @param upstream value of the attribute in the manifest + * + * @since 6.10 + */ + void setUpstream(String upstream) { + this.upstream = upstream; + } + private String getPathWithSlash() { if (path.endsWith("/")) { //$NON-NLS-1$ return path; -- cgit v1.2.3 From 48465f84014904edddcdd48258c67bc19555d4c3 Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Thu, 30 May 2024 14:04:56 -0700 Subject: RepoCommand: Copy manifest upstream into .gitmodules ref field Project entries in the manifest with a specific sha1 as revision can use the "upstream" field to report the ref pointing to that sha1. This information is very valuable for downstream tools, as they can limit their search for a blob to the relevant ref, but it gets lost in the translation to .gitmodules. Save the value of the upstream field when available/relevant in the ref field of the .gitmodules entry. Change-Id: I14a2395925618d5e6b34be85466e32f5ef8fbf6e --- .../jgit/gitrepo/BareSuperprojectWriterTest.java | 45 +++++++++++ .../org/eclipse/jgit/gitrepo/RepoCommandTest.java | 88 ++++++++++++++++++++++ .../jgit/gitrepo/BareSuperprojectWriter.java | 3 + .../src/org/eclipse/jgit/gitrepo/RepoCommand.java | 1 + 4 files changed, 137 insertions(+) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/gitrepo/BareSuperprojectWriterTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/gitrepo/BareSuperprojectWriterTest.java index c3b93879b2..5065b57840 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/gitrepo/BareSuperprojectWriterTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/gitrepo/BareSuperprojectWriterTest.java @@ -12,6 +12,7 @@ package org.eclipse.jgit.gitrepo; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -22,6 +23,7 @@ import java.util.List; import org.eclipse.jgit.gitrepo.BareSuperprojectWriter.BareWriterConfig; import org.eclipse.jgit.gitrepo.RepoCommand.RemoteReader; import org.eclipse.jgit.junit.RepositoryTestCase; +import org.eclipse.jgit.lib.Config; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectReader; import org.eclipse.jgit.lib.Repository; @@ -67,6 +69,49 @@ public class BareSuperprojectWriterTest extends RepositoryTestCase { } } + @Test + public void write_setGitModulesContents_pinned() throws Exception { + try (Repository bareRepo = createBareRepository()) { + RepoProject pinWithUpstream = new RepoProject("pinWithUpstream", + "path/x", "cbc0fae7e1911d27e1de37d364698dba4411c78b", + "remote", ""); + pinWithUpstream.setUrl("http://example.com/a"); + pinWithUpstream.setUpstream("branchX"); + + RepoProject pinWithoutUpstream = new RepoProject( + "pinWithoutUpstream", "path/y", + "cbc0fae7e1911d27e1de37d364698dba4411c78b", "remote", ""); + pinWithoutUpstream.setUrl("http://example.com/b"); + + RemoteReader mockRemoteReader = mock(RemoteReader.class); + + BareSuperprojectWriter w = new BareSuperprojectWriter(bareRepo, + null, "refs/heads/master", author, mockRemoteReader, + BareWriterConfig.getDefault(), List.of()); + + RevCommit commit = w + .write(Arrays.asList(pinWithUpstream, pinWithoutUpstream)); + + String contents = readContents(bareRepo, commit, ".gitmodules"); + Config cfg = new Config(); + cfg.fromText(contents); + + assertThat(cfg.getString("submodule", "pinWithUpstream", "path"), + is("path/x")); + assertThat(cfg.getString("submodule", "pinWithUpstream", "url"), + is("http://example.com/a")); + assertThat(cfg.getString("submodule", "pinWithUpstream", "ref"), + is("branchX")); + + assertThat(cfg.getString("submodule", "pinWithoutUpstream", "path"), + is("path/y")); + assertThat(cfg.getString("submodule", "pinWithoutUpstream", "url"), + is("http://example.com/b")); + assertThat(cfg.getString("submodule", "pinWithoutUpstream", "ref"), + nullValue()); + } + } + @Test public void write_setExtraContents() throws Exception { try (Repository bareRepo = createBareRepository()) { diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/gitrepo/RepoCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/gitrepo/RepoCommandTest.java index ca6f2e1053..3162e7910b 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/gitrepo/RepoCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/gitrepo/RepoCommandTest.java @@ -1171,6 +1171,94 @@ public class RepoCommandTest extends RepositoryTestCase { } } + @Test + public void testRecordRemoteBranch_pinned() throws Exception { + Repository remoteDb = createBareRepository(); + Repository tempDb = createWorkRepository(); + + StringBuilder xmlContent = new StringBuilder(); + xmlContent.append("\n") + .append("") + .append("") + .append("") + .append("") + .append("") + .append(""); + JGitTestUtil.writeTrashFile(tempDb, "manifest.xml", + xmlContent.toString()); + + RepoCommand command = new RepoCommand(remoteDb); + command.setPath( + tempDb.getWorkTree().getAbsolutePath() + "/manifest.xml") + .setURI(rootUri).setRecordRemoteBranch(true).call(); + // Clone it + File directory = createTempDirectory("testBareRepo"); + try (Repository localDb = Git.cloneRepository().setDirectory(directory) + .setURI(remoteDb.getDirectory().toURI().toString()).call() + .getRepository();) { + // The .gitmodules file should exist + File gitmodules = new File(localDb.getWorkTree(), ".gitmodules"); + assertTrue("The .gitmodules file should exist", + gitmodules.exists()); + FileBasedConfig c = new FileBasedConfig(gitmodules, FS.DETECTED); + c.load(); + assertEquals("Pinned submodule with upstream records the ref", + "branchX", c.getString("submodule", "pin-upstream", "ref")); + assertNull("Pinned submodule without upstream don't have ref", + c.getString("submodule", "pin-noupstream", "ref")); + } + } + + @Test + public void testRecordRemoteBranch_pinned_nameConflict() throws Exception { + Repository remoteDb = createBareRepository(); + Repository tempDb = createWorkRepository(); + + StringBuilder xmlContent = new StringBuilder(); + xmlContent.append("\n") + .append("") + .append("") + .append("") + .append("") + .append("") + .append(""); + JGitTestUtil.writeTrashFile(tempDb, "manifest.xml", + xmlContent.toString()); + + RepoCommand command = new RepoCommand(remoteDb); + command.setPath( + tempDb.getWorkTree().getAbsolutePath() + "/manifest.xml") + .setURI(rootUri).setRecordRemoteBranch(true).call(); + // Clone it + File directory = createTempDirectory("testBareRepo"); + try (Repository localDb = Git.cloneRepository().setDirectory(directory) + .setURI(remoteDb.getDirectory().toURI().toString()).call() + .getRepository();) { + // The .gitmodules file should exist + File gitmodules = new File(localDb.getWorkTree(), ".gitmodules"); + assertTrue("The .gitmodules file should exist", + gitmodules.exists()); + FileBasedConfig c = new FileBasedConfig(gitmodules, FS.DETECTED); + c.load(); + assertEquals("Upstream is preserved in name conflict", "branchX", + c.getString("submodule", "pin-upstream/pin-upstream", + "ref")); + assertEquals("Upstream is preserved in name conflict (other side)", + "branchX", c.getString("submodule", + "pin-upstream/pin-upstream-name-conflict", "ref")); + } + } @Test public void testRecordSubmoduleLabels() throws Exception { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/BareSuperprojectWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/BareSuperprojectWriter.java index 3ce97a4ff7..d191e23399 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/BareSuperprojectWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/BareSuperprojectWriter.java @@ -156,6 +156,9 @@ class BareSuperprojectWriter { ObjectId objectId; if (ObjectId.isId(proj.getRevision())) { objectId = ObjectId.fromString(proj.getRevision()); + if (config.recordRemoteBranch && proj.getUpstream() != null) { + cfg.setString("submodule", name, "ref", proj.getUpstream()); + } } else { objectId = callback.sha1(url, proj.getRevision()); if (objectId == null && !config.ignoreRemoteFailures) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoCommand.java index 95c1c8b22e..3aaef387f9 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoCommand.java @@ -615,6 +615,7 @@ public class RepoCommand extends GitCommand { p.setUrl(proj.getUrl()); p.addCopyFiles(proj.getCopyFiles()); p.addLinkFiles(proj.getLinkFiles()); + p.setUpstream(proj.getUpstream()); ret.add(p); } } -- cgit v1.2.3 From 37a36201a1999690500e5af7f16fa881a484a0ca Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Fri, 31 May 2024 12:12:57 -0700 Subject: CommitGraphWriter: Move path diff calculation to its own class To verify that we have the right paths between commits we are writing the bloom filters, reading them and querying. The path diff calculation is tricky enough for correctness and performance that should be tested on its own. Move the path diff calculation to its own class, so we can test it on its own. This is a noop refactor so we can verify later the steps taken in the walk. Change-Id: Ifbdcb752891c4adb08553802f87287de1155bb7c --- .../storage/commitgraph/CommitGraphWriterTest.java | 24 ++++++++ .../storage/commitgraph/CommitGraphWriter.java | 72 ++++++++++++---------- 2 files changed, 63 insertions(+), 33 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphWriterTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphWriterTest.java index 9f65ee2074..7130d59b95 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphWriterTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphWriterTest.java @@ -10,6 +10,7 @@ package org.eclipse.jgit.internal.storage.commitgraph; +import static java.util.stream.Collectors.toList; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.junit.Assert.assertArrayEquals; @@ -19,8 +20,12 @@ import static org.junit.Assert.assertTrue; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.HashSet; +import java.util.List; +import java.util.Optional; import java.util.Set; import org.eclipse.jgit.dircache.DirCacheEntry; @@ -413,6 +418,25 @@ public class CommitGraphWriterTest extends RepositoryTestCase { "119,69,63,-8,0,")); } + @Test + public void testPathDiffCalculator_skipUnchangedTree() throws Exception { + RevCommit root = tr.commit(tr.tree( + tr.file("d/sd1/f1", tr.blob("f1")), + tr.file("d/sd2/f2", tr.blob("f2")))); + RevCommit tip = tr.commit(tr.tree( + tr.file("d/sd1/f1", tr.blob("f1")), + tr.file("d/sd2/f2", tr.blob("f2B"))), root); + CommitGraphWriter.PathDiffCalculator c = new CommitGraphWriter.PathDiffCalculator(); + + Optional> byteBuffers = c.changedPaths(walk.getObjectReader(), tip); + + assertTrue(byteBuffers.isPresent()); + List asString = byteBuffers.get().stream() + .map(b -> StandardCharsets.UTF_8.decode(b).toString()) + .collect(toList()); + assertThat(asString, containsInAnyOrder("d", "d/sd2", "d/sd2/f2")); + } + RevCommit commit(RevCommit... parents) throws Exception { return tr.commit(parents); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphWriter.java index 0d9815eceb..37a0de84c4 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphWriter.java @@ -71,6 +71,9 @@ public class CommitGraphWriter { private static final int MAX_CHANGED_PATHS = 512; + private static final PathDiffCalculator PATH_DIFF_CALCULATOR + = new PathDiffCalculator(); + private final int hashsz; private final GraphCommits graphCommits; @@ -374,37 +377,6 @@ public class CommitGraphWriter { return generations; } - private static Optional> computeBloomFilterPaths( - ObjectReader or, RevCommit cmit) throws MissingObjectException, - IncorrectObjectTypeException, CorruptObjectException, IOException { - HashSet paths = new HashSet<>(); - try (TreeWalk walk = new TreeWalk(null, or)) { - walk.setRecursive(true); - if (cmit.getParentCount() == 0) { - walk.addTree(new EmptyTreeIterator()); - } else { - walk.addTree(cmit.getParent(0).getTree()); - } - walk.addTree(cmit.getTree()); - while (walk.next()) { - if (walk.idEqual(0, 1)) { - continue; - } - byte[] rawPath = walk.getRawPath(); - paths.add(ByteBuffer.wrap(rawPath)); - for (int i = 0; i < rawPath.length; i++) { - if (rawPath[i] == '/') { - paths.add(ByteBuffer.wrap(rawPath, 0, i)); - } - if (paths.size() > MAX_CHANGED_PATHS) { - return Optional.empty(); - } - } - } - } - return Optional.of(paths); - } - private BloomFilterChunks computeBloomFilterChunks(ProgressMonitor monitor) throws MissingObjectException, IncorrectObjectTypeException, CorruptObjectException, IOException { @@ -435,8 +407,8 @@ public class CommitGraphWriter { filtersReused++; } else { filtersComputed++; - Optional> paths = computeBloomFilterPaths( - graphCommits.getObjectReader(), cmit); + Optional> paths = PATH_DIFF_CALCULATOR + .changedPaths(graphCommits.getObjectReader(), cmit); if (paths.isEmpty()) { cpf = ChangedPathFilter.FULL; } else { @@ -473,6 +445,40 @@ public class CommitGraphWriter { } } + // Visible for testing + static class PathDiffCalculator { + Optional> changedPaths( + ObjectReader or, RevCommit cmit) throws MissingObjectException, + IncorrectObjectTypeException, CorruptObjectException, IOException { + HashSet paths = new HashSet<>(); + try (TreeWalk walk = new TreeWalk(null, or)) { + walk.setRecursive(true); + if (cmit.getParentCount() == 0) { + walk.addTree(new EmptyTreeIterator()); + } else { + walk.addTree(cmit.getParent(0).getTree()); + } + walk.addTree(cmit.getTree()); + while (walk.next()) { + if (walk.idEqual(0, 1)) { + continue; + } + byte[] rawPath = walk.getRawPath(); + paths.add(ByteBuffer.wrap(rawPath)); + for (int i = 0; i < rawPath.length; i++) { + if (rawPath[i] == '/') { + paths.add(ByteBuffer.wrap(rawPath, 0, i)); + } + if (paths.size() > MAX_CHANGED_PATHS) { + return Optional.empty(); + } + } + } + } + return Optional.of(paths); + } + } + private static class ChunkHeader { final int id; -- cgit v1.2.3 From 5e8fcfa853d99e036df9c9dc638370e40804fd38 Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Fri, 31 May 2024 12:20:07 -0700 Subject: CommitGraphWriter: use ANY_DIFF instead of idEquals inside next() Calculating the paths modified in a commit respect its parents is taking undue amount of time in big trees. Use ANY_DIFF filter, instead of #idEquals() inside the #next(). This shorcuts the tree browsing earlier. Change-Id: I318eee3ae817b7b9004d60bdb8d0f1bf19b9962d --- .../internal/storage/commitgraph/CommitGraphWriterTest.java | 2 ++ .../jgit/internal/storage/commitgraph/CommitGraphWriter.java | 11 ++++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphWriterTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphWriterTest.java index 7130d59b95..80a0f0cea5 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphWriterTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphWriterTest.java @@ -435,6 +435,8 @@ public class CommitGraphWriterTest extends RepositoryTestCase { .map(b -> StandardCharsets.UTF_8.decode(b).toString()) .collect(toList()); assertThat(asString, containsInAnyOrder("d", "d/sd2", "d/sd2/f2")); + // We don't walk into d/sd1/f1 + assertEquals(1, c.stepCounter); } RevCommit commit(RevCommit... parents) throws Exception { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphWriter.java index 37a0de84c4..55539e2a66 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/commitgraph/CommitGraphWriter.java @@ -52,6 +52,7 @@ import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.treewalk.EmptyTreeIterator; import org.eclipse.jgit.treewalk.TreeWalk; +import org.eclipse.jgit.treewalk.filter.TreeFilter; import org.eclipse.jgit.util.NB; /** @@ -447,12 +448,18 @@ public class CommitGraphWriter { // Visible for testing static class PathDiffCalculator { + + // Walk steps in the last invocation of changedPaths + int stepCounter; + Optional> changedPaths( ObjectReader or, RevCommit cmit) throws MissingObjectException, IncorrectObjectTypeException, CorruptObjectException, IOException { + stepCounter = 0; HashSet paths = new HashSet<>(); try (TreeWalk walk = new TreeWalk(null, or)) { walk.setRecursive(true); + walk.setFilter(TreeFilter.ANY_DIFF); if (cmit.getParentCount() == 0) { walk.addTree(new EmptyTreeIterator()); } else { @@ -460,9 +467,7 @@ public class CommitGraphWriter { } walk.addTree(cmit.getTree()); while (walk.next()) { - if (walk.idEqual(0, 1)) { - continue; - } + stepCounter += 1; byte[] rawPath = walk.getRawPath(); paths.add(ByteBuffer.wrap(rawPath)); for (int i = 0; i < rawPath.length; i++) { -- cgit v1.2.3 From b41187429359cf9830fd34368d28a454653e187d Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Thu, 6 Jun 2024 12:01:04 -0700 Subject: RepoCommand: Add error to ManifestErrorException RepoCommand wraps errors in the manifest in a ManifestErrorException with a fixed message ("Invalid manifest"). Callers like supermanifest plugin cannot return a meaningful error to the client without digging into the cause chain. Add the actual error message to the ManifestErrorException, so callers can rely on #getMessage() to see what happens. Change-Id: I18be17fb5e4aaaf4f11ebd627580a91fe330eaca --- org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoCommand.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoCommand.java index 3aaef387f9..9979664ceb 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoCommand.java @@ -255,7 +255,7 @@ public class RepoCommand extends GitCommand { @SuppressWarnings("serial") static class ManifestErrorException extends GitAPIException { ManifestErrorException(Throwable cause) { - super(RepoText.get().invalidManifest, cause); + super(RepoText.get().invalidManifest + " " + cause.getMessage(), cause); } } -- cgit v1.2.3 From d3ff5b4e3e52e396d89aa9622aeff8aced61dc77 Mon Sep 17 00:00:00 2001 From: Luca Milanesio Date: Fri, 26 Apr 2024 01:20:25 +0100 Subject: Introduce GPG_SIGNATURE_PREFIX constant Change-Id: If6cabae76d7b38ce26fca534da6fe13973ebbf4f --- org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java | 7 +++++++ org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevTag.java | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java index 60a23dde05..1835dc76a2 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java @@ -493,6 +493,13 @@ public final class Constants { */ public static final String ATTR_BUILTIN_BINARY_MERGER = "binary"; //$NON-NLS-1$ + /** + * Prefix of a GPG signature. + * + * @since 7.0 + */ + public static final String GPG_SIGNATURE_PREFIX = "-----BEGIN PGP SIGNATURE-----"; //$NON-NLS-1$ + /** * Create a new digest function for objects. * diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevTag.java b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevTag.java index 75dbd57740..5b50c2afd7 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevTag.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevTag.java @@ -13,6 +13,7 @@ package org.eclipse.jgit.revwalk; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.eclipse.jgit.lib.Constants.GPG_SIGNATURE_PREFIX; import java.io.IOException; import java.nio.charset.Charset; @@ -39,7 +40,7 @@ import org.eclipse.jgit.util.StringUtils; public class RevTag extends RevObject { private static final byte[] hSignature = Constants - .encodeASCII("-----BEGIN PGP SIGNATURE-----"); //$NON-NLS-1$ + .encodeASCII(GPG_SIGNATURE_PREFIX); /** * Parse an annotated tag from its canonical format. -- cgit v1.2.3 From e240b380f4fa98461e61adfcb8b9591c46bee4e3 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Tue, 4 Jun 2024 17:54:52 +0200 Subject: RepoProject: Fix @since tag of new methods The change Ia50b85b95bfd3710f9fbda2050be5950dd686941 didn't reach the 6.10 release hence update the API version to 7.0. Change-Id: If25121797d2955e1e741eec465f69a482af353d1 --- org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoProject.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoProject.java b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoProject.java index aa1af21d23..b96faab452 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoProject.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoProject.java @@ -395,7 +395,7 @@ public class RepoProject implements Comparable { * * @return the upstream value if present, null otherwise. * - * @since 6.10 + * @since 7.0 */ public String getUpstream() { return this.upstream; @@ -407,9 +407,10 @@ public class RepoProject implements Comparable { * Name of the git ref in which a sha1 can be found, when the revision is a * sha1. * - * @param upstream value of the attribute in the manifest + * @param upstream + * value of the attribute in the manifest * - * @since 6.10 + * @since 7.0 */ void setUpstream(String upstream) { this.upstream = upstream; -- cgit v1.2.3 From e57c52dfb97a70975f05b8bc1bcaf424f46ee7c0 Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Tue, 9 Apr 2024 13:25:43 -0700 Subject: DfsPackFile: Do not set primery index local ref from cache callback DfsPackFile assumes the indices are in pack streams, but we would like to consider other formats and storage. Currently, the local ref in the DfsPackFile to the index is set in the cache loading callback, which prevents abstracting the loading. Reorganize the code so: the loadPackIndex function just parses the bytes returning a reference and the caller sets the loaded index in the local ref and DfsBlockCache. We will follow this pattern with other indices in follow-up changes. Note that although DfsPackFile is used only in one thread, the loading in DfsBlockCache can happen from multiple threads concurrently and we want to keep only one ref around. Change-Id: Ie99de6c80784225484c0f99c51caa44c6a372c45 --- .../jgit/internal/storage/dfs/DfsPackFile.java | 43 ++++++++++++++-------- 1 file changed, 27 insertions(+), 16 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java index 5cc2a57aba..3e4d4d300c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java @@ -29,6 +29,7 @@ import java.nio.channels.Channels; import java.text.MessageFormat; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import java.util.zip.CRC32; import java.util.zip.DataFormatException; import java.util.zip.Inflater; @@ -196,18 +197,24 @@ public final class DfsPackFile extends BlockBasedFile { .dispatch(new BeforeDfsPackIndexLoadedEvent(this)); try { DfsStreamKey idxKey = desc.getStreamKey(INDEX); - AtomicBoolean cacheHit = new AtomicBoolean(true); - DfsBlockCache.Ref idxref = cache.getOrLoadRef(idxKey, + // Keep the value parsed in the loader, in case the Ref<> is + // nullified in ClockBlockCacheTable#reserveSpace + // before we read its value. + AtomicReference loadedRef = new AtomicReference<>(null); + DfsBlockCache.Ref cachedRef = cache.getOrLoadRef(idxKey, REF_POSITION, () -> { - cacheHit.set(false); - return loadPackIndex(ctx, idxKey); + RefWithSize idx = loadPackIndex(ctx); + loadedRef.set(idx.ref); + return new DfsBlockCache.Ref<>(idxKey, REF_POSITION, + idx.size, idx.ref); }); - if (cacheHit.get()) { + if (loadedRef.get() == null) { ctx.stats.idxCacheHit++; } - PackIndex idx = idxref.get(); - if (index == null && idx != null) { - index = idx; + index = cachedRef.get() != null ? cachedRef.get() : loadedRef.get(); + if (index == null) { + throw new IOException( + "Couldn't get a reference to the primary index"); //$NON-NLS-1$ } ctx.emitIndexLoad(desc, INDEX, index); return index; @@ -1210,20 +1217,15 @@ public final class DfsPackFile extends BlockBasedFile { } } - private DfsBlockCache.Ref loadPackIndex( - DfsReader ctx, DfsStreamKey idxKey) throws IOException { + private RefWithSize loadPackIndex(DfsReader ctx) + throws IOException { try { ctx.stats.readIdx++; long start = System.nanoTime(); try (ReadableChannel rc = ctx.db.openFile(desc, INDEX)) { PackIndex idx = PackIndex.read(alignTo8kBlocks(rc)); ctx.stats.readIdxBytes += rc.position(); - index = idx; - return new DfsBlockCache.Ref<>( - idxKey, - REF_POSITION, - idx.getObjectCount() * REC_SIZE, - idx); + return new RefWithSize<>(idx, idx.getObjectCount() * REC_SIZE); } finally { ctx.stats.readIdxMicros += elapsedMicros(start); } @@ -1449,4 +1451,13 @@ public final class DfsPackFile extends BlockBasedFile { } } } + + private static final class RefWithSize { + final V ref; + final long size; + RefWithSize(V ref, long size) { + this.ref = ref; + this.size = size; + } + } } -- cgit v1.2.3 From e3e0a1ea35a27e50e0280715a417be7d69fa3345 Mon Sep 17 00:00:00 2001 From: Kaushik Lingarkar Date: Thu, 27 Jun 2024 18:05:30 -0700 Subject: Make RepoProject#setUpstream public Applications using JGit such as Gerrit plugins may have their own manifest parsers. They can start using RepoProject to some extent with this change. Eventually, they can be migrated to use the ManifestParser in JGit, however until then, this change can help make the migration incremental. Change-Id: I6a32d4f4622c3842eedf7873cdfed2f1ca998f6f --- org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoProject.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoProject.java b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoProject.java index b96faab452..09ce28386c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoProject.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoProject.java @@ -412,7 +412,7 @@ public class RepoProject implements Comparable { * * @since 7.0 */ - void setUpstream(String upstream) { + public void setUpstream(String upstream) { this.upstream = upstream; } -- cgit v1.2.3 From 47fd412affd8d7578606ae9b3015a911b71b13ed Mon Sep 17 00:00:00 2001 From: Kaushik Lingarkar Date: Fri, 28 Jun 2024 10:02:05 -0700 Subject: RepoProject: read the 'dest-branch' attribute of a project The manifest spec [1] defines a "dest-branch" attribute. Parse its value and store it in the RepoProject. Also, create a getter/setter for dest-branch. [1] https://gerrit.googlesource.com/git-repo/+/master/docs/manifest-format.md#Element-project Change-Id: I8ad83b0fec59d2b0967864e4de4fefde4ab971ff --- .../eclipse/jgit/gitrepo/ManifestParserTest.java | 30 ++++++++++++++++++++++ .../org/eclipse/jgit/gitrepo/ManifestParser.java | 2 ++ .../src/org/eclipse/jgit/gitrepo/RepoProject.java | 26 +++++++++++++++++++ 3 files changed, 58 insertions(+) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/gitrepo/ManifestParserTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/gitrepo/ManifestParserTest.java index 76176fe347..fca27d32aa 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/gitrepo/ManifestParserTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/gitrepo/ManifestParserTest.java @@ -177,6 +177,36 @@ public class ManifestParserTest { assertNull(bar.getUpstream()); } + @Test + public void testWithDestBranch() throws Exception { + StringBuilder xmlContent = new StringBuilder(); + xmlContent.append("\n") + .append("") + .append("") + .append("") + .append("") + .append("") + .append(""); + + ManifestParser parser = new ManifestParser(null, null, "master", + "https://git.google.com/", null, null); + parser.read(new ByteArrayInputStream( + xmlContent.toString().getBytes(UTF_8))); + + Map repos = parser.getProjects().stream().collect( + Collectors.toMap(RepoProject::getName, Function.identity())); + assertEquals(2, repos.size()); + + RepoProject foo = repos.get("foo"); + assertEquals("foo", foo.getName()); + assertEquals("branchX", foo.getDestBranch()); + + RepoProject bar = repos.get("bar"); + assertEquals("bar", bar.getName()); + assertNull(bar.getDestBranch()); + } + void testNormalize(String in, String want) { URI got = ManifestParser.normalizeEmptyPath(URI.create(in)); if (!got.toString().equals(want)) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/ManifestParser.java b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/ManifestParser.java index 7402c760c3..b033177e05 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/ManifestParser.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/ManifestParser.java @@ -178,6 +178,8 @@ public class ManifestParser extends DefaultHandler { .setRecommendShallow(attributes.getValue("clone-depth")); currentProject .setUpstream(attributes.getValue("upstream")); + currentProject + .setDestBranch(attributes.getValue("dest-branch")); break; case "remote": String alias = attributes.getValue("alias"); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoProject.java b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoProject.java index 09ce28386c..d8fec78493 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoProject.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoProject.java @@ -39,6 +39,7 @@ public class RepoProject implements Comparable { private final List copyfiles; private final List linkfiles; private String upstream; + private String destBranch; private String recommendShallow; private String url; private String defaultRevision; @@ -401,6 +402,17 @@ public class RepoProject implements Comparable { return this.upstream; } + /** + * Return the dest-branch attribute of the project + * + * @return the dest-branch value if present, null otherwise. + * + * @since 7.0 + */ + public String getDestBranch() { + return this.destBranch; + } + /** * Set the upstream attribute of the project * @@ -416,6 +428,20 @@ public class RepoProject implements Comparable { this.upstream = upstream; } + /** + * Set the dest-branch attribute of the project + * + * Name of a Git branch. + * + * @param destBranch + * value of the attribute in the manifest + * + * @since 7.0 + */ + public void setDestBranch(String destBranch) { + this.destBranch = destBranch; + } + private String getPathWithSlash() { if (path.endsWith("/")) { //$NON-NLS-1$ return path; -- cgit v1.2.3 From e5534111577b0f9c9b50140461adea31f789c06c Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Thu, 30 May 2024 10:56:20 -0700 Subject: RepoProject: read the "upstream" attribute of a project The manifest spec [1] defines the "upstream" attribute: "name of the git ref in which a sha1 can be found", when the revision is a sha1. The parser is ignoring it, but RepoCommand could use it to populate the "ref=" field of pinned submodules. Parse the value and store it in the RepoProject. RepoProject is public API and the current constructors are not telescopic, so we cannot just add a new constructor with an extra argument. Use plain getter/setters.j [1] https://gerrit.googlesource.com/git-repo/+/master/docs/manifest-format.md#Element-project Change-Id: Ia50b85b95bfd3710f9fbda2050be5950dd686941 (cherry picked from commit 1dd6324d4b4d9596813b18a44e315295f559ea12) --- .../eclipse/jgit/gitrepo/ManifestParserTest.java | 39 ++++++++++++++++++++++ .../org/eclipse/jgit/gitrepo/ManifestParser.java | 2 ++ .../src/org/eclipse/jgit/gitrepo/RepoProject.java | 26 +++++++++++++++ 3 files changed, 67 insertions(+) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/gitrepo/ManifestParserTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/gitrepo/ManifestParserTest.java index 20958a812c..76176fe347 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/gitrepo/ManifestParserTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/gitrepo/ManifestParserTest.java @@ -11,6 +11,7 @@ package org.eclipse.jgit.gitrepo; import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -18,7 +19,9 @@ import java.io.ByteArrayInputStream; import java.io.IOException; import java.net.URI; import java.util.HashSet; +import java.util.Map; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -138,6 +141,42 @@ public class ManifestParserTest { .collect(Collectors.toSet())); } + @Test + public void testPinProjectWithUpstream() throws Exception { + StringBuilder xmlContent = new StringBuilder(); + xmlContent.append("\n") + .append("") + .append("") + .append("") + .append("") + .append("") + .append(""); + + ManifestParser parser = new ManifestParser(null, null, "master", + "https://git.google.com/", null, null); + parser.read(new ByteArrayInputStream( + xmlContent.toString().getBytes(UTF_8))); + + Map repos = parser.getProjects().stream().collect( + Collectors.toMap(RepoProject::getName, Function.identity())); + assertEquals(2, repos.size()); + + RepoProject foo = repos.get("pin-with-upstream"); + assertEquals("pin-with-upstream", foo.getName()); + assertEquals("9b2fe85c0279f4d5ac69f07ddcd48566c3555405", + foo.getRevision()); + assertEquals("branchX", foo.getUpstream()); + + RepoProject bar = repos.get("pin-without-upstream"); + assertEquals("pin-without-upstream", bar.getName()); + assertEquals("76ce6d91a2e07fdfcbfc8df6970c9e98a98e36a0", + bar.getRevision()); + assertNull(bar.getUpstream()); + } + void testNormalize(String in, String want) { URI got = ManifestParser.normalizeEmptyPath(URI.create(in)); if (!got.toString().equals(want)) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/ManifestParser.java b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/ManifestParser.java index 957b3869f2..7402c760c3 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/ManifestParser.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/ManifestParser.java @@ -176,6 +176,8 @@ public class ManifestParser extends DefaultHandler { attributes.getValue("groups")); currentProject .setRecommendShallow(attributes.getValue("clone-depth")); + currentProject + .setUpstream(attributes.getValue("upstream")); break; case "remote": String alias = attributes.getValue("alias"); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoProject.java b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoProject.java index 8deb7386a6..aa1af21d23 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoProject.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoProject.java @@ -38,6 +38,7 @@ public class RepoProject implements Comparable { private final Set groups; private final List copyfiles; private final List linkfiles; + private String upstream; private String recommendShallow; private String url; private String defaultRevision; @@ -389,6 +390,31 @@ public class RepoProject implements Comparable { this.linkfiles.clear(); } + /** + * Return the upstream attribute of the project + * + * @return the upstream value if present, null otherwise. + * + * @since 6.10 + */ + public String getUpstream() { + return this.upstream; + } + + /** + * Set the upstream attribute of the project + * + * Name of the git ref in which a sha1 can be found, when the revision is a + * sha1. + * + * @param upstream value of the attribute in the manifest + * + * @since 6.10 + */ + void setUpstream(String upstream) { + this.upstream = upstream; + } + private String getPathWithSlash() { if (path.endsWith("/")) { //$NON-NLS-1$ return path; -- cgit v1.2.3 From 784f0baecbbb68f1e136efe862087634d10826eb Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Thu, 30 May 2024 14:04:56 -0700 Subject: RepoCommand: Copy manifest upstream into .gitmodules ref field Project entries in the manifest with a specific sha1 as revision can use the "upstream" field to report the ref pointing to that sha1. This information is very valuable for downstream tools, as they can limit their search for a blob to the relevant ref, but it gets lost in the translation to .gitmodules. Save the value of the upstream field when available/relevant in the ref field of the .gitmodules entry. Change-Id: I14a2395925618d5e6b34be85466e32f5ef8fbf6e (cherry picked from commit 48465f84014904edddcdd48258c67bc19555d4c3) --- .../jgit/gitrepo/BareSuperprojectWriterTest.java | 45 +++++++++++ .../org/eclipse/jgit/gitrepo/RepoCommandTest.java | 88 ++++++++++++++++++++++ .../jgit/gitrepo/BareSuperprojectWriter.java | 3 + .../src/org/eclipse/jgit/gitrepo/RepoCommand.java | 1 + 4 files changed, 137 insertions(+) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/gitrepo/BareSuperprojectWriterTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/gitrepo/BareSuperprojectWriterTest.java index c3b93879b2..5065b57840 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/gitrepo/BareSuperprojectWriterTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/gitrepo/BareSuperprojectWriterTest.java @@ -12,6 +12,7 @@ package org.eclipse.jgit.gitrepo; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -22,6 +23,7 @@ import java.util.List; import org.eclipse.jgit.gitrepo.BareSuperprojectWriter.BareWriterConfig; import org.eclipse.jgit.gitrepo.RepoCommand.RemoteReader; import org.eclipse.jgit.junit.RepositoryTestCase; +import org.eclipse.jgit.lib.Config; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectReader; import org.eclipse.jgit.lib.Repository; @@ -67,6 +69,49 @@ public class BareSuperprojectWriterTest extends RepositoryTestCase { } } + @Test + public void write_setGitModulesContents_pinned() throws Exception { + try (Repository bareRepo = createBareRepository()) { + RepoProject pinWithUpstream = new RepoProject("pinWithUpstream", + "path/x", "cbc0fae7e1911d27e1de37d364698dba4411c78b", + "remote", ""); + pinWithUpstream.setUrl("http://example.com/a"); + pinWithUpstream.setUpstream("branchX"); + + RepoProject pinWithoutUpstream = new RepoProject( + "pinWithoutUpstream", "path/y", + "cbc0fae7e1911d27e1de37d364698dba4411c78b", "remote", ""); + pinWithoutUpstream.setUrl("http://example.com/b"); + + RemoteReader mockRemoteReader = mock(RemoteReader.class); + + BareSuperprojectWriter w = new BareSuperprojectWriter(bareRepo, + null, "refs/heads/master", author, mockRemoteReader, + BareWriterConfig.getDefault(), List.of()); + + RevCommit commit = w + .write(Arrays.asList(pinWithUpstream, pinWithoutUpstream)); + + String contents = readContents(bareRepo, commit, ".gitmodules"); + Config cfg = new Config(); + cfg.fromText(contents); + + assertThat(cfg.getString("submodule", "pinWithUpstream", "path"), + is("path/x")); + assertThat(cfg.getString("submodule", "pinWithUpstream", "url"), + is("http://example.com/a")); + assertThat(cfg.getString("submodule", "pinWithUpstream", "ref"), + is("branchX")); + + assertThat(cfg.getString("submodule", "pinWithoutUpstream", "path"), + is("path/y")); + assertThat(cfg.getString("submodule", "pinWithoutUpstream", "url"), + is("http://example.com/b")); + assertThat(cfg.getString("submodule", "pinWithoutUpstream", "ref"), + nullValue()); + } + } + @Test public void write_setExtraContents() throws Exception { try (Repository bareRepo = createBareRepository()) { diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/gitrepo/RepoCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/gitrepo/RepoCommandTest.java index ca6f2e1053..3162e7910b 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/gitrepo/RepoCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/gitrepo/RepoCommandTest.java @@ -1171,6 +1171,94 @@ public class RepoCommandTest extends RepositoryTestCase { } } + @Test + public void testRecordRemoteBranch_pinned() throws Exception { + Repository remoteDb = createBareRepository(); + Repository tempDb = createWorkRepository(); + + StringBuilder xmlContent = new StringBuilder(); + xmlContent.append("\n") + .append("") + .append("") + .append("") + .append("") + .append("") + .append(""); + JGitTestUtil.writeTrashFile(tempDb, "manifest.xml", + xmlContent.toString()); + + RepoCommand command = new RepoCommand(remoteDb); + command.setPath( + tempDb.getWorkTree().getAbsolutePath() + "/manifest.xml") + .setURI(rootUri).setRecordRemoteBranch(true).call(); + // Clone it + File directory = createTempDirectory("testBareRepo"); + try (Repository localDb = Git.cloneRepository().setDirectory(directory) + .setURI(remoteDb.getDirectory().toURI().toString()).call() + .getRepository();) { + // The .gitmodules file should exist + File gitmodules = new File(localDb.getWorkTree(), ".gitmodules"); + assertTrue("The .gitmodules file should exist", + gitmodules.exists()); + FileBasedConfig c = new FileBasedConfig(gitmodules, FS.DETECTED); + c.load(); + assertEquals("Pinned submodule with upstream records the ref", + "branchX", c.getString("submodule", "pin-upstream", "ref")); + assertNull("Pinned submodule without upstream don't have ref", + c.getString("submodule", "pin-noupstream", "ref")); + } + } + + @Test + public void testRecordRemoteBranch_pinned_nameConflict() throws Exception { + Repository remoteDb = createBareRepository(); + Repository tempDb = createWorkRepository(); + + StringBuilder xmlContent = new StringBuilder(); + xmlContent.append("\n") + .append("") + .append("") + .append("") + .append("") + .append("") + .append(""); + JGitTestUtil.writeTrashFile(tempDb, "manifest.xml", + xmlContent.toString()); + + RepoCommand command = new RepoCommand(remoteDb); + command.setPath( + tempDb.getWorkTree().getAbsolutePath() + "/manifest.xml") + .setURI(rootUri).setRecordRemoteBranch(true).call(); + // Clone it + File directory = createTempDirectory("testBareRepo"); + try (Repository localDb = Git.cloneRepository().setDirectory(directory) + .setURI(remoteDb.getDirectory().toURI().toString()).call() + .getRepository();) { + // The .gitmodules file should exist + File gitmodules = new File(localDb.getWorkTree(), ".gitmodules"); + assertTrue("The .gitmodules file should exist", + gitmodules.exists()); + FileBasedConfig c = new FileBasedConfig(gitmodules, FS.DETECTED); + c.load(); + assertEquals("Upstream is preserved in name conflict", "branchX", + c.getString("submodule", "pin-upstream/pin-upstream", + "ref")); + assertEquals("Upstream is preserved in name conflict (other side)", + "branchX", c.getString("submodule", + "pin-upstream/pin-upstream-name-conflict", "ref")); + } + } @Test public void testRecordSubmoduleLabels() throws Exception { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/BareSuperprojectWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/BareSuperprojectWriter.java index 3ce97a4ff7..d191e23399 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/BareSuperprojectWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/BareSuperprojectWriter.java @@ -156,6 +156,9 @@ class BareSuperprojectWriter { ObjectId objectId; if (ObjectId.isId(proj.getRevision())) { objectId = ObjectId.fromString(proj.getRevision()); + if (config.recordRemoteBranch && proj.getUpstream() != null) { + cfg.setString("submodule", name, "ref", proj.getUpstream()); + } } else { objectId = callback.sha1(url, proj.getRevision()); if (objectId == null && !config.ignoreRemoteFailures) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoCommand.java index 95c1c8b22e..3aaef387f9 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoCommand.java @@ -615,6 +615,7 @@ public class RepoCommand extends GitCommand { p.setUrl(proj.getUrl()); p.addCopyFiles(proj.getCopyFiles()); p.addLinkFiles(proj.getLinkFiles()); + p.setUpstream(proj.getUpstream()); ret.add(p); } } -- cgit v1.2.3 From 4bc6eb8a70281b30e40ebe2fb12927ce2d2640df Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Thu, 6 Jun 2024 12:01:04 -0700 Subject: RepoCommand: Add error to ManifestErrorException RepoCommand wraps errors in the manifest in a ManifestErrorException with a fixed message ("Invalid manifest"). Callers like supermanifest plugin cannot return a meaningful error to the client without digging into the cause chain. Add the actual error message to the ManifestErrorException, so callers can rely on #getMessage() to see what happens. Change-Id: I18be17fb5e4aaaf4f11ebd627580a91fe330eaca (cherry picked from commit b41187429359cf9830fd34368d28a454653e187d) --- org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoCommand.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoCommand.java index 3aaef387f9..9979664ceb 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoCommand.java @@ -255,7 +255,7 @@ public class RepoCommand extends GitCommand { @SuppressWarnings("serial") static class ManifestErrorException extends GitAPIException { ManifestErrorException(Throwable cause) { - super(RepoText.get().invalidManifest, cause); + super(RepoText.get().invalidManifest + " " + cause.getMessage(), cause); } } -- cgit v1.2.3 From 3d8420ecd83745355a5f4f08541d019032f8bac8 Mon Sep 17 00:00:00 2001 From: Kaushik Lingarkar Date: Thu, 27 Jun 2024 18:05:30 -0700 Subject: Make RepoProject#setUpstream public Applications using JGit such as Gerrit plugins may have their own manifest parsers. They can start using RepoProject to some extent with this change. Eventually, they can be migrated to use the ManifestParser in JGit, however until then, this change can help make the migration incremental. Change-Id: I6a32d4f4622c3842eedf7873cdfed2f1ca998f6f (cherry picked from commit e3e0a1ea35a27e50e0280715a417be7d69fa3345) --- org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoProject.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoProject.java b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoProject.java index aa1af21d23..3ed0bef317 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoProject.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoProject.java @@ -411,7 +411,7 @@ public class RepoProject implements Comparable { * * @since 6.10 */ - void setUpstream(String upstream) { + public void setUpstream(String upstream) { this.upstream = upstream; } -- cgit v1.2.3 From cfdfb01f4c7de1d61a4913deac1d02837b53df82 Mon Sep 17 00:00:00 2001 From: Kaushik Lingarkar Date: Fri, 28 Jun 2024 10:02:05 -0700 Subject: RepoProject: read the 'dest-branch' attribute of a project The manifest spec [1] defines a "dest-branch" attribute. Parse its value and store it in the RepoProject. Also, create a getter/setter for dest-branch. [1] https://gerrit.googlesource.com/git-repo/+/master/docs/manifest-format.md#Element-project Change-Id: I8ad83b0fec59d2b0967864e4de4fefde4ab971ff (cherry picked from commit 47fd412affd8d7578606ae9b3015a911b71b13ed) --- .../eclipse/jgit/gitrepo/ManifestParserTest.java | 30 ++++++++++++++++++++++ org.eclipse.jgit/.settings/.api_filters | 19 ++++++++++++++ .../org/eclipse/jgit/gitrepo/ManifestParser.java | 2 ++ .../src/org/eclipse/jgit/gitrepo/RepoProject.java | 26 +++++++++++++++++++ 4 files changed, 77 insertions(+) create mode 100644 org.eclipse.jgit/.settings/.api_filters (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/gitrepo/ManifestParserTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/gitrepo/ManifestParserTest.java index 76176fe347..fca27d32aa 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/gitrepo/ManifestParserTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/gitrepo/ManifestParserTest.java @@ -177,6 +177,36 @@ public class ManifestParserTest { assertNull(bar.getUpstream()); } + @Test + public void testWithDestBranch() throws Exception { + StringBuilder xmlContent = new StringBuilder(); + xmlContent.append("\n") + .append("") + .append("") + .append("") + .append("") + .append("") + .append(""); + + ManifestParser parser = new ManifestParser(null, null, "master", + "https://git.google.com/", null, null); + parser.read(new ByteArrayInputStream( + xmlContent.toString().getBytes(UTF_8))); + + Map repos = parser.getProjects().stream().collect( + Collectors.toMap(RepoProject::getName, Function.identity())); + assertEquals(2, repos.size()); + + RepoProject foo = repos.get("foo"); + assertEquals("foo", foo.getName()); + assertEquals("branchX", foo.getDestBranch()); + + RepoProject bar = repos.get("bar"); + assertEquals("bar", bar.getName()); + assertNull(bar.getDestBranch()); + } + void testNormalize(String in, String want) { URI got = ManifestParser.normalizeEmptyPath(URI.create(in)); if (!got.toString().equals(want)) { diff --git a/org.eclipse.jgit/.settings/.api_filters b/org.eclipse.jgit/.settings/.api_filters new file mode 100644 index 0000000000..50a04d20f0 --- /dev/null +++ b/org.eclipse.jgit/.settings/.api_filters @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/ManifestParser.java b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/ManifestParser.java index 7402c760c3..b033177e05 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/ManifestParser.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/ManifestParser.java @@ -178,6 +178,8 @@ public class ManifestParser extends DefaultHandler { .setRecommendShallow(attributes.getValue("clone-depth")); currentProject .setUpstream(attributes.getValue("upstream")); + currentProject + .setDestBranch(attributes.getValue("dest-branch")); break; case "remote": String alias = attributes.getValue("alias"); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoProject.java b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoProject.java index 3ed0bef317..b7a9ac5b73 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoProject.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoProject.java @@ -39,6 +39,7 @@ public class RepoProject implements Comparable { private final List copyfiles; private final List linkfiles; private String upstream; + private String destBranch; private String recommendShallow; private String url; private String defaultRevision; @@ -401,6 +402,17 @@ public class RepoProject implements Comparable { return this.upstream; } + /** + * Return the dest-branch attribute of the project + * + * @return the dest-branch value if present, null otherwise. + * + * @since 7.0 + */ + public String getDestBranch() { + return this.destBranch; + } + /** * Set the upstream attribute of the project * @@ -415,6 +427,20 @@ public class RepoProject implements Comparable { this.upstream = upstream; } + /** + * Set the dest-branch attribute of the project + * + * Name of a Git branch. + * + * @param destBranch + * value of the attribute in the manifest + * + * @since 7.0 + */ + public void setDestBranch(String destBranch) { + this.destBranch = destBranch; + } + private String getPathWithSlash() { if (path.endsWith("/")) { //$NON-NLS-1$ return path; -- cgit v1.2.3 From 2ea9ccf3e8ccc6603fa13336903b453bb5c140d4 Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Mon, 1 Jul 2024 12:24:36 -0700 Subject: DfsPackFile: Enable/disable object size index via DfsReaderOptions DfsPackFile always uses the object size index if available. That is the desired final state, but for a safe rollout, we should be able to disable using the object size index. Add an option (dfs.useObjectSizeIndex) to enable/disable the usage of the object size index. False by default. This changes the default from true to false. It only makes a different for the DFS stack when writing of the index was explicitely enabled. This is an optimization, so it shouldn't cause any regression. Operators can restore previous behaviour setting "dfs.useObjectSizeIndex" to true. Change-Id: I44bf5a57e3942a4ecfe66d58bfa9175e99f96fcc --- .../storage/dfs/DfsGarbageCollectorTest.java | 3 +++ .../jgit/internal/storage/dfs/DfsInserterTest.java | 1 + .../jgit/internal/storage/dfs/DfsPackFileTest.java | 1 + .../internal/storage/dfs/DfsPackParserTest.java | 1 + .../jgit/internal/storage/dfs/DfsReaderTest.java | 2 ++ .../jgit/internal/storage/dfs/DfsPackFile.java | 1 + .../internal/storage/dfs/DfsReaderOptions.java | 29 ++++++++++++++++++++++ .../src/org/eclipse/jgit/lib/ConfigConstants.java | 7 ++++++ 8 files changed, 45 insertions(+) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollectorTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollectorTest.java index e193de9764..2be11d32ea 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollectorTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollectorTest.java @@ -18,6 +18,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.Collections; import java.util.concurrent.TimeUnit; + import org.eclipse.jgit.internal.storage.commitgraph.CommitGraph; import org.eclipse.jgit.internal.storage.commitgraph.CommitGraphWriter; import org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource; @@ -1171,6 +1172,7 @@ public class DfsGarbageCollectorTest { gcWithObjectSizeIndex(10); + odb.getReaderOptions().setUseObjectSizeIndex(true); DfsReader reader = odb.newReader(); DfsPackFile gcPack = findFirstBySource(odb.getPacks(), GC); assertTrue(gcPack.hasObjectSizeIndex(reader)); @@ -1191,6 +1193,7 @@ public class DfsGarbageCollectorTest { gcWithObjectSizeIndex(10); + odb.getReaderOptions().setUseObjectSizeIndex(true); DfsReader reader = odb.newReader(); DfsPackFile gcPack = findFirstBySource(odb.getPacks(), GC); assertTrue(gcPack.hasObjectSizeIndex(reader)); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsInserterTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsInserterTest.java index b84a0b00ae..0b558edf2c 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsInserterTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsInserterTest.java @@ -295,6 +295,7 @@ public class DfsInserterTest { public void testObjectSizeIndexOnInsert() throws IOException { db.getConfig().setInt(CONFIG_PACK_SECTION, null, CONFIG_KEY_MIN_BYTES_OBJ_SIZE_INDEX, 0); + db.getObjectDatabase().getReaderOptions().setUseObjectSizeIndex(true); byte[] contents = Constants.encode("foo"); ObjectId fooId; diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackFileTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackFileTest.java index d21e51f276..bc851f8dde 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackFileTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackFileTest.java @@ -126,6 +126,7 @@ public class DfsPackFileTest { setObjectSizeIndexMinBytes(0); ObjectId blobId = setupPack(512, 800); + db.getObjectDatabase().getReaderOptions().setUseObjectSizeIndex(true); DfsReader reader = db.getObjectDatabase().newReader(); DfsPackFile pack = db.getObjectDatabase().getPacks()[0]; assertTrue(pack.hasObjectSizeIndex(reader)); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackParserTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackParserTest.java index 130af27773..c1cd231c66 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackParserTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackParserTest.java @@ -61,6 +61,7 @@ public class DfsPackParserTest { ins.flush(); } + repo.getObjectDatabase().getReaderOptions().setUseObjectSizeIndex(true); DfsReader reader = repo.getObjectDatabase().newReader(); PackList packList = repo.getObjectDatabase().getPackList(); assertEquals(1, packList.packs.length); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsReaderTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsReaderTest.java index 254184ee80..a0c228906e 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsReaderTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsReaderTest.java @@ -37,6 +37,8 @@ public class DfsReaderTest { @Before public void setUp() { db = new InMemoryRepository(new DfsRepositoryDescription("test")); + // These tests assume the object size index is enabled. + db.getObjectDatabase().getReaderOptions().setUseObjectSizeIndex(true); } @Test diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java index 3e4d4d300c..b94a84a41c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java @@ -354,6 +354,7 @@ public final class DfsPackFile extends BlockBasedFile { } if (objectSizeIndexLoadAttempted + || !ctx.getOptions().shouldUseObjectSizeIndex() || !desc.hasFileExt(OBJECT_SIZE_INDEX)) { // Pack doesn't have object size index return null; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReaderOptions.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReaderOptions.java index f2ac461deb..5f5e81977b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReaderOptions.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReaderOptions.java @@ -15,6 +15,7 @@ import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_DFS_SECTION; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_DELTA_BASE_CACHE_LIMIT; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_STREAM_BUFFER; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_STREAM_FILE_THRESHOLD; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_USE_OBJECT_SIZE_INDEX; import org.eclipse.jgit.lib.Config; import org.eclipse.jgit.storage.pack.PackConfig; @@ -36,6 +37,8 @@ public class DfsReaderOptions { private boolean loadRevIndexInParallel; + private boolean useObjectSizeIndex; + /** * Create a default reader configuration. */ @@ -136,6 +139,28 @@ public class DfsReaderOptions { return this; } + /** + * Use the object size index if available. + * + * @return true if the reader should try to use the object size index. if + * false, the reader ignores that index. + */ + public boolean shouldUseObjectSizeIndex() { + return useObjectSizeIndex; + } + + /** + * Set if the reader should try to use the object size index + * + * @param useObjectSizeIndex true to use it, false to ignore the object size index + * + * @return {@code this} + */ + public DfsReaderOptions setUseObjectSizeIndex(boolean useObjectSizeIndex) { + this.useObjectSizeIndex = useObjectSizeIndex; + return this; + } + /** * Update properties by setting fields from the configuration. *

@@ -168,6 +193,10 @@ public class DfsReaderOptions { CONFIG_DFS_SECTION, CONFIG_KEY_STREAM_BUFFER, getStreamPackBufferSize())); + + setUseObjectSizeIndex(rc.getBoolean(CONFIG_CORE_SECTION, + CONFIG_DFS_SECTION, CONFIG_KEY_USE_OBJECT_SIZE_INDEX, + false)); return this; } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java index 0edf3c5ad0..e9fbc65696 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java @@ -1012,4 +1012,11 @@ public final class ConfigConstants { * @since 6.7 */ public static final String CONFIG_KEY_READ_CHANGED_PATHS = "readChangedPaths"; + + /** + * The "useObjectSizeIndex" key + * + * @since 7.0 + */ + public static final String CONFIG_KEY_USE_OBJECT_SIZE_INDEX = "useObjectSizeIndex"; } -- cgit v1.2.3 From b343442bdb8333ec31d4a17a82bb6e02aee3fbe6 Mon Sep 17 00:00:00 2001 From: Laura Hamelin Date: Fri, 7 Jun 2024 16:12:18 -0700 Subject: DfsBlockCacheConfig: support configurations for dfs cache tables per extensions Parse configurations for tables containing a set of extensions, defined in [core "dfs.*"] sections. Parse configurations for cache tables according to configurations defined in [core "dfs.*"] git config sections for sets of extensions. The current [core "dfs"] is the default to any extension not listed in any other table. Configuration falls back to the defaults defined in the DfsBlockCacheConfig.java file when not set on each cache table configuration. Sample format for individual cache tables: In this example: 1. PACK types would go to the "default" table 2. INDEX and BITMAP_INDEX types would go to the "multipleExtensionCache" table 3. REFTABLE types would go to the "reftableCache" table [core "dfs"] // Configuration for the "default" cache table. blockSize = 512 blockLimit = 100 concurrencyLevel = 5 (...) [core "dfs.multipleExtensionCache"] packExtensions = "INDEX BITMAP_INDEX" blockSize = 512 blockLimit = 100 concurrencyLevel = 5 (...) [core "dfs.reftableCache"] packExtensions = "REFTABLE" blockSize = 512 blockLimit = 100 concurrencyLevel = 5 (...) Change-Id: I0e534e6d78b684832e3d3d269cee2590aa0f1911 --- .../storage/dfs/DfsBlockCacheConfigTest.java | 159 ++++++++++++++++ .../org/eclipse/jgit/internal/JGitText.properties | 4 + .../src/org/eclipse/jgit/internal/JGitText.java | 4 + .../internal/storage/dfs/DfsBlockCacheConfig.java | 199 +++++++++++++++++---- .../src/org/eclipse/jgit/lib/ConfigConstants.java | 10 ++ 5 files changed, 342 insertions(+), 34 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfigTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfigTest.java index 2df0ba1b05..6ca0ff6a16 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfigTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfigTest.java @@ -38,11 +38,27 @@ package org.eclipse.jgit.internal.storage.dfs; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_CORE_SECTION; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_DFS_CACHE_PREFIX; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_DFS_SECTION; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_BLOCK_LIMIT; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_BLOCK_SIZE; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_CONCURRENCY_LEVEL; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_PACK_EXTENSIONS; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_STREAM_RATIO; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertThrows; +import java.util.List; +import java.util.stream.Collectors; + import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.internal.storage.dfs.DfsBlockCacheConfig.DfsBlockCachePackExtConfig; +import org.eclipse.jgit.internal.storage.pack.PackExt; +import org.eclipse.jgit.lib.Config; import org.junit.Test; public class DfsBlockCacheConfigTest { @@ -80,4 +96,147 @@ public class DfsBlockCacheConfigTest { assertThat(config.getBlockSize(), is(65536)); } + + @Test + public void fromConfigs() { + Config config = new Config(); + config.setLong(CONFIG_CORE_SECTION, CONFIG_DFS_SECTION, + CONFIG_KEY_BLOCK_LIMIT, 50 * 1024); + config.setInt(CONFIG_CORE_SECTION, CONFIG_DFS_SECTION, + CONFIG_KEY_BLOCK_SIZE, 1024); + config.setInt(CONFIG_CORE_SECTION, CONFIG_DFS_SECTION, + CONFIG_KEY_CONCURRENCY_LEVEL, 3); + config.setString(CONFIG_CORE_SECTION, CONFIG_DFS_SECTION, + CONFIG_KEY_STREAM_RATIO, "0.5"); + + DfsBlockCacheConfig cacheConfig = new DfsBlockCacheConfig() + .fromConfig(config); + assertThat(cacheConfig.getBlockLimit(), is(50L * 1024L)); + assertThat(cacheConfig.getBlockSize(), is(1024)); + assertThat(cacheConfig.getConcurrencyLevel(), is(3)); + assertThat(cacheConfig.getStreamRatio(), closeTo(0.5, 0.0001)); + } + + @Test + public void fromConfig_blockLimitNotAMultipleOfBlockSize_throws() { + Config config = new Config(); + config.setLong(CONFIG_CORE_SECTION, CONFIG_DFS_SECTION, + CONFIG_KEY_BLOCK_LIMIT, 1025); + config.setInt(CONFIG_CORE_SECTION, CONFIG_DFS_SECTION, + CONFIG_KEY_BLOCK_SIZE, 1024); + + assertThrows(IllegalArgumentException.class, + () -> new DfsBlockCacheConfig().fromConfig(config)); + } + + @Test + public void fromConfig_streamRatioInvalidFormat_throws() { + Config config = new Config(); + config.setString(CONFIG_CORE_SECTION, CONFIG_DFS_SECTION, + CONFIG_KEY_STREAM_RATIO, "0.a5"); + + assertThrows(IllegalArgumentException.class, + () -> new DfsBlockCacheConfig().fromConfig(config)); + } + + @Test + public void fromConfig_generatesDfsBlockCachePackExtConfigs() { + Config config = new Config(); + addPackExtConfigEntry(config, "pack", List.of(PackExt.PACK), + /* blockLimit= */ 20 * 512, /* blockSize= */ 512); + + addPackExtConfigEntry(config, "bitmap", List.of(PackExt.BITMAP_INDEX), + /* blockLimit= */ 25 * 1024, /* blockSize= */ 1024); + + addPackExtConfigEntry(config, "index", + List.of(PackExt.INDEX, PackExt.OBJECT_SIZE_INDEX, + PackExt.REVERSE_INDEX), + /* blockLimit= */ 30 * 1024, /* blockSize= */ 1024); + + DfsBlockCacheConfig cacheConfig = new DfsBlockCacheConfig() + .fromConfig(config); + var configs = cacheConfig.getPackExtCacheConfigurations(); + assertThat(configs, hasSize(3)); + var packConfig = getConfigForExt(configs, PackExt.PACK); + assertThat(packConfig.getBlockLimit(), is(20L * 512L)); + assertThat(packConfig.getBlockSize(), is(512)); + + var bitmapConfig = getConfigForExt(configs, PackExt.BITMAP_INDEX); + assertThat(bitmapConfig.getBlockLimit(), is(25L * 1024L)); + assertThat(bitmapConfig.getBlockSize(), is(1024)); + + var indexConfig = getConfigForExt(configs, PackExt.INDEX); + assertThat(indexConfig.getBlockLimit(), is(30L * 1024L)); + assertThat(indexConfig.getBlockSize(), is(1024)); + assertThat(getConfigForExt(configs, PackExt.OBJECT_SIZE_INDEX), + is(indexConfig)); + assertThat(getConfigForExt(configs, PackExt.REVERSE_INDEX), + is(indexConfig)); + } + + @Test + public void fromConfigs_dfsBlockCachePackExtConfigWithDuplicateExtensions_throws() { + Config config = new Config(); + config.setString(CONFIG_CORE_SECTION, CONFIG_DFS_CACHE_PREFIX + "pack1", + CONFIG_KEY_PACK_EXTENSIONS, PackExt.PACK.name()); + + config.setString(CONFIG_CORE_SECTION, CONFIG_DFS_CACHE_PREFIX + "pack2", + CONFIG_KEY_PACK_EXTENSIONS, PackExt.PACK.name()); + + assertThrows(IllegalArgumentException.class, + () -> new DfsBlockCacheConfig().fromConfig(config)); + } + + @Test + public void fromConfigs_dfsBlockCachePackExtConfigWithEmptyExtensions_throws() { + Config config = new Config(); + config.setString(CONFIG_CORE_SECTION, CONFIG_DFS_CACHE_PREFIX + "pack1", + CONFIG_KEY_PACK_EXTENSIONS, ""); + + assertThrows(IllegalArgumentException.class, + () -> new DfsBlockCacheConfig().fromConfig(config)); + } + + @Test + public void fromConfigs_dfsBlockCachePackExtConfigWithNoExtensions_throws() { + Config config = new Config(); + config.setInt(CONFIG_CORE_SECTION, CONFIG_DFS_CACHE_PREFIX + "pack1", + CONFIG_KEY_BLOCK_SIZE, 0); + + assertThrows(IllegalArgumentException.class, + () -> new DfsBlockCacheConfig().fromConfig(config)); + } + + @Test + public void fromConfigs_dfsBlockCachePackExtConfigWithUnknownExtensions_throws() { + Config config = new Config(); + config.setString(CONFIG_CORE_SECTION, + CONFIG_DFS_CACHE_PREFIX + "unknownExt", + CONFIG_KEY_PACK_EXTENSIONS, "NotAKnownExt"); + + assertThrows(IllegalArgumentException.class, + () -> new DfsBlockCacheConfig().fromConfig(config)); + } + + private static void addPackExtConfigEntry(Config config, String configName, + List packExts, long blockLimit, int blockSize) { + String packExtConfigName = CONFIG_DFS_CACHE_PREFIX + configName; + config.setString(CONFIG_CORE_SECTION, packExtConfigName, + CONFIG_KEY_PACK_EXTENSIONS, packExts.stream().map(PackExt::name) + .collect(Collectors.joining(" "))); + config.setLong(CONFIG_CORE_SECTION, packExtConfigName, + CONFIG_KEY_BLOCK_LIMIT, blockLimit); + config.setInt(CONFIG_CORE_SECTION, packExtConfigName, + CONFIG_KEY_BLOCK_SIZE, blockSize); + } + + private static DfsBlockCacheConfig getConfigForExt( + List configs, PackExt packExt) { + for (DfsBlockCachePackExtConfig config : configs) { + if (config.getPackExts().contains(packExt)) { + return config.getPackExtCacheConfiguration(); + } + } + return null; + } } diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties index 19c90086aa..9d12facb33 100644 --- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties +++ b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties @@ -285,6 +285,7 @@ DIRCUnrecognizedExtendedFlags=Unrecognized extended flags: {0} downloadCancelled=Download cancelled downloadCancelledDuringIndexing=Download cancelled during indexing duplicateAdvertisementsOf=duplicate advertisements of {0} +duplicatePackExtensionsSet=Attempting to configure duplicate pack extensions: {0}.{1}.{2} contains {3} duplicateRef=Duplicate ref: {0} duplicateRefAttribute=Duplicate ref attribute: {0} duplicateRemoteRefUpdateIsIllegal=Duplicate remote ref update is illegal. Affected remote name: {0} @@ -539,6 +540,8 @@ noMergeBase=No merge base could be determined. Reason={0}. {1} noMergeHeadSpecified=No merge head specified nonBareLinkFilesNotSupported=Link files are not supported with nonbare repos nonCommitToHeads=Cannot point a branch to a non-commit object +noPackExtConfigurationGiven=No PackExt configuration given +noPackExtGivenForConfiguration=No PackExt given for configuration noPathAttributesFound=No Attributes found for {0}. noSuchRef=no such ref noSuchRefKnown=no such ref: {0} @@ -829,6 +832,7 @@ unknownObject=unknown object unknownObjectInIndex=unknown object {0} found in index but not in pack file unknownObjectType=Unknown object type {0}. unknownObjectType2=unknown +unknownPackExtension=Unknown pack extension: {0}.{1}.{2}={3} unknownPositionEncoding=Unknown position encoding %s unknownRefStorageFormat=Unknown ref storage format "{0}" unknownRepositoryFormat=Unknown repository format diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java index 700b54a7a6..311d9c22aa 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java @@ -315,6 +315,7 @@ public class JGitText extends TranslationBundle { /***/ public String downloadCancelled; /***/ public String downloadCancelledDuringIndexing; /***/ public String duplicateAdvertisementsOf; + /***/ public String duplicatePackExtensionsSet; /***/ public String duplicateRef; /***/ public String duplicateRefAttribute; /***/ public String duplicateRemoteRefUpdateIsIllegal; @@ -569,6 +570,8 @@ public class JGitText extends TranslationBundle { /***/ public String noMergeHeadSpecified; /***/ public String nonBareLinkFilesNotSupported; /***/ public String nonCommitToHeads; + /***/ public String noPackExtConfigurationGiven; + /***/ public String noPackExtGivenForConfiguration; /***/ public String noPathAttributesFound; /***/ public String noSuchRef; /***/ public String noSuchRefKnown; @@ -858,6 +861,7 @@ public class JGitText extends TranslationBundle { /***/ public String unknownObjectInIndex; /***/ public String unknownObjectType; /***/ public String unknownObjectType2; + /***/ public String unknownPackExtension; /***/ public String unknownPositionEncoding; /***/ public String unknownRefStorageFormat; /***/ public String unknownRepositoryFormat; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java index 77273cec61..fa86701de8 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java @@ -11,17 +11,25 @@ package org.eclipse.jgit.internal.storage.dfs; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_CORE_SECTION; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_DFS_CACHE_PREFIX; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_DFS_SECTION; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_BLOCK_LIMIT; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_BLOCK_SIZE; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_CONCURRENCY_LEVEL; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_PACK_EXTENSIONS; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_STREAM_RATIO; import java.text.MessageFormat; import java.time.Duration; +import java.util.ArrayList; import java.util.Collections; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Consumer; +import java.util.stream.Collectors; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.pack.PackExt; @@ -42,15 +50,21 @@ public class DfsBlockCacheConfig { public static final int DEFAULT_CACHE_HOT_MAX = 1; private long blockLimit; + private int blockSize; + private double streamRatio; + private int concurrencyLevel; private Consumer refLock; + private Map cacheHotMap; private IndexEventConsumer indexEventConsumer; + private List packExtCacheConfigurations; + /** * Create a default configuration. */ @@ -60,6 +74,7 @@ public class DfsBlockCacheConfig { setStreamRatio(0.30); setConcurrencyLevel(32); cacheHotMap = Collections.emptyMap(); + packExtCacheConfigurations = Collections.emptyList(); } /** @@ -77,10 +92,10 @@ public class DfsBlockCacheConfig { * Set maximum number bytes of heap memory to dedicate to caching pack file * data. *

- * It is strongly recommended to set the block limit to be an integer multiple - * of the block size. This constraint is not enforced by this method (since - * it may be called before {@link #setBlockSize(int)}), but it is enforced by - * {@link #fromConfig(Config)}. + * It is strongly recommended to set the block limit to be an integer + * multiple of the block size. This constraint is not enforced by this + * method (since it may be called before {@link #setBlockSize(int)}), but it + * is enforced by {@link #fromConfig(Config)}. * * @param newLimit * maximum number bytes of heap memory to dedicate to caching @@ -89,9 +104,9 @@ public class DfsBlockCacheConfig { */ public DfsBlockCacheConfig setBlockLimit(long newLimit) { if (newLimit <= 0) { - throw new IllegalArgumentException(MessageFormat.format( - JGitText.get().blockLimitNotPositive, - Long.valueOf(newLimit))); + throw new IllegalArgumentException( + MessageFormat.format(JGitText.get().blockLimitNotPositive, + Long.valueOf(newLimit))); } blockLimit = newLimit; return this; @@ -239,62 +254,101 @@ public class DfsBlockCacheConfig { return this; } + /** + * Get the list of pack ext cache configs. + * + * @return the list of pack ext cache configs. + */ + List getPackExtCacheConfigurations() { + return packExtCacheConfigurations; + } + /** * Update properties by setting fields from the configuration. *

* If a property is not defined in the configuration, then it is left * unmodified. *

- * Enforces certain constraints on the combination of settings in the config, - * for example that the block limit is a multiple of the block size. + * Enforces certain constraints on the combination of settings in the + * config, for example that the block limit is a multiple of the block size. * * @param rc * configuration to read properties from. * @return {@code this} */ public DfsBlockCacheConfig fromConfig(Config rc) { - long cfgBlockLimit = rc.getLong( - CONFIG_CORE_SECTION, - CONFIG_DFS_SECTION, - CONFIG_KEY_BLOCK_LIMIT, - getBlockLimit()); - int cfgBlockSize = rc.getInt( - CONFIG_CORE_SECTION, - CONFIG_DFS_SECTION, - CONFIG_KEY_BLOCK_SIZE, + fromConfig(CONFIG_CORE_SECTION, CONFIG_DFS_SECTION, rc); + loadPackExtConfigs(rc); + return this; + } + + private void fromConfig(String section, String subSection, Config rc) { + long cfgBlockLimit = rc.getLong(section, subSection, + CONFIG_KEY_BLOCK_LIMIT, getBlockLimit()); + int cfgBlockSize = rc.getInt(section, subSection, CONFIG_KEY_BLOCK_SIZE, getBlockSize()); if (cfgBlockLimit % cfgBlockSize != 0) { throw new IllegalArgumentException(MessageFormat.format( JGitText.get().blockLimitNotMultipleOfBlockSize, - Long.valueOf(cfgBlockLimit), - Long.valueOf(cfgBlockSize))); + Long.valueOf(cfgBlockLimit), Long.valueOf(cfgBlockSize))); } setBlockLimit(cfgBlockLimit); setBlockSize(cfgBlockSize); - setConcurrencyLevel(rc.getInt( - CONFIG_CORE_SECTION, - CONFIG_DFS_SECTION, - CONFIG_KEY_CONCURRENCY_LEVEL, - getConcurrencyLevel())); + setConcurrencyLevel(rc.getInt(section, subSection, + CONFIG_KEY_CONCURRENCY_LEVEL, getConcurrencyLevel())); - String v = rc.getString( - CONFIG_CORE_SECTION, - CONFIG_DFS_SECTION, - CONFIG_KEY_STREAM_RATIO); + String v = rc.getString(section, subSection, CONFIG_KEY_STREAM_RATIO); if (v != null) { try { setStreamRatio(Double.parseDouble(v)); } catch (NumberFormatException e) { throw new IllegalArgumentException(MessageFormat.format( - JGitText.get().enumValueNotSupported3, - CONFIG_CORE_SECTION, - CONFIG_DFS_SECTION, - CONFIG_KEY_STREAM_RATIO, v), e); + JGitText.get().enumValueNotSupported3, section, + subSection, CONFIG_KEY_STREAM_RATIO, v), e); } } - return this; + } + + private void loadPackExtConfigs(Config config) { + List subSections = config.getSubsections(CONFIG_CORE_SECTION) + .stream() + .filter(section -> section.startsWith(CONFIG_DFS_CACHE_PREFIX)) + .collect(Collectors.toList()); + if (subSections.size() == 0) { + return; + } + ArrayList cacheConfigs = new ArrayList<>(); + Set extensionsSeen = new HashSet<>(); + for (String subSection : subSections) { + var cacheConfig = DfsBlockCachePackExtConfig.fromConfig(config, + CONFIG_CORE_SECTION, subSection); + Set packExtsDuplicates = intersection(extensionsSeen, + cacheConfig.packExts); + if (packExtsDuplicates.size() > 0) { + String duplicatePackExts = packExtsDuplicates.stream() + .map(PackExt::toString) + .collect(Collectors.joining(",")); + throw new IllegalArgumentException(MessageFormat.format( + JGitText.get().duplicatePackExtensionsSet, + CONFIG_CORE_SECTION, subSection, + CONFIG_KEY_PACK_EXTENSIONS, duplicatePackExts)); + } + extensionsSeen.addAll(cacheConfig.packExts); + cacheConfigs.add(cacheConfig); + } + packExtCacheConfigurations = cacheConfigs; + } + + private static Set intersection(Set first, Set second) { + Set ret = new HashSet<>(); + for (T entry : second) { + if (first.contains(entry)) { + ret.add(entry); + } + } + return ret; } /** Consumer of DfsBlockCache loading and eviction events for indexes. */ @@ -346,4 +400,81 @@ public class DfsBlockCacheConfig { return false; } } + + /** + * A configuration for a single cache table storing 1 or more Pack + * extensions. + *

+ * The current pack ext cache tables implementation supports the same + * parameters the ClockBlockCacheTable (current default implementation). + *

+ * Configuration falls back to the defaults coded values defined in the + * {@link DfsBlockCacheConfig} when not set on each cache table + * configuration and NOT the values of the basic dfs section. + *

+ * + * + * Format: + * [core "dfs.packCache"] + * packExtensions = "PACK" + * blockSize = 512 + * blockLimit = 100 + * concurrencyLevel = 5 + * + * [core "dfs.multipleExtensionCache"] + * packExtensions = "INDEX REFTABLE BITMAP_INDEX" + * blockSize = 512 + * blockLimit = 100 + * concurrencyLevel = 5 + * + */ + static class DfsBlockCachePackExtConfig { + // Set of pack extensions that will map to the cache instance. + private final EnumSet packExts; + + // Configuration for the cache instance. + private final DfsBlockCacheConfig packExtCacheConfiguration; + + private DfsBlockCachePackExtConfig(EnumSet packExts, + DfsBlockCacheConfig packExtCacheConfiguration) { + this.packExts = packExts; + this.packExtCacheConfiguration = packExtCacheConfiguration; + } + + Set getPackExts() { + return packExts; + } + + DfsBlockCacheConfig getPackExtCacheConfiguration() { + return packExtCacheConfiguration; + } + + private static DfsBlockCachePackExtConfig fromConfig(Config config, + String section, String subSection) { + String packExtensions = config.getString(section, subSection, + CONFIG_KEY_PACK_EXTENSIONS); + if (packExtensions == null) { + throw new IllegalArgumentException( + JGitText.get().noPackExtGivenForConfiguration); + } + String[] extensions = packExtensions.split(" ", -1); + Set packExts = new HashSet<>(extensions.length); + for (String extension : extensions) { + try { + packExts.add(PackExt.valueOf(extension)); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException(MessageFormat.format( + JGitText.get().unknownPackExtension, section, + subSection, CONFIG_KEY_PACK_EXTENSIONS, extension), + e); + } + } + + DfsBlockCacheConfig dfsBlockCacheConfig = new DfsBlockCacheConfig(); + dfsBlockCacheConfig.fromConfig(section, subSection, config); + return new DfsBlockCachePackExtConfig(EnumSet.copyOf(packExts), + dfsBlockCacheConfig); + } + + } } \ No newline at end of file diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java index 0edf3c5ad0..61db6a0762 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java @@ -77,6 +77,9 @@ public final class ConfigConstants { /** The "dfs" section */ public static final String CONFIG_DFS_SECTION = "dfs"; + /** The dfs cache subsection prefix */ + public static final String CONFIG_DFS_CACHE_PREFIX = "dfs."; + /** * The "receive" section * @since 4.6 @@ -331,6 +334,13 @@ public final class ConfigConstants { /** The "deltaBaseCacheLimit" key */ public static final String CONFIG_KEY_DELTA_BASE_CACHE_LIMIT = "deltaBaseCacheLimit"; + /** + * The "packExtensions" key + * + * @since 7.0 + **/ + public static final String CONFIG_KEY_PACK_EXTENSIONS = "packExtensions"; + /** * The "symlinks" key * @since 3.3 -- cgit v1.2.3 From f9beeb3b33e438fa3a10eb3871206573ae986abb Mon Sep 17 00:00:00 2001 From: Janne Valkealahti Date: Thu, 4 Jul 2024 07:44:49 +0100 Subject: Add worktrees read support Based on deritative work done in Andre's work in [1]. This change focuses on adding support for reading the repository state when branches are checked out using git's worktrees. I've refactored original work by removing all unrelevant changes which were mostly around refactoring to extract i.e. constants which mostly created noise for a review. I've tried to address original review comments: - Not adding non-behavioral changes - "HEAD" should get resolved from gitDir - Reftable recently landed in cgit 2.45, see https://github.com/git/git/blob/master/Documentation/RelNotes/2.45.0.txt#L8 We can add worktree support for reftable in a later change. - Some new tests to read from a linked worktree which is created manually as there's no write support. [1] https://git.eclipse.org/r/c/jgit/jgit/+/163940/18 Change-Id: Id077d58fb6c09ecb090eb09d5dbc7edc351a581d --- .../src/org/eclipse/jgit/pgm/Config.java | 2 +- .../org/eclipse/jgit/api/EolRepositoryTest.java | 2 +- .../org/eclipse/jgit/api/LinkedWorktreeTest.java | 192 +++++++++++++++++++++ .../jgit/attributes/AttributesHandlerTest.java | 2 +- .../internal/storage/file/BatchRefUpdateTest.java | 2 +- .../jgit/internal/storage/file/GcPackRefsTest.java | 2 +- .../jgit/internal/storage/file/GcReflogTest.java | 2 +- .../internal/storage/file/RefDirectoryTest.java | 31 ++-- .../internal/storage/file/ReflogReaderTest.java | 2 +- .../internal/storage/file/ReflogWriterTest.java | 2 +- .../org/eclipse/jgit/api/SubmoduleAddCommand.java | 2 +- .../eclipse/jgit/api/SubmoduleUpdateCommand.java | 2 +- .../eclipse/jgit/dircache/DirCacheCheckout.java | 2 + .../storage/file/FileReftableDatabase.java | 8 +- .../jgit/internal/storage/file/FileRepository.java | 39 +++-- .../org/eclipse/jgit/internal/storage/file/GC.java | 2 +- .../eclipse/jgit/internal/storage/file/GcLog.java | 2 +- .../internal/storage/file/InfoAttributesNode.java | 2 +- .../jgit/internal/storage/file/RefDirectory.java | 20 ++- .../internal/storage/file/ReflogReaderImpl.java | 6 +- .../eclipse/jgit/lib/BaseRepositoryBuilder.java | 116 +++++++++++-- .../src/org/eclipse/jgit/lib/Constants.java | 52 ++++++ .../src/org/eclipse/jgit/lib/IndexDiff.java | 2 +- .../src/org/eclipse/jgit/lib/Repository.java | 16 +- .../src/org/eclipse/jgit/lib/RepositoryCache.java | 51 ++++-- .../eclipse/jgit/transport/TransportGitSsh.java | 6 + .../org/eclipse/jgit/transport/TransportLocal.java | 1 + .../eclipse/jgit/treewalk/WorkingTreeIterator.java | 4 +- org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java | 33 +++- 29 files changed, 520 insertions(+), 85 deletions(-) create mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/api/LinkedWorktreeTest.java (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Config.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Config.java index 52f40c2957..f5de7045d0 100644 --- a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Config.java +++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Config.java @@ -94,7 +94,7 @@ class Config extends TextBuiltin { if (global || isListAll()) list(SystemReader.getInstance().openUserConfig(null, fs)); if (local || isListAll()) - list(new FileBasedConfig(fs.resolve(getRepository().getDirectory(), + list(new FileBasedConfig(fs.resolve(getRepository().getCommonDirectory(), Constants.CONFIG), fs)); } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/EolRepositoryTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/EolRepositoryTest.java index b937b1f6a9..4c971ffb6b 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/EolRepositoryTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/EolRepositoryTest.java @@ -559,7 +559,7 @@ public class EolRepositoryTest extends RepositoryTestCase { } if (infoAttributesContent != null) { - File f = new File(db.getDirectory(), Constants.INFO_ATTRIBUTES); + File f = new File(db.getCommonDirectory(), Constants.INFO_ATTRIBUTES); write(f, infoAttributesContent); } config.save(); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/LinkedWorktreeTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/LinkedWorktreeTest.java new file mode 100644 index 0000000000..3b60e1b5c0 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/LinkedWorktreeTest.java @@ -0,0 +1,192 @@ +/* + * Copyright (C) 2024, Broadcom and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.api; + +import static org.eclipse.jgit.lib.Constants.HEAD; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Collection; +import java.util.Iterator; + +import org.eclipse.jgit.internal.storage.file.FileRepository; +import org.eclipse.jgit.junit.JGitTestUtil; +import org.eclipse.jgit.junit.RepositoryTestCase; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ReflogEntry; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.util.FS; +import org.eclipse.jgit.util.FS.ExecutionResult; +import org.eclipse.jgit.util.RawParseUtils; +import org.eclipse.jgit.util.TemporaryBuffer; +import org.junit.Test; + +public class LinkedWorktreeTest extends RepositoryTestCase { + + @Override + public void setUp() throws Exception { + super.setUp(); + + try (Git git = new Git(db)) { + git.commit().setMessage("Initial commit").call(); + } + } + + @Test + public void testWeCanReadFromLinkedWorktreeFromBare() throws Exception { + FS fs = db.getFS(); + File directory = trash.getParentFile(); + String dbDirName = db.getWorkTree().getName(); + cloneBare(fs, directory, dbDirName, "bare"); + File bareDirectory = new File(directory, "bare"); + worktreeAddExisting(fs, bareDirectory, "master"); + + File worktreesDir = new File(bareDirectory, "worktrees"); + File masterWorktreesDir = new File(worktreesDir, "master"); + + FileRepository repository = new FileRepository(masterWorktreesDir); + try (Git git = new Git(repository)) { + ObjectId objectId = repository.resolve(HEAD); + assertNotNull(objectId); + + Iterator log = git.log().all().call().iterator(); + assertTrue(log.hasNext()); + assertTrue("Initial commit".equals(log.next().getShortMessage())); + + // we have reflog entry + // depending on git version we either have one or + // two entries where extra is zeroid entry with + // same message or no message + Collection reflog = git.reflog().call(); + assertNotNull(reflog); + assertTrue(reflog.size() > 0); + ReflogEntry[] reflogs = reflog.toArray(new ReflogEntry[0]); + assertEquals(reflogs[reflogs.length - 1].getComment(), + "reset: moving to HEAD"); + + // index works with file changes + File masterDir = new File(directory, "master"); + File testFile = new File(masterDir, "test"); + + Status status = git.status().call(); + assertTrue(status.getUncommittedChanges().size() == 0); + assertTrue(status.getUntracked().size() == 0); + + JGitTestUtil.write(testFile, "test"); + status = git.status().call(); + assertTrue(status.getUncommittedChanges().size() == 0); + assertTrue(status.getUntracked().size() == 1); + + git.add().addFilepattern("test").call(); + status = git.status().call(); + assertTrue(status.getUncommittedChanges().size() == 1); + assertTrue(status.getUntracked().size() == 0); + } + } + + @Test + public void testWeCanReadFromLinkedWorktreeFromNonBare() throws Exception { + FS fs = db.getFS(); + worktreeAddNew(fs, db.getWorkTree(), "wt"); + + File worktreesDir = new File(db.getDirectory(), "worktrees"); + File masterWorktreesDir = new File(worktreesDir, "wt"); + + FileRepository repository = new FileRepository(masterWorktreesDir); + try (Git git = new Git(repository)) { + ObjectId objectId = repository.resolve(HEAD); + assertNotNull(objectId); + + Iterator log = git.log().all().call().iterator(); + assertTrue(log.hasNext()); + assertTrue("Initial commit".equals(log.next().getShortMessage())); + + // we have reflog entry + Collection reflog = git.reflog().call(); + assertNotNull(reflog); + assertTrue(reflog.size() > 0); + ReflogEntry[] reflogs = reflog.toArray(new ReflogEntry[0]); + assertEquals(reflogs[reflogs.length - 1].getComment(), + "reset: moving to HEAD"); + + // index works with file changes + File directory = trash.getParentFile(); + File wtDir = new File(directory, "wt"); + File testFile = new File(wtDir, "test"); + + Status status = git.status().call(); + assertTrue(status.getUncommittedChanges().size() == 0); + assertTrue(status.getUntracked().size() == 0); + + JGitTestUtil.write(testFile, "test"); + status = git.status().call(); + assertTrue(status.getUncommittedChanges().size() == 0); + assertTrue(status.getUntracked().size() == 1); + + git.add().addFilepattern("test").call(); + status = git.status().call(); + assertTrue(status.getUncommittedChanges().size() == 1); + assertTrue(status.getUntracked().size() == 0); + } + + } + + private static void cloneBare(FS fs, File directory, String from, String to) throws IOException, InterruptedException { + ProcessBuilder builder = fs.runInShell("git", + new String[] { "clone", "--bare", from, to }); + builder.directory(directory); + builder.environment().put("HOME", fs.userHome().getAbsolutePath()); + StringBuilder input = new StringBuilder(); + ExecutionResult result = fs.execute(builder, new ByteArrayInputStream( + input.toString().getBytes(StandardCharsets.UTF_8))); + String stdOut = toString(result.getStdout()); + String errorOut = toString(result.getStderr()); + assertNotNull(stdOut); + assertNotNull(errorOut); + } + + private static void worktreeAddExisting(FS fs, File directory, String name) throws IOException, InterruptedException { + ProcessBuilder builder = fs.runInShell("git", + new String[] { "worktree", "add", "../" + name, name }); + builder.directory(directory); + builder.environment().put("HOME", fs.userHome().getAbsolutePath()); + StringBuilder input = new StringBuilder(); + ExecutionResult result = fs.execute(builder, new ByteArrayInputStream( + input.toString().getBytes(StandardCharsets.UTF_8))); + String stdOut = toString(result.getStdout()); + String errorOut = toString(result.getStderr()); + assertNotNull(stdOut); + assertNotNull(errorOut); + } + + private static void worktreeAddNew(FS fs, File directory, String name) throws IOException, InterruptedException { + ProcessBuilder builder = fs.runInShell("git", + new String[] { "worktree", "add", "-b", name, "../" + name, "master"}); + builder.directory(directory); + builder.environment().put("HOME", fs.userHome().getAbsolutePath()); + StringBuilder input = new StringBuilder(); + ExecutionResult result = fs.execute(builder, new ByteArrayInputStream( + input.toString().getBytes(StandardCharsets.UTF_8))); + String stdOut = toString(result.getStdout()); + String errorOut = toString(result.getStderr()); + assertNotNull(stdOut); + assertNotNull(errorOut); + } + + private static String toString(TemporaryBuffer b) throws IOException { + return RawParseUtils.decode(b.toByteArray()); + } + +} diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/attributes/AttributesHandlerTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/attributes/AttributesHandlerTest.java index 7fb98ec53b..c41dd81add 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/attributes/AttributesHandlerTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/attributes/AttributesHandlerTest.java @@ -584,7 +584,7 @@ public class AttributesHandlerTest extends RepositoryTestCase { } if (infoAttributesContent != null) { - File f = new File(db.getDirectory(), Constants.INFO_ATTRIBUTES); + File f = new File(db.getCommonDirectory(), Constants.INFO_ATTRIBUTES); write(f, infoAttributesContent); } config.save(); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/BatchRefUpdateTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/BatchRefUpdateTest.java index daf4382719..1af42cb229 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/BatchRefUpdateTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/BatchRefUpdateTest.java @@ -171,7 +171,7 @@ public class BatchRefUpdateTest extends LocalDiskRepositoryTestCase { assertEquals(c2.getResult(), ReceiveCommand.Result.OK); } - File packed = new File(diskRepo.getDirectory(), "packed-refs"); + File packed = new File(diskRepo.getCommonDirectory(), "packed-refs"); String packedStr = new String(Files.readAllBytes(packed.toPath()), UTF_8); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcPackRefsTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcPackRefsTest.java index 8baa3cc341..c57295518d 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcPackRefsTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcPackRefsTest.java @@ -58,7 +58,7 @@ public class GcPackRefsTest extends GcTestCase { String ref = "dir/ref"; tr.branch(ref).commit().create(); String name = repo.findRef(ref).getName(); - Path dir = repo.getDirectory().toPath().resolve(name).getParent(); + Path dir = repo.getCommonDirectory().toPath().resolve(name).getParent(); assertNotNull(dir); gc.packRefs(); assertFalse(Files.exists(dir)); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcReflogTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcReflogTest.java index e6c1ee5fd6..29f180d76b 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcReflogTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcReflogTest.java @@ -30,7 +30,7 @@ public class GcReflogTest extends GcTestCase { BranchBuilder bb = tr.branch("refs/heads/master"); bb.commit().add("A", "A").add("B", "B").create(); bb.commit().add("A", "A2").add("B", "B2").create(); - new File(repo.getDirectory(), Constants.LOGS + "/refs/heads/master") + new File(repo.getCommonDirectory(), Constants.LOGS + "/refs/heads/master") .delete(); stats = gc.getStatistics(); assertEquals(8, stats.numberOfLooseObjects); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/RefDirectoryTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/RefDirectoryTest.java index 2bafde65d3..baa0182b87 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/RefDirectoryTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/RefDirectoryTest.java @@ -90,25 +90,26 @@ public class RefDirectoryTest extends LocalDiskRepositoryTestCase { @Test public void testCreate() throws IOException { // setUp above created the directory. We just have to test it. - File d = diskRepo.getDirectory(); + File gitDir = diskRepo.getDirectory(); + File commonDir = diskRepo.getCommonDirectory(); assertSame(diskRepo, refdir.getRepository()); - assertTrue(new File(d, "refs").isDirectory()); - assertTrue(new File(d, "logs").isDirectory()); - assertTrue(new File(d, "logs/refs").isDirectory()); - assertFalse(new File(d, "packed-refs").exists()); + assertTrue(new File(commonDir, "refs").isDirectory()); + assertTrue(new File(commonDir, "logs").isDirectory()); + assertTrue(new File(commonDir, "logs/refs").isDirectory()); + assertFalse(new File(commonDir, "packed-refs").exists()); - assertTrue(new File(d, "refs/heads").isDirectory()); - assertTrue(new File(d, "refs/tags").isDirectory()); - assertEquals(2, new File(d, "refs").list().length); - assertEquals(0, new File(d, "refs/heads").list().length); - assertEquals(0, new File(d, "refs/tags").list().length); + assertTrue(new File(commonDir, "refs/heads").isDirectory()); + assertTrue(new File(commonDir, "refs/tags").isDirectory()); + assertEquals(2, new File(commonDir, "refs").list().length); + assertEquals(0, new File(commonDir, "refs/heads").list().length); + assertEquals(0, new File(commonDir, "refs/tags").list().length); - assertTrue(new File(d, "logs/refs/heads").isDirectory()); - assertFalse(new File(d, "logs/HEAD").exists()); - assertEquals(0, new File(d, "logs/refs/heads").list().length); + assertTrue(new File(commonDir, "logs/refs/heads").isDirectory()); + assertFalse(new File(gitDir, "logs/HEAD").exists()); + assertEquals(0, new File(commonDir, "logs/refs/heads").list().length); - assertEquals("ref: refs/heads/master\n", read(new File(d, HEAD))); + assertEquals("ref: refs/heads/master\n", read(new File(gitDir, HEAD))); } @Test(expected = UnsupportedOperationException.class) @@ -1382,7 +1383,7 @@ public class RefDirectoryTest extends LocalDiskRepositoryTestCase { } private void deleteLooseRef(String name) { - File path = new File(diskRepo.getDirectory(), name); + File path = new File(diskRepo.getCommonDirectory(), name); assertTrue("deleted " + name, path.delete()); } } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/ReflogReaderTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/ReflogReaderTest.java index dc0e749373..eb521ff9eb 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/ReflogReaderTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/ReflogReaderTest.java @@ -238,7 +238,7 @@ public class ReflogReaderTest extends SampleDataRepositoryTestCase { private void setupReflog(String logName, byte[] data) throws FileNotFoundException, IOException { - File logfile = new File(db.getDirectory(), logName); + File logfile = new File(db.getCommonDirectory(), logName); if (!logfile.getParentFile().mkdirs() && !logfile.getParentFile().isDirectory()) { throw new IOException( diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/ReflogWriterTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/ReflogWriterTest.java index 8d0e99dea0..8e9b7b84bd 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/ReflogWriterTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/ReflogWriterTest.java @@ -48,7 +48,7 @@ public class ReflogWriterTest extends SampleDataRepositoryTestCase { private void readReflog(byte[] buffer) throws FileNotFoundException, IOException { - File logfile = new File(db.getDirectory(), "logs/refs/heads/master"); + File logfile = new File(db.getCommonDirectory(), "logs/refs/heads/master"); if (!logfile.getParentFile().mkdirs() && !logfile.getParentFile().isDirectory()) { throw new IOException( diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleAddCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleAddCommand.java index 8fb5d60b85..401f069e4e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleAddCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleAddCommand.java @@ -176,7 +176,7 @@ public class SubmoduleAddCommand extends CloneCommand clone = Git.cloneRepository(); configure(clone); clone.setDirectory(moduleDirectory); - clone.setGitDir(new File(new File(repo.getDirectory(), + clone.setGitDir(new File(new File(repo.getCommonDirectory(), Constants.MODULES), path)); clone.setURI(resolvedUri); if (monitor != null) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleUpdateCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleUpdateCommand.java index df73164161..751dabcd6b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleUpdateCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleUpdateCommand.java @@ -128,7 +128,7 @@ public class SubmoduleUpdateCommand extends clone.setURI(url); clone.setDirectory(generator.getDirectory()); clone.setGitDir( - new File(new File(repo.getDirectory(), Constants.MODULES), + new File(new File(repo.getCommonDirectory(), Constants.MODULES), generator.getPath())); if (monitor != null) { clone.setProgressMonitor(monitor); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java index 6ae5153c12..fa0a82fd58 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java @@ -1647,6 +1647,8 @@ public class DirCacheCheckout { filterProcessBuilder.directory(repo.getWorkTree()); filterProcessBuilder.environment().put(Constants.GIT_DIR_KEY, repo.getDirectory().getAbsolutePath()); + filterProcessBuilder.environment().put(Constants.GIT_COMMON_DIR_KEY, + repo.getCommonDirectory().getAbsolutePath()); ExecutionResult result; int rc; try { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java index ed2516ddd0..80240e5062 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java @@ -68,14 +68,14 @@ public class FileReftableDatabase extends RefDatabase { private final FileReftableStack reftableStack; FileReftableDatabase(FileRepository repo) throws IOException { - this(repo, new File(new File(repo.getDirectory(), Constants.REFTABLE), + this(repo, new File(new File(repo.getCommonDirectory(), Constants.REFTABLE), Constants.TABLES_LIST)); } FileReftableDatabase(FileRepository repo, File refstackName) throws IOException { this.fileRepository = repo; this.reftableStack = new FileReftableStack(refstackName, - new File(fileRepository.getDirectory(), Constants.REFTABLE), + new File(fileRepository.getCommonDirectory(), Constants.REFTABLE), () -> fileRepository.fireEvent(new RefsChangedEvent()), () -> fileRepository.getConfig()); this.reftableDatabase = new ReftableDatabase() { @@ -318,7 +318,7 @@ public class FileReftableDatabase extends RefDatabase { @Override public void create() throws IOException { FileUtils.mkdir( - new File(fileRepository.getDirectory(), Constants.REFTABLE), + new File(fileRepository.getCommonDirectory(), Constants.REFTABLE), true); } @@ -615,7 +615,7 @@ public class FileReftableDatabase extends RefDatabase { FileReftableDatabase newDb = null; File reftableList = null; try { - File reftableDir = new File(repo.getDirectory(), + File reftableDir = new File(repo.getCommonDirectory(), Constants.REFTABLE); reftableList = new File(reftableDir, Constants.TABLES_LIST); if (!reftableDir.isDirectory()) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java index e5a00d3925..b5d29a3fc8 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java @@ -165,7 +165,7 @@ public class FileRepository extends Repository { throw new IOException(e.getMessage(), e); } repoConfig = new FileBasedConfig(userConfig, getFS().resolve( - getDirectory(), Constants.CONFIG), + getCommonDirectory(), Constants.CONFIG), getFS()); loadRepoConfig(); @@ -193,7 +193,7 @@ public class FileRepository extends Repository { options.getObjectDirectory(), // options.getAlternateObjectDirectories(), // getFS(), // - new File(getDirectory(), Constants.SHALLOW)); + new File(getCommonDirectory(), Constants.SHALLOW)); if (objectDatabase.exists()) { if (repositoryFormatVersion > 1) @@ -622,16 +622,17 @@ public class FileRepository extends Repository { * on IO problem */ void convertToPackedRefs(boolean writeLogs, boolean backup) throws IOException { + File commonDirectory = getCommonDirectory(); List all = refs.getRefs(); - File packedRefs = new File(getDirectory(), Constants.PACKED_REFS); + File packedRefs = new File(commonDirectory, Constants.PACKED_REFS); if (packedRefs.exists()) { throw new IOException(MessageFormat.format(JGitText.get().fileAlreadyExists, packedRefs.getName())); } - File refsFile = new File(getDirectory(), "refs"); //$NON-NLS-1$ + File refsFile = new File(commonDirectory, "refs"); //$NON-NLS-1$ File refsHeadsFile = new File(refsFile, "heads");//$NON-NLS-1$ - File headFile = new File(getDirectory(), Constants.HEAD); + File headFile = new File(commonDirectory, Constants.HEAD); FileReftableDatabase oldDb = (FileReftableDatabase) refs; // Remove the dummy files that ensure compatibility with older git @@ -701,7 +702,7 @@ public class FileRepository extends Repository { } if (!backup) { - File reftableDir = new File(getDirectory(), Constants.REFTABLE); + File reftableDir = new File(commonDirectory, Constants.REFTABLE); FileUtils.delete(reftableDir, FileUtils.RECURSIVE | FileUtils.IGNORE_ERRORS); } @@ -730,8 +731,10 @@ public class FileRepository extends Repository { @SuppressWarnings("nls") void convertToReftable(boolean writeLogs, boolean backup) throws IOException { - File reftableDir = new File(getDirectory(), Constants.REFTABLE); - File headFile = new File(getDirectory(), Constants.HEAD); + File commonDirectory = getCommonDirectory(); + File directory = getDirectory(); + File reftableDir = new File(commonDirectory, Constants.REFTABLE); + File headFile = new File(directory, Constants.HEAD); if (reftableDir.exists() && FileUtils.hasFiles(reftableDir.toPath())) { throw new IOException(JGitText.get().reftableDirExists); } @@ -739,28 +742,28 @@ public class FileRepository extends Repository { // Ignore return value, as it is tied to temporary newRefs file. FileReftableDatabase.convertFrom(this, writeLogs); - File refsFile = new File(getDirectory(), "refs"); + File refsFile = new File(commonDirectory, "refs"); // non-atomic: remove old data. - File packedRefs = new File(getDirectory(), Constants.PACKED_REFS); - File logsDir = new File(getDirectory(), Constants.LOGS); + File packedRefs = new File(commonDirectory, Constants.PACKED_REFS); + File logsDir = new File(commonDirectory, Constants.LOGS); List additional = getRefDatabase().getAdditionalRefs().stream() .map(Ref::getName).collect(toList()); additional.add(Constants.HEAD); if (backup) { - FileUtils.rename(refsFile, new File(getDirectory(), "refs.old")); + FileUtils.rename(refsFile, new File(commonDirectory, "refs.old")); if (packedRefs.exists()) { - FileUtils.rename(packedRefs, new File(getDirectory(), + FileUtils.rename(packedRefs, new File(commonDirectory, Constants.PACKED_REFS + ".old")); } if (logsDir.exists()) { FileUtils.rename(logsDir, - new File(getDirectory(), Constants.LOGS + ".old")); + new File(commonDirectory, Constants.LOGS + ".old")); } for (String r : additional) { - FileUtils.rename(new File(getDirectory(), r), - new File(getDirectory(), r + ".old")); + FileUtils.rename(new File(commonDirectory, r), + new File(commonDirectory, r + ".old")); } } else { FileUtils.delete(packedRefs, FileUtils.SKIP_MISSING); @@ -770,7 +773,7 @@ public class FileRepository extends Repository { FileUtils.delete(refsFile, FileUtils.RECURSIVE | FileUtils.SKIP_MISSING); for (String r : additional) { - new File(getDirectory(), r).delete(); + new File(commonDirectory, r).delete(); } } @@ -784,7 +787,7 @@ public class FileRepository extends Repository { // Some tools might write directly into .git/refs/heads/BRANCH. By // putting a file here, this fails spectacularly. - FileUtils.createNewFile(new File(refsFile, "heads")); + FileUtils.createNewFile(new File(refsFile, Constants.HEADS)); repoConfig.setString(ConfigConstants.CONFIG_EXTENSIONS_SECTION, null, ConfigConstants.CONFIG_KEY_REF_STORAGE, diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java index cf26f8d284..4fafc5a088 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java @@ -1047,7 +1047,7 @@ public class GC { } private void deleteEmptyRefsFolders() throws IOException { - Path refs = repo.getDirectory().toPath().resolve(Constants.R_REFS); + Path refs = repo.getCommonDirectory().toPath().resolve(Constants.R_REFS); // Avoid deleting a folder that was created after the threshold so that concurrent // operations trying to create a reference are not impacted Instant threshold = Instant.now().minus(30, ChronoUnit.SECONDS); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GcLog.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GcLog.java index 628bf5db0c..8647b3e664 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GcLog.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GcLog.java @@ -50,7 +50,7 @@ class GcLog { */ GcLog(FileRepository repo) { this.repo = repo; - logFile = new File(repo.getDirectory(), "gc.log"); //$NON-NLS-1$ + logFile = new File(repo.getCommonDirectory(), "gc.log"); //$NON-NLS-1$ lock = new LockFile(logFile); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/InfoAttributesNode.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/InfoAttributesNode.java index 11d842b246..e8d442b8fb 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/InfoAttributesNode.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/InfoAttributesNode.java @@ -46,7 +46,7 @@ public class InfoAttributesNode extends AttributesNode { FS fs = repository.getFS(); - File attributes = fs.resolve(repository.getDirectory(), + File attributes = fs.resolve(repository.getCommonDirectory(), Constants.INFO_ATTRIBUTES); FileRepository.AttributesNodeProviderImpl.loadRulesFromFile(r, attributes); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java index 8e57bf9f2f..604868133e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java @@ -16,6 +16,7 @@ package org.eclipse.jgit.internal.storage.file; import static java.nio.charset.StandardCharsets.UTF_8; import static org.eclipse.jgit.lib.Constants.HEAD; import static org.eclipse.jgit.lib.Constants.LOGS; +import static org.eclipse.jgit.lib.Constants.L_LOGS; import static org.eclipse.jgit.lib.Constants.OBJECT_ID_STRING_LENGTH; import static org.eclipse.jgit.lib.Constants.PACKED_REFS; import static org.eclipse.jgit.lib.Constants.R_HEADS; @@ -124,6 +125,8 @@ public class RefDirectory extends RefDatabase { private final File gitDir; + private final File gitCommonDir; + final File refsDir; final File packedRefsFile; @@ -188,6 +191,7 @@ public class RefDirectory extends RefDatabase { RefDirectory(RefDirectory refDb) { parent = refDb.parent; gitDir = refDb.gitDir; + gitCommonDir = refDb.gitCommonDir; refsDir = refDb.refsDir; logsDir = refDb.logsDir; logsRefsDir = refDb.logsRefsDir; @@ -204,10 +208,11 @@ public class RefDirectory extends RefDatabase { final FS fs = db.getFS(); parent = db; gitDir = db.getDirectory(); - refsDir = fs.resolve(gitDir, R_REFS); - logsDir = fs.resolve(gitDir, LOGS); - logsRefsDir = fs.resolve(gitDir, LOGS + '/' + R_REFS); - packedRefsFile = fs.resolve(gitDir, PACKED_REFS); + gitCommonDir = db.getCommonDirectory(); + refsDir = fs.resolve(gitCommonDir, R_REFS); + logsDir = fs.resolve(gitCommonDir, LOGS); + logsRefsDir = fs.resolve(gitCommonDir, L_LOGS + R_REFS); + packedRefsFile = fs.resolve(gitCommonDir, PACKED_REFS); looseRefs.set(RefList. emptyList()); packedRefs.set(NO_PACKED_REFS); @@ -1329,7 +1334,12 @@ public class RefDirectory extends RefDatabase { name = name.substring(R_REFS.length()); return new File(refsDir, name); } - return new File(gitDir, name); + // HEAD needs to get resolved from git dir as resolving it from common dir + // would always lead back to current default branch + if (name.equals(HEAD)) { + return new File(gitDir, name); + } + return new File(gitCommonDir, name); } static int levelsIn(String name) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ReflogReaderImpl.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ReflogReaderImpl.java index 21b5a54eb7..f1888eb90f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ReflogReaderImpl.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ReflogReaderImpl.java @@ -10,6 +10,8 @@ package org.eclipse.jgit.internal.storage.file; +import static org.eclipse.jgit.lib.Constants.HEAD; + import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; @@ -37,7 +39,9 @@ class ReflogReaderImpl implements ReflogReader { * {@code Ref} name */ ReflogReaderImpl(Repository db, String refname) { - logName = new File(db.getDirectory(), Constants.LOGS + '/' + refname); + File logBaseDir = refname.equals(HEAD) ? db.getDirectory() + : db.getCommonDirectory(); + logName = new File(logBaseDir, Constants.L_LOGS + refname); } /* (non-Javadoc) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/BaseRepositoryBuilder.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/BaseRepositoryBuilder.java index 5dfb648faa..d232be6276 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/BaseRepositoryBuilder.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/BaseRepositoryBuilder.java @@ -13,13 +13,17 @@ package org.eclipse.jgit.lib; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_CORE_SECTION; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_BARE; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_WORKTREE; +import static org.eclipse.jgit.lib.Constants.CONFIG; import static org.eclipse.jgit.lib.Constants.DOT_GIT; import static org.eclipse.jgit.lib.Constants.GIT_ALTERNATE_OBJECT_DIRECTORIES_KEY; import static org.eclipse.jgit.lib.Constants.GIT_CEILING_DIRECTORIES_KEY; +import static org.eclipse.jgit.lib.Constants.GIT_COMMON_DIR_KEY; import static org.eclipse.jgit.lib.Constants.GIT_DIR_KEY; import static org.eclipse.jgit.lib.Constants.GIT_INDEX_FILE_KEY; import static org.eclipse.jgit.lib.Constants.GIT_OBJECT_DIRECTORY_KEY; import static org.eclipse.jgit.lib.Constants.GIT_WORK_TREE_KEY; +import static org.eclipse.jgit.lib.Constants.OBJECTS; +import static org.eclipse.jgit.lib.Constants.GITDIR_FILE; import java.io.File; import java.io.IOException; @@ -70,7 +74,21 @@ public class BaseRepositoryBuilder alternateObjectDirectories; @@ -171,6 +191,30 @@ public class BaseRepositoryBuilder * This method tries to read the standard Git environment variables, such as - * {@code GIT_DIR} and {@code GIT_WORK_TREE} to configure this builder - * instance. If an environment variable is set, it overrides the value - * already set in this builder. + * {@code GIT_DIR}, {@code GIT_COMMON_DIR}, {@code GIT_WORK_TREE} etc. to + * configure this builder instance. If an environment variable is set, it + * overrides the value already set in this builder. * * @return {@code this} (for chaining calls). */ @@ -410,9 +454,9 @@ public class BaseRepositoryBuilder * This method tries to read the standard Git environment variables, such as - * {@code GIT_DIR} and {@code GIT_WORK_TREE} to configure this builder - * instance. If a property is already set in the builder, the environment - * variable is not used. + * {@code GIT_DIR}, {@code GIT_COMMON_DIR}, {@code GIT_WORK_TREE} etc. to + * configure this builder instance. If a property is already set in the + * builder, the environment variable is not used. * * @param sr * the SystemReader abstraction to access the environment. @@ -425,6 +469,13 @@ public class BaseRepositoryBuilder @@ -695,8 +761,12 @@ public class BaseRepositoryBuilder Date: Mon, 15 Jul 2024 11:35:56 -0700 Subject: DfsBlockCacheTable: extract stats get* methods to interface Having the DfsBlockCacheTable methods extracted to an interface will allow alternative implementations of BlockCacheStats not tied to the current implementation. Change-Id: I534f7998f46253cdb7a68d5ec21d4f42ea586e8e --- .../internal/storage/dfs/ClockBlockCacheTable.java | 2 +- .../jgit/internal/storage/dfs/DfsBlockCache.java | 12 +-- .../internal/storage/dfs/DfsBlockCacheTable.java | 117 ++++++++++++--------- 3 files changed, 77 insertions(+), 54 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/ClockBlockCacheTable.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/ClockBlockCacheTable.java index d0907bcc8d..ce71a71d67 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/ClockBlockCacheTable.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/ClockBlockCacheTable.java @@ -135,7 +135,7 @@ final class ClockBlockCacheTable implements DfsBlockCacheTable { } @Override - public DfsBlockCacheStats getDfsBlockCacheStats() { + public BlockCacheStats getBlockCacheStats() { return dfsBlockCacheStats; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCache.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCache.java index 56719cf0f4..3e1300c867 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCache.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCache.java @@ -119,7 +119,7 @@ public final class DfsBlockCache { * @return total number of bytes in the cache, per pack file extension. */ public long[] getCurrentSize() { - return dfsBlockCacheTable.getDfsBlockCacheStats().getCurrentSize(); + return dfsBlockCacheTable.getBlockCacheStats().getCurrentSize(); } /** @@ -138,7 +138,7 @@ public final class DfsBlockCache { * extension. */ public long[] getHitCount() { - return dfsBlockCacheTable.getDfsBlockCacheStats().getHitCount(); + return dfsBlockCacheTable.getBlockCacheStats().getHitCount(); } /** @@ -149,7 +149,7 @@ public final class DfsBlockCache { * extension. */ public long[] getMissCount() { - return dfsBlockCacheTable.getDfsBlockCacheStats().getMissCount(); + return dfsBlockCacheTable.getBlockCacheStats().getMissCount(); } /** @@ -158,7 +158,7 @@ public final class DfsBlockCache { * @return total number of requests (hit + miss), per pack file extension. */ public long[] getTotalRequestCount() { - return dfsBlockCacheTable.getDfsBlockCacheStats() + return dfsBlockCacheTable.getBlockCacheStats() .getTotalRequestCount(); } @@ -168,7 +168,7 @@ public final class DfsBlockCache { * @return hit ratios */ public long[] getHitRatio() { - return dfsBlockCacheTable.getDfsBlockCacheStats().getHitRatio(); + return dfsBlockCacheTable.getBlockCacheStats().getHitRatio(); } /** @@ -179,7 +179,7 @@ public final class DfsBlockCache { * file extension. */ public long[] getEvictions() { - return dfsBlockCacheTable.getDfsBlockCacheStats().getEvictions(); + return dfsBlockCacheTable.getBlockCacheStats().getEvictions(); } /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheTable.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheTable.java index 701d1fdce3..309f2d1595 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheTable.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheTable.java @@ -129,18 +129,72 @@ public interface DfsBlockCacheTable { T get(DfsStreamKey key, long position); /** - * Get the DfsBlockCacheStats object for this block cache table's + * Get the {@link BlockCacheStats} object for this block cache table's * statistics. * - * @return the DfsBlockCacheStats tracking this block cache table's + * @return the {@link BlockCacheStats} tracking this block cache table's * statistics. */ - DfsBlockCacheStats getDfsBlockCacheStats(); + BlockCacheStats getBlockCacheStats(); + + /** + * Provides methods used with Block Cache statistics. + */ + interface BlockCacheStats { + /** + * Get total number of bytes in the cache, per pack file extension. + * + * @return total number of bytes in the cache, per pack file extension. + */ + long[] getCurrentSize(); + + /** + * Get number of requests for items in the cache, per pack file + * extension. + * + * @return the number of requests for items in the cache, per pack file + * extension. + */ + long[] getHitCount(); + + /** + * Get number of requests for items not in the cache, per pack file + * extension. + * + * @return the number of requests for items not in the cache, per pack + * file extension. + */ + long[] getMissCount(); + + /** + * Get total number of requests (hit + miss), per pack file extension. + * + * @return total number of requests (hit + miss), per pack file + * extension. + */ + long[] getTotalRequestCount(); + + /** + * Get hit ratios. + * + * @return hit ratios. + */ + long[] getHitRatio(); + + /** + * Get number of evictions performed due to cache being full, per pack + * file extension. + * + * @return the number of evictions performed due to cache being full, + * per pack file extension. + */ + long[] getEvictions(); + } /** * Keeps track of stats for a Block Cache table. */ - class DfsBlockCacheStats { + class DfsBlockCacheStats implements BlockCacheStats { /** * Number of times a block was found in the cache, per pack file * extension. @@ -214,44 +268,23 @@ public interface DfsBlockCacheTable { getStat(liveBytes, key).addAndGet(size); } - /** - * Get total number of bytes in the cache, per pack file extension. - * - * @return total number of bytes in the cache, per pack file extension. - */ - long[] getCurrentSize() { + @Override + public long[] getCurrentSize() { return getStatVals(liveBytes); } - /** - * Get number of requests for items in the cache, per pack file - * extension. - * - * @return the number of requests for items in the cache, per pack file - * extension. - */ - long[] getHitCount() { + @Override + public long[] getHitCount() { return getStatVals(statHit); } - /** - * Get number of requests for items not in the cache, per pack file - * extension. - * - * @return the number of requests for items not in the cache, per pack - * file extension. - */ - long[] getMissCount() { + @Override + public long[] getMissCount() { return getStatVals(statMiss); } - /** - * Get total number of requests (hit + miss), per pack file extension. - * - * @return total number of requests (hit + miss), per pack file - * extension. - */ - long[] getTotalRequestCount() { + @Override + public long[] getTotalRequestCount() { AtomicLong[] hit = statHit.get(); AtomicLong[] miss = statMiss.get(); long[] cnt = new long[Math.max(hit.length, miss.length)]; @@ -264,12 +297,8 @@ public interface DfsBlockCacheTable { return cnt; } - /** - * Get hit ratios. - * - * @return hit ratios. - */ - long[] getHitRatio() { + @Override + public long[] getHitRatio() { AtomicLong[] hit = statHit.get(); AtomicLong[] miss = statMiss.get(); long[] ratio = new long[Math.max(hit.length, miss.length)]; @@ -288,14 +317,8 @@ public interface DfsBlockCacheTable { return ratio; } - /** - * Get number of evictions performed due to cache being full, per pack - * file extension. - * - * @return the number of evictions performed due to cache being full, - * per pack file extension. - */ - long[] getEvictions() { + @Override + public long[] getEvictions() { return getStatVals(statEvict); } -- cgit v1.2.3 From 439512d7f82c503638e6b12689e751bb8debc015 Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Wed, 10 Apr 2024 10:52:44 -0700 Subject: DfsPackFile: Do not set local reverse index ref from cache callback The DfsBlockCache loading callback sets the local reference to the index in the DfsPackFile. This prevents abstracting the loading to implement it over multiple backends. Reorg the code so the loadReverseIndex do only the loading, the caller sets it into DfsBlockCache and the external code sets the local reference in DfsPackFile. This is the same pattern we did with the PackIndex in the parent commit. Change-Id: I3a395b347965fa7b0e5a3398c4da311dc11c58a1 --- .../jgit/internal/storage/dfs/DfsPackFile.java | 38 ++++++++++++---------- 1 file changed, 21 insertions(+), 17 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java index b94a84a41c..cdd10618e0 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java @@ -330,18 +330,27 @@ public final class DfsPackFile extends BlockBasedFile { PackIndex idx = idx(ctx); DfsStreamKey revKey = desc.getStreamKey(REVERSE_INDEX); - AtomicBoolean cacheHit = new AtomicBoolean(true); - DfsBlockCache.Ref revref = cache.getOrLoadRef(revKey, - REF_POSITION, () -> { - cacheHit.set(false); - return loadReverseIdx(ctx, revKey, idx); + // Keep the value parsed in the loader, in case the Ref<> is + // nullified in ClockBlockCacheTable#reserveSpace + // before we read its value. + AtomicReference loadedRef = new AtomicReference<>( + null); + DfsBlockCache.Ref cachedRef = cache + .getOrLoadRef(revKey, REF_POSITION, () -> { + RefWithSize ridx = loadReverseIdx(ctx, + idx); + loadedRef.set(ridx.ref); + return new DfsBlockCache.Ref<>(revKey, REF_POSITION, + ridx.size, ridx.ref); }); - if (cacheHit.get()) { + if (loadedRef.get() == null) { ctx.stats.ridxCacheHit++; } - PackReverseIndex revidx = revref.get(); - if (reverseIndex == null && revidx != null) { - reverseIndex = revidx; + reverseIndex = cachedRef.get() != null ? cachedRef.get() + : loadedRef.get(); + if (reverseIndex == null) { + throw new IOException( + "Couldn't get a reference to the reverse index"); //$NON-NLS-1$ } ctx.emitIndexLoad(desc, REVERSE_INDEX, reverseIndex); return reverseIndex; @@ -1241,18 +1250,13 @@ public final class DfsPackFile extends BlockBasedFile { } } - private DfsBlockCache.Ref loadReverseIdx( - DfsReader ctx, DfsStreamKey revKey, PackIndex idx) { + private static RefWithSize loadReverseIdx(DfsReader ctx, + PackIndex idx) { ctx.stats.readReverseIdx++; long start = System.nanoTime(); PackReverseIndex revidx = PackReverseIndexFactory.computeFromIndex(idx); - reverseIndex = revidx; ctx.stats.readReverseIdxMicros += elapsedMicros(start); - return new DfsBlockCache.Ref<>( - revKey, - REF_POSITION, - idx.getObjectCount() * 8, - revidx); + return new RefWithSize<>(revidx, idx.getObjectCount() * 8); } private DfsBlockCache.Ref loadObjectSizeIndex( -- cgit v1.2.3 From 1c9d1a49b0f9fcf7980775f22302df35293a3776 Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Fri, 19 Jul 2024 15:44:15 -0700 Subject: PackObjectSizeIndex: Read all bytes and use the byte[] directly The parser reads N integers one by one from the stream, assuming the InputStream does some ahead reading from storage. We see some very slow loading of indexes and suspect that this preemptive reading is not happening. The slow loading can be reproduced in clones, and it produces higher latencies and locks many threads waiting for the loading. Read the whole array from storage in one shot to avoid many small IO reads. Work directly on the resulting byte[], so there is no need of a second copy to cast to int/long. This is how other indexes, like primary or commit graph, work. Change-Id: I60058606e2c457f60aa4646a1f10ae7b28ce34c2 --- .../org/eclipse/jgit/internal/JGitText.properties | 1 + .../src/org/eclipse/jgit/internal/JGitText.java | 1 + .../storage/file/PackObjectSizeIndexV1.java | 181 ++++++++++++++------- 3 files changed, 122 insertions(+), 61 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties index 9d12facb33..8c02bbeeab 100644 --- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties +++ b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties @@ -806,6 +806,7 @@ truncatedHunkOldLinesMissing=Truncated hunk, at least {0} old lines is missing tSizeMustBeGreaterOrEqual1=tSize must be >= 1 unableToCheckConnectivity=Unable to check connectivity. unableToCreateNewObject=Unable to create new object: {0} +unableToReadFullArray=Unable to read an array with {0} elements from the stream unableToReadFullInt=Unable to read a full int from the stream unableToReadPackfile=Unable to read packfile {0} unableToRemovePath=Unable to remove path ''{0}'' diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java index 311d9c22aa..cbda506f99 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java @@ -836,6 +836,7 @@ public class JGitText extends TranslationBundle { /***/ public String unableToCheckConnectivity; /***/ public String unableToCreateNewObject; /***/ public String unableToReadFullInt; + /***/ public String unableToReadFullArray; /***/ public String unableToReadPackfile; /***/ public String unableToRemovePath; /***/ public String unableToWrite; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackObjectSizeIndexV1.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackObjectSizeIndexV1.java index a3d74be040..e172f141f5 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackObjectSizeIndexV1.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackObjectSizeIndexV1.java @@ -12,7 +12,7 @@ package org.eclipse.jgit.internal.storage.file; import java.io.IOException; import java.io.InputStream; import java.io.UnsupportedEncodingException; -import java.util.Arrays; +import java.text.MessageFormat; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.util.NB; @@ -35,7 +35,7 @@ class PackObjectSizeIndexV1 implements PackObjectSizeIndex { private final UInt24Array positions24; - private final int[] positions32; + private final IntArray positions32; /** * Parallel array to concat(positions24, positions32) with the size of the @@ -45,35 +45,37 @@ class PackObjectSizeIndexV1 implements PackObjectSizeIndex { * doesn't fit in an int and |value|-1 is the position for the size in the * size64 array e.g. a value of -1 is sizes64[0], -2 = sizes64[1], ... */ - private final int[] sizes32; + private final IntArray sizes32; - private final long[] sizes64; + private final LongArray sizes64; static PackObjectSizeIndex parse(InputStream in) throws IOException { /** Header and version already out of the input */ - IndexInputStreamReader stream = new IndexInputStreamReader(in); - int threshold = stream.readInt(); // minSize - int objCount = stream.readInt(); + byte[] buffer = new byte[8]; + in.readNBytes(buffer, 0, 8); + int threshold = NB.decodeInt32(buffer, 0); // minSize + int objCount = NB.decodeInt32(buffer, 4); if (objCount == 0) { return new EmptyPackObjectSizeIndex(threshold); } - return new PackObjectSizeIndexV1(stream, threshold, objCount); + return new PackObjectSizeIndexV1(in, threshold, objCount); } - private PackObjectSizeIndexV1(IndexInputStreamReader stream, int threshold, + private PackObjectSizeIndexV1(InputStream stream, int threshold, int objCount) throws IOException { this.threshold = threshold; UInt24Array pos24 = null; - int[] pos32 = null; + IntArray pos32 = null; + StreamHelper helper = new StreamHelper(); byte positionEncoding; - while ((positionEncoding = stream.readByte()) != 0) { + while ((positionEncoding = helper.readByte(stream)) != 0) { if (Byte.compareUnsigned(positionEncoding, BITS_24) == 0) { - int sz = stream.readInt(); + int sz = helper.readInt(stream); pos24 = new UInt24Array(stream.readNBytes(sz * 3)); } else if (Byte.compareUnsigned(positionEncoding, BITS_32) == 0) { - int sz = stream.readInt(); - pos32 = stream.readIntArray(sz); + int sz = helper.readInt(stream); + pos32 = IntArray.from(stream, sz); } else { throw new UnsupportedEncodingException( String.format(JGitText.get().unknownPositionEncoding, @@ -81,16 +83,16 @@ class PackObjectSizeIndexV1 implements PackObjectSizeIndex { } } positions24 = pos24 != null ? pos24 : UInt24Array.EMPTY; - positions32 = pos32 != null ? pos32 : new int[0]; + positions32 = pos32 != null ? pos32 : IntArray.EMPTY; - sizes32 = stream.readIntArray(objCount); - int c64sizes = stream.readInt(); + sizes32 = IntArray.from(stream, objCount); + int c64sizes = helper.readInt(stream); if (c64sizes == 0) { - sizes64 = new long[0]; + sizes64 = LongArray.EMPTY; return; } - sizes64 = stream.readLongArray(c64sizes); - int c128sizes = stream.readInt(); + sizes64 = LongArray.from(stream, c64sizes); + int c128sizes = helper.readInt(stream); if (c128sizes != 0) { // this MUST be 0 (we don't support 128 bits sizes yet) throw new IOException(JGitText.get().unsupportedSizesObjSizeIndex); @@ -102,8 +104,8 @@ class PackObjectSizeIndexV1 implements PackObjectSizeIndex { int pos = -1; if (!positions24.isEmpty() && idxOffset <= positions24.getLastValue()) { pos = positions24.binarySearch(idxOffset); - } else if (positions32.length > 0 && idxOffset >= positions32[0]) { - int pos32 = Arrays.binarySearch(positions32, idxOffset); + } else if (!positions32.empty() && idxOffset >= positions32.get(0)) { + int pos32 = positions32.binarySearch(idxOffset); if (pos32 >= 0) { pos = pos32 + positions24.size(); } @@ -112,17 +114,17 @@ class PackObjectSizeIndexV1 implements PackObjectSizeIndex { return -1; } - int objSize = sizes32[pos]; + int objSize = sizes32.get(pos); if (objSize < 0) { int secondPos = Math.abs(objSize) - 1; - return sizes64[secondPos]; + return sizes64.get(secondPos); } return objSize; } @Override public long getObjectCount() { - return (long) positions24.size() + positions32.length; + return (long) positions24.size() + positions32.size(); } @Override @@ -131,69 +133,126 @@ class PackObjectSizeIndexV1 implements PackObjectSizeIndex { } /** - * Wrapper to read parsed content from the byte stream + * A byte[] that should be interpreted as an int[] */ - private static class IndexInputStreamReader { + private static class IntArray { + private static final IntArray EMPTY = new IntArray(new byte[0]); - private final byte[] buffer = new byte[8]; + private static final int INT_SIZE = 4; - private final InputStream in; + private final byte[] data; - IndexInputStreamReader(InputStream in) { - this.in = in; - } + private final int size; - int readInt() throws IOException { - int n = in.readNBytes(buffer, 0, 4); - if (n < 4) { - throw new IOException(JGitText.get().unableToReadFullInt); + static IntArray from(InputStream in, int ints) throws IOException { + int expectedBytes = ints * INT_SIZE; + byte[] data = in.readNBytes(expectedBytes); + if (data.length < expectedBytes) { + throw new IOException(MessageFormat + .format(JGitText.get().unableToReadFullArray, ints)); } - return NB.decodeInt32(buffer, 0); + return new IntArray(data); + } + + private IntArray(byte[] data) { + this.data = data; + size = data.length / INT_SIZE; } - int[] readIntArray(int intsCount) throws IOException { - if (intsCount == 0) { - return new int[0]; + /** + * Returns position of element in array, -1 if not there + * + * @param needle + * element to look for + * @return position of the element in the array or -1 if not found + */ + int binarySearch(int needle) { + if (size == 0) { + return -1; } + int high = size; + int low = 0; + do { + int mid = (low + high) >>> 1; + int cmp = Integer.compare(needle, get(mid)); + if (cmp < 0) + high = mid; + else if (cmp == 0) { + return mid; + } else + low = mid + 1; + } while (low < high); + return -1; + } - int[] dest = new int[intsCount]; - for (int i = 0; i < intsCount; i++) { - dest[i] = readInt(); + int get(int position) { + if (position < 0 || position >= size) { + throw new IndexOutOfBoundsException(position); } - return dest; + return NB.decodeInt32(data, position * INT_SIZE); } - long readLong() throws IOException { - int n = in.readNBytes(buffer, 0, 8); - if (n < 8) { - throw new IOException(JGitText.get().unableToReadFullInt); + boolean empty() { + return size == 0; + } + + int size() { + return size; + } + } + + /** + * A byte[] that should be interpreted as an long[] + */ + private static class LongArray { + private static final LongArray EMPTY = new LongArray(new byte[0]); + + private static final int LONG_SIZE = 8; // bytes + + private final byte[] data; + + private final int size; + + static LongArray from(InputStream in, int longs) throws IOException { + byte[] data = in.readNBytes(longs * LONG_SIZE); + if (data.length < longs * LONG_SIZE) { + throw new IOException(MessageFormat + .format(JGitText.get().unableToReadFullArray, longs)); } - return NB.decodeInt64(buffer, 0); + return new LongArray(data); } - long[] readLongArray(int longsCount) throws IOException { - if (longsCount == 0) { - return new long[0]; + private LongArray(byte[] data) { + this.data = data; + size = data.length / LONG_SIZE; + } + + long get(int position) { + if (position < 0 || position >= size) { + throw new IndexOutOfBoundsException(position); } + return NB.decodeInt64(data, position * LONG_SIZE); + } + } - long[] dest = new long[longsCount]; - for (int i = 0; i < longsCount; i++) { - dest[i] = readLong(); + private static class StreamHelper { + private final byte[] buffer = new byte[8]; + + int readInt(InputStream in) throws IOException { + int n = in.readNBytes(buffer, 0, 4); + if (n < 4) { + throw new IOException(JGitText.get().unableToReadFullInt); } - return dest; + return NB.decodeInt32(buffer, 0); } - byte readByte() throws IOException { + byte readByte(InputStream in) throws IOException { int n = in.readNBytes(buffer, 0, 1); if (n != 1) { throw new IOException(JGitText.get().cannotReadByte); } return buffer[0]; } - - byte[] readNBytes(int sz) throws IOException { - return in.readNBytes(sz); - } } private static class EmptyPackObjectSizeIndex -- cgit v1.2.3 From a2a5cdddd78a1e066aed7caf86ba9510450710c9 Mon Sep 17 00:00:00 2001 From: Laura Hamelin Date: Fri, 7 Jun 2024 16:11:21 -0700 Subject: PackExtBlockCacheTable: spread extensions over multiple dfs tables The existing DfsBlockCache uses a single table for all extensions (idx, ridx, ...). This change introduces an implementation of the table interface that can keep extensions in different cache tables. This selects the appropriate cache to use for a specific PackExt or DfsStreamKey's PackExt type, allowing the separation of entries from different pack types to help limit churn in cache caused by entries of differing sizes. This is especially useful in fine-tuning caches and influencing interactions by extension type. For example, a table holding INDEX types only will not influence evictions of other PackExt types and vice versa. The PackExtBlockCacheTable allowing setting the underlying DfsBlockCacheTables and mappinh directly, letting users implement and use custom DfsBlockCacheTables. Change-Id: Icee7b644ef6b600aa473d35645469d6aa1bce345 --- .../storage/dfs/PackExtBlockCacheTableTest.java | 588 +++++++++++++++++++++ .../org/eclipse/jgit/internal/JGitText.properties | 2 + .../src/org/eclipse/jgit/internal/JGitText.java | 2 + .../internal/storage/dfs/DfsBlockCacheConfig.java | 26 +- .../storage/dfs/PackExtBlockCacheTable.java | 289 ++++++++++ 5 files changed, 905 insertions(+), 2 deletions(-) create mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/PackExtBlockCacheTableTest.java create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/PackExtBlockCacheTable.java (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/PackExtBlockCacheTableTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/PackExtBlockCacheTableTest.java new file mode 100644 index 0000000000..d506bfba40 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/PackExtBlockCacheTableTest.java @@ -0,0 +1,588 @@ +/* + * Copyright (c) 2024, Google LLC and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.storage.dfs; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.sameInstance; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.when; + +import java.util.EnumSet; +import java.util.List; +import java.util.Map; + +import org.eclipse.jgit.internal.storage.dfs.DfsBlockCache.Ref; +import org.eclipse.jgit.internal.storage.dfs.DfsBlockCache.RefLoader; +import org.eclipse.jgit.internal.storage.dfs.DfsBlockCacheConfig.DfsBlockCachePackExtConfig; +import org.eclipse.jgit.internal.storage.dfs.DfsBlockCacheTable.DfsBlockCacheStats; +import org.eclipse.jgit.internal.storage.pack.PackExt; +import org.junit.Test; +import org.mockito.Mockito; + +public class PackExtBlockCacheTableTest { + @Test + public void fromBlockCacheConfigs_createsDfsPackExtBlockCacheTables() { + DfsBlockCacheConfig cacheConfig = new DfsBlockCacheConfig(); + cacheConfig.setPackExtCacheConfigurations( + List.of(new DfsBlockCachePackExtConfig(EnumSet.of(PackExt.PACK), + new DfsBlockCacheConfig()))); + assertNotNull( + PackExtBlockCacheTable.fromBlockCacheConfigs(cacheConfig)); + } + + @Test + public void fromBlockCacheConfigs_noPackExtConfigurationGiven_packExtCacheConfigurationsIsEmpty_throws() { + DfsBlockCacheConfig cacheConfig = new DfsBlockCacheConfig(); + cacheConfig.setPackExtCacheConfigurations(List.of()); + assertThrows(IllegalArgumentException.class, + () -> PackExtBlockCacheTable + .fromBlockCacheConfigs(cacheConfig)); + } + + @Test + public void hasBlock0_packExtMapsToCacheTable_callsBitmapIndexCacheTable() { + DfsStreamKey streamKey = new TestKey(PackExt.BITMAP_INDEX); + DfsBlockCacheTable defaultBlockCacheTable = mock( + DfsBlockCacheTable.class); + DfsBlockCacheTable bitmapIndexCacheTable = mock( + DfsBlockCacheTable.class); + when(bitmapIndexCacheTable.hasBlock0(any(DfsStreamKey.class))) + .thenReturn(true); + + PackExtBlockCacheTable tables = PackExtBlockCacheTable.fromCacheTables( + defaultBlockCacheTable, + Map.of(PackExt.BITMAP_INDEX, bitmapIndexCacheTable)); + + assertTrue(tables.hasBlock0(streamKey)); + } + + @Test + public void hasBlock0_packExtDoesNotMapToCacheTable_callsDefaultCache() { + DfsStreamKey streamKey = new TestKey(PackExt.PACK); + DfsBlockCacheTable defaultBlockCacheTable = mock( + DfsBlockCacheTable.class); + when(defaultBlockCacheTable.hasBlock0(any(DfsStreamKey.class))) + .thenReturn(true); + DfsBlockCacheTable bitmapIndexCacheTable = mock( + DfsBlockCacheTable.class); + + PackExtBlockCacheTable tables = PackExtBlockCacheTable.fromCacheTables( + defaultBlockCacheTable, + Map.of(PackExt.BITMAP_INDEX, bitmapIndexCacheTable)); + + assertTrue(tables.hasBlock0(streamKey)); + } + + @Test + public void getOrLoad_packExtMapsToCacheTable_callsBitmapIndexCacheTable() + throws Exception { + BlockBasedFile blockBasedFile = new BlockBasedFile(null, + mock(DfsPackDescription.class), PackExt.BITMAP_INDEX) { + }; + DfsBlock dfsBlock = mock(DfsBlock.class); + DfsBlockCacheTable defaultBlockCacheTable = mock( + DfsBlockCacheTable.class); + when(defaultBlockCacheTable.getOrLoad(any(BlockBasedFile.class), + anyLong(), any(DfsReader.class), + any(DfsBlockCache.ReadableChannelSupplier.class))) + .thenReturn(mock(DfsBlock.class)); + DfsBlockCacheTable bitmapIndexCacheTable = mock( + DfsBlockCacheTable.class); + when(bitmapIndexCacheTable.getOrLoad(any(BlockBasedFile.class), + anyLong(), any(DfsReader.class), + any(DfsBlockCache.ReadableChannelSupplier.class))) + .thenReturn(dfsBlock); + + PackExtBlockCacheTable tables = PackExtBlockCacheTable.fromCacheTables( + defaultBlockCacheTable, + Map.of(PackExt.BITMAP_INDEX, bitmapIndexCacheTable)); + + assertThat( + tables.getOrLoad(blockBasedFile, 0, mock(DfsReader.class), + mock(DfsBlockCache.ReadableChannelSupplier.class)), + sameInstance(dfsBlock)); + } + + @Test + public void getOrLoad_packExtDoesNotMapToCacheTable_callsDefaultCache() + throws Exception { + BlockBasedFile blockBasedFile = new BlockBasedFile(null, + mock(DfsPackDescription.class), PackExt.PACK) { + }; + DfsBlock dfsBlock = mock(DfsBlock.class); + DfsBlockCacheTable defaultBlockCacheTable = mock( + DfsBlockCacheTable.class); + when(defaultBlockCacheTable.getOrLoad(any(BlockBasedFile.class), + anyLong(), any(DfsReader.class), + any(DfsBlockCache.ReadableChannelSupplier.class))) + .thenReturn(dfsBlock); + DfsBlockCacheTable bitmapIndexCacheTable = mock( + DfsBlockCacheTable.class); + when(bitmapIndexCacheTable.getOrLoad(any(BlockBasedFile.class), + anyLong(), any(DfsReader.class), + any(DfsBlockCache.ReadableChannelSupplier.class))) + .thenReturn(mock(DfsBlock.class)); + + PackExtBlockCacheTable tables = PackExtBlockCacheTable.fromCacheTables( + defaultBlockCacheTable, + Map.of(PackExt.BITMAP_INDEX, bitmapIndexCacheTable)); + + assertThat( + tables.getOrLoad(blockBasedFile, 0, mock(DfsReader.class), + mock(DfsBlockCache.ReadableChannelSupplier.class)), + sameInstance(dfsBlock)); + } + + @Test + public void getOrLoadRef_packExtMapsToCacheTable_callsBitmapIndexCacheTable() + throws Exception { + Ref ref = mock(Ref.class); + DfsStreamKey dfsStreamKey = new TestKey(PackExt.BITMAP_INDEX); + DfsBlockCacheTable defaultBlockCacheTable = mock( + DfsBlockCacheTable.class); + when(defaultBlockCacheTable.getOrLoadRef(any(DfsStreamKey.class), + anyLong(), any(RefLoader.class))).thenReturn(mock(Ref.class)); + DfsBlockCacheTable bitmapIndexCacheTable = mock( + DfsBlockCacheTable.class); + when(bitmapIndexCacheTable.getOrLoadRef(any(DfsStreamKey.class), + anyLong(), any(RefLoader.class))).thenReturn(ref); + + PackExtBlockCacheTable tables = PackExtBlockCacheTable.fromCacheTables( + defaultBlockCacheTable, + Map.of(PackExt.BITMAP_INDEX, bitmapIndexCacheTable)); + + assertThat(tables.getOrLoadRef(dfsStreamKey, 0, mock(RefLoader.class)), + sameInstance(ref)); + } + + @Test + public void getOrLoadRef_packExtDoesNotMapToCacheTable_callsDefaultCache() + throws Exception { + Ref ref = mock(Ref.class); + DfsStreamKey dfsStreamKey = new TestKey(PackExt.PACK); + DfsBlockCacheTable defaultBlockCacheTable = mock( + DfsBlockCacheTable.class); + when(defaultBlockCacheTable.getOrLoadRef(any(DfsStreamKey.class), + anyLong(), any(RefLoader.class))).thenReturn(ref); + DfsBlockCacheTable bitmapIndexCacheTable = mock( + DfsBlockCacheTable.class); + when(bitmapIndexCacheTable.getOrLoadRef(any(DfsStreamKey.class), + anyLong(), any(RefLoader.class))).thenReturn(mock(Ref.class)); + + PackExtBlockCacheTable tables = PackExtBlockCacheTable.fromCacheTables( + defaultBlockCacheTable, + Map.of(PackExt.BITMAP_INDEX, bitmapIndexCacheTable)); + + assertThat(tables.getOrLoadRef(dfsStreamKey, 0, mock(RefLoader.class)), + sameInstance(ref)); + } + + @Test + public void putDfsBlock_packExtMapsToCacheTable_callsBitmapIndexCacheTable() { + DfsStreamKey dfsStreamKey = new TestKey(PackExt.BITMAP_INDEX); + DfsBlock dfsBlock = new DfsBlock(dfsStreamKey, 0, new byte[0]); + DfsBlockCacheTable defaultBlockCacheTable = mock( + DfsBlockCacheTable.class); + DfsBlockCacheTable bitmapIndexCacheTable = mock( + DfsBlockCacheTable.class); + + PackExtBlockCacheTable tables = PackExtBlockCacheTable.fromCacheTables( + defaultBlockCacheTable, + Map.of(PackExt.BITMAP_INDEX, bitmapIndexCacheTable)); + + tables.put(dfsBlock); + Mockito.verify(bitmapIndexCacheTable, times(1)).put(dfsBlock); + } + + @Test + public void putDfsBlock_packExtDoesNotMapToCacheTable_callsDefaultCache() { + DfsStreamKey dfsStreamKey = new TestKey(PackExt.PACK); + DfsBlock dfsBlock = new DfsBlock(dfsStreamKey, 0, new byte[0]); + DfsBlockCacheTable defaultBlockCacheTable = mock( + DfsBlockCacheTable.class); + DfsBlockCacheTable bitmapIndexCacheTable = mock( + DfsBlockCacheTable.class); + + PackExtBlockCacheTable tables = PackExtBlockCacheTable.fromCacheTables( + defaultBlockCacheTable, + Map.of(PackExt.BITMAP_INDEX, bitmapIndexCacheTable)); + + tables.put(dfsBlock); + Mockito.verify(defaultBlockCacheTable, times(1)).put(dfsBlock); + } + + @Test + public void putDfsStreamKey_packExtMapsToCacheTable_callsBitmapIndexCacheTable() { + DfsStreamKey dfsStreamKey = new TestKey(PackExt.BITMAP_INDEX); + Ref ref = mock(Ref.class); + DfsBlockCacheTable defaultBlockCacheTable = mock( + DfsBlockCacheTable.class); + when(defaultBlockCacheTable.put(any(DfsStreamKey.class), anyLong(), + anyLong(), anyInt())).thenReturn(mock(Ref.class)); + DfsBlockCacheTable bitmapIndexCacheTable = mock( + DfsBlockCacheTable.class); + when(bitmapIndexCacheTable.put(any(DfsStreamKey.class), anyLong(), + anyLong(), anyInt())).thenReturn(ref); + + PackExtBlockCacheTable tables = PackExtBlockCacheTable.fromCacheTables( + defaultBlockCacheTable, + Map.of(PackExt.BITMAP_INDEX, bitmapIndexCacheTable)); + + assertThat(tables.put(dfsStreamKey, 0, 0, 0), sameInstance(ref)); + } + + @Test + public void putDfsStreamKey_packExtDoesNotMapToCacheTable_callsDefaultCache() { + DfsStreamKey dfsStreamKey = new TestKey(PackExt.PACK); + Ref ref = mock(Ref.class); + DfsBlockCacheTable defaultBlockCacheTable = mock( + DfsBlockCacheTable.class); + when(defaultBlockCacheTable.put(any(DfsStreamKey.class), anyLong(), + anyLong(), anyInt())).thenReturn(ref); + DfsBlockCacheTable bitmapIndexCacheTable = mock( + DfsBlockCacheTable.class); + when(bitmapIndexCacheTable.put(any(DfsStreamKey.class), anyLong(), + anyLong(), anyInt())).thenReturn(mock(Ref.class)); + + PackExtBlockCacheTable tables = PackExtBlockCacheTable.fromCacheTables( + defaultBlockCacheTable, + Map.of(PackExt.BITMAP_INDEX, bitmapIndexCacheTable)); + + assertThat(tables.put(dfsStreamKey, 0, 0, 0), sameInstance(ref)); + } + + @Test + public void putRef_packExtMapsToCacheTable_callsBitmapIndexCacheTable() { + DfsStreamKey dfsStreamKey = new TestKey(PackExt.BITMAP_INDEX); + Ref ref = mock(Ref.class); + DfsBlockCacheTable defaultBlockCacheTable = mock( + DfsBlockCacheTable.class); + when(defaultBlockCacheTable.putRef(any(DfsStreamKey.class), anyLong(), + anyInt())).thenReturn(mock(Ref.class)); + DfsBlockCacheTable bitmapIndexCacheTable = mock( + DfsBlockCacheTable.class); + when(bitmapIndexCacheTable.putRef(any(DfsStreamKey.class), anyLong(), + anyInt())).thenReturn(ref); + + PackExtBlockCacheTable tables = PackExtBlockCacheTable.fromCacheTables( + defaultBlockCacheTable, + Map.of(PackExt.BITMAP_INDEX, bitmapIndexCacheTable)); + + assertThat(tables.putRef(dfsStreamKey, 0, 0), sameInstance(ref)); + } + + @Test + public void putRef_packExtDoesNotMapToCacheTable_callsDefaultCache() { + DfsStreamKey dfsStreamKey = new TestKey(PackExt.PACK); + Ref ref = mock(Ref.class); + DfsBlockCacheTable defaultBlockCacheTable = mock( + DfsBlockCacheTable.class); + when(defaultBlockCacheTable.putRef(any(DfsStreamKey.class), anyLong(), + anyInt())).thenReturn(ref); + DfsBlockCacheTable bitmapIndexCacheTable = mock( + DfsBlockCacheTable.class); + when(bitmapIndexCacheTable.putRef(any(DfsStreamKey.class), anyLong(), + anyInt())).thenReturn(mock(Ref.class)); + + PackExtBlockCacheTable tables = PackExtBlockCacheTable.fromCacheTables( + defaultBlockCacheTable, + Map.of(PackExt.BITMAP_INDEX, bitmapIndexCacheTable)); + + assertThat(tables.putRef(dfsStreamKey, 0, 0), sameInstance(ref)); + } + + @Test + public void contains_packExtMapsToCacheTable_callsBitmapIndexCacheTable() { + DfsStreamKey streamKey = new TestKey(PackExt.BITMAP_INDEX); + DfsBlockCacheTable defaultBlockCacheTable = mock( + DfsBlockCacheTable.class); + DfsBlockCacheTable bitmapIndexCacheTable = mock( + DfsBlockCacheTable.class); + when(bitmapIndexCacheTable.contains(any(DfsStreamKey.class), anyLong())) + .thenReturn(true); + + PackExtBlockCacheTable tables = PackExtBlockCacheTable.fromCacheTables( + defaultBlockCacheTable, + Map.of(PackExt.BITMAP_INDEX, bitmapIndexCacheTable)); + + assertTrue(tables.contains(streamKey, 0)); + } + + @Test + public void contains_packExtDoesNotMapToCacheTable_callsDefaultCache() { + DfsStreamKey streamKey = new TestKey(PackExt.PACK); + DfsBlockCacheTable defaultBlockCacheTable = mock( + DfsBlockCacheTable.class); + when(defaultBlockCacheTable.contains(any(DfsStreamKey.class), + anyLong())).thenReturn(true); + DfsBlockCacheTable bitmapIndexCacheTable = mock( + DfsBlockCacheTable.class); + + PackExtBlockCacheTable tables = PackExtBlockCacheTable.fromCacheTables( + defaultBlockCacheTable, + Map.of(PackExt.BITMAP_INDEX, bitmapIndexCacheTable)); + + assertTrue(tables.contains(streamKey, 0)); + } + + @Test + public void get_packExtMapsToCacheTable_callsBitmapIndexCacheTable() { + DfsStreamKey dfsStreamKey = new TestKey(PackExt.BITMAP_INDEX); + Ref ref = mock(Ref.class); + DfsBlockCacheTable defaultBlockCacheTable = mock( + DfsBlockCacheTable.class); + when(defaultBlockCacheTable.get(any(DfsStreamKey.class), anyLong())) + .thenReturn(mock(Ref.class)); + DfsBlockCacheTable bitmapIndexCacheTable = mock( + DfsBlockCacheTable.class); + when(bitmapIndexCacheTable.get(any(DfsStreamKey.class), anyLong())) + .thenReturn(ref); + + PackExtBlockCacheTable tables = PackExtBlockCacheTable.fromCacheTables( + defaultBlockCacheTable, + Map.of(PackExt.BITMAP_INDEX, bitmapIndexCacheTable)); + + assertThat(tables.get(dfsStreamKey, 0), sameInstance(ref)); + } + + @Test + public void get_packExtDoesNotMapToCacheTable_callsDefaultCache() { + DfsStreamKey dfsStreamKey = new TestKey(PackExt.PACK); + Ref ref = mock(Ref.class); + DfsBlockCacheTable defaultBlockCacheTable = mock( + DfsBlockCacheTable.class); + when(defaultBlockCacheTable.get(any(DfsStreamKey.class), anyLong())) + .thenReturn(ref); + DfsBlockCacheTable bitmapIndexCacheTable = mock( + DfsBlockCacheTable.class); + when(bitmapIndexCacheTable.get(any(DfsStreamKey.class), anyLong())) + .thenReturn(mock(Ref.class)); + + PackExtBlockCacheTable tables = PackExtBlockCacheTable.fromCacheTables( + defaultBlockCacheTable, + Map.of(PackExt.BITMAP_INDEX, bitmapIndexCacheTable)); + + assertThat(tables.get(dfsStreamKey, 0), sameInstance(ref)); + } + + @Test + public void getBlockCacheStats_getCurrentSize_consolidatesAllTableCurrentSizes() { + long[] currentSizes = createEmptyStatsArray(); + + DfsBlockCacheStats packStats = new DfsBlockCacheStats(); + packStats.addToLiveBytes(new TestKey(PackExt.PACK), 5); + currentSizes[PackExt.PACK.getPosition()] = 5; + + DfsBlockCacheStats bitmapStats = new DfsBlockCacheStats(); + bitmapStats.addToLiveBytes(new TestKey(PackExt.BITMAP_INDEX), 6); + currentSizes[PackExt.BITMAP_INDEX.getPosition()] = 6; + + DfsBlockCacheStats indexStats = new DfsBlockCacheStats(); + indexStats.addToLiveBytes(new TestKey(PackExt.INDEX), 7); + currentSizes[PackExt.INDEX.getPosition()] = 7; + + PackExtBlockCacheTable tables = PackExtBlockCacheTable + .fromCacheTables(cacheTableWithStats(packStats), + Map.of(PackExt.BITMAP_INDEX, + cacheTableWithStats(bitmapStats), PackExt.INDEX, + cacheTableWithStats(indexStats))); + + assertArrayEquals(tables.getBlockCacheStats().getCurrentSize(), + currentSizes); + } + + @Test + public void getBlockCacheStats_GetHitCount_consolidatesAllTableHitCounts() { + long[] hitCounts = createEmptyStatsArray(); + + DfsBlockCacheStats packStats = new DfsBlockCacheStats(); + incrementCounter(5, + () -> packStats.incrementHit(new TestKey(PackExt.PACK))); + hitCounts[PackExt.PACK.getPosition()] = 5; + + DfsBlockCacheStats bitmapStats = new DfsBlockCacheStats(); + incrementCounter(6, () -> bitmapStats + .incrementHit(new TestKey(PackExt.BITMAP_INDEX))); + hitCounts[PackExt.BITMAP_INDEX.getPosition()] = 6; + + DfsBlockCacheStats indexStats = new DfsBlockCacheStats(); + incrementCounter(7, + () -> indexStats.incrementHit(new TestKey(PackExt.INDEX))); + hitCounts[PackExt.INDEX.getPosition()] = 7; + + PackExtBlockCacheTable tables = PackExtBlockCacheTable + .fromCacheTables(cacheTableWithStats(packStats), + Map.of(PackExt.BITMAP_INDEX, + cacheTableWithStats(bitmapStats), PackExt.INDEX, + cacheTableWithStats(indexStats))); + + assertArrayEquals(tables.getBlockCacheStats().getHitCount(), hitCounts); + } + + @Test + public void getBlockCacheStats_getMissCount_consolidatesAllTableMissCounts() { + long[] missCounts = createEmptyStatsArray(); + + DfsBlockCacheStats packStats = new DfsBlockCacheStats(); + incrementCounter(5, + () -> packStats.incrementMiss(new TestKey(PackExt.PACK))); + missCounts[PackExt.PACK.getPosition()] = 5; + + DfsBlockCacheStats bitmapStats = new DfsBlockCacheStats(); + incrementCounter(6, () -> bitmapStats + .incrementMiss(new TestKey(PackExt.BITMAP_INDEX))); + missCounts[PackExt.BITMAP_INDEX.getPosition()] = 6; + + DfsBlockCacheStats indexStats = new DfsBlockCacheStats(); + incrementCounter(7, + () -> indexStats.incrementMiss(new TestKey(PackExt.INDEX))); + missCounts[PackExt.INDEX.getPosition()] = 7; + + PackExtBlockCacheTable tables = PackExtBlockCacheTable + .fromCacheTables(cacheTableWithStats(packStats), + Map.of(PackExt.BITMAP_INDEX, + cacheTableWithStats(bitmapStats), PackExt.INDEX, + cacheTableWithStats(indexStats))); + + assertArrayEquals(tables.getBlockCacheStats().getMissCount(), + missCounts); + } + + @Test + public void getBlockCacheStats_getTotalRequestCount_consolidatesAllTableTotalRequestCounts() { + long[] totalRequestCounts = createEmptyStatsArray(); + + DfsBlockCacheStats packStats = new DfsBlockCacheStats(); + incrementCounter(5, () -> { + packStats.incrementHit(new TestKey(PackExt.PACK)); + packStats.incrementMiss(new TestKey(PackExt.PACK)); + }); + totalRequestCounts[PackExt.PACK.getPosition()] = 10; + + DfsBlockCacheStats bitmapStats = new DfsBlockCacheStats(); + incrementCounter(6, () -> { + bitmapStats.incrementHit(new TestKey(PackExt.BITMAP_INDEX)); + bitmapStats.incrementMiss(new TestKey(PackExt.BITMAP_INDEX)); + }); + totalRequestCounts[PackExt.BITMAP_INDEX.getPosition()] = 12; + + DfsBlockCacheStats indexStats = new DfsBlockCacheStats(); + incrementCounter(7, () -> { + indexStats.incrementHit(new TestKey(PackExt.INDEX)); + indexStats.incrementMiss(new TestKey(PackExt.INDEX)); + }); + totalRequestCounts[PackExt.INDEX.getPosition()] = 14; + + PackExtBlockCacheTable tables = PackExtBlockCacheTable + .fromCacheTables(cacheTableWithStats(packStats), + Map.of(PackExt.BITMAP_INDEX, + cacheTableWithStats(bitmapStats), PackExt.INDEX, + cacheTableWithStats(indexStats))); + + assertArrayEquals(tables.getBlockCacheStats().getTotalRequestCount(), + totalRequestCounts); + } + + @Test + public void getBlockCacheStats_getHitRatio_consolidatesAllTableHitRatios() { + long[] hitRatios = createEmptyStatsArray(); + + DfsBlockCacheStats packStats = new DfsBlockCacheStats(); + incrementCounter(5, + () -> packStats.incrementHit(new TestKey(PackExt.PACK))); + hitRatios[PackExt.PACK.getPosition()] = 100; + + DfsBlockCacheStats bitmapStats = new DfsBlockCacheStats(); + incrementCounter(6, () -> { + bitmapStats.incrementHit(new TestKey(PackExt.BITMAP_INDEX)); + bitmapStats.incrementMiss(new TestKey(PackExt.BITMAP_INDEX)); + }); + hitRatios[PackExt.BITMAP_INDEX.getPosition()] = 50; + + DfsBlockCacheStats indexStats = new DfsBlockCacheStats(); + incrementCounter(7, + () -> indexStats.incrementMiss(new TestKey(PackExt.INDEX))); + hitRatios[PackExt.INDEX.getPosition()] = 0; + + PackExtBlockCacheTable tables = PackExtBlockCacheTable + .fromCacheTables(cacheTableWithStats(packStats), + Map.of(PackExt.BITMAP_INDEX, + cacheTableWithStats(bitmapStats), PackExt.INDEX, + cacheTableWithStats(indexStats))); + + assertArrayEquals(tables.getBlockCacheStats().getHitRatio(), hitRatios); + } + + @Test + public void getBlockCacheStats_getEvictions_consolidatesAllTableEvictions() { + long[] evictions = createEmptyStatsArray(); + + DfsBlockCacheStats packStats = new DfsBlockCacheStats(); + incrementCounter(5, + () -> packStats.incrementEvict(new TestKey(PackExt.PACK))); + evictions[PackExt.PACK.getPosition()] = 5; + + DfsBlockCacheStats bitmapStats = new DfsBlockCacheStats(); + incrementCounter(6, () -> bitmapStats + .incrementEvict(new TestKey(PackExt.BITMAP_INDEX))); + evictions[PackExt.BITMAP_INDEX.getPosition()] = 6; + + DfsBlockCacheStats indexStats = new DfsBlockCacheStats(); + incrementCounter(7, + () -> indexStats.incrementEvict(new TestKey(PackExt.INDEX))); + evictions[PackExt.INDEX.getPosition()] = 7; + + PackExtBlockCacheTable tables = PackExtBlockCacheTable + .fromCacheTables(cacheTableWithStats(packStats), + Map.of(PackExt.BITMAP_INDEX, + cacheTableWithStats(bitmapStats), PackExt.INDEX, + cacheTableWithStats(indexStats))); + + assertArrayEquals(tables.getBlockCacheStats().getEvictions(), + evictions); + } + + private static void incrementCounter(int amount, Runnable fn) { + for (int i = 0; i < amount; i++) { + fn.run(); + } + } + + private static long[] createEmptyStatsArray() { + return new long[PackExt.values().length]; + } + + private static DfsBlockCacheTable cacheTableWithStats( + DfsBlockCacheStats dfsBlockCacheStats) { + DfsBlockCacheTable cacheTable = mock(DfsBlockCacheTable.class); + when(cacheTable.getBlockCacheStats()).thenReturn(dfsBlockCacheStats); + return cacheTable; + } + + private static class TestKey extends DfsStreamKey { + TestKey(PackExt packExt) { + super(0, packExt); + } + + @Override + public boolean equals(Object o) { + return false; + } + } +} diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties index 9d12facb33..a15fe1f372 100644 --- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties +++ b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties @@ -285,6 +285,8 @@ DIRCUnrecognizedExtendedFlags=Unrecognized extended flags: {0} downloadCancelled=Download cancelled downloadCancelledDuringIndexing=Download cancelled during indexing duplicateAdvertisementsOf=duplicate advertisements of {0} +duplicateCacheTablesGiven=Duplicate cache tables given +duplicatePackExtensionsForCacheTables=Duplicate pack extension {0} in cache tables duplicatePackExtensionsSet=Attempting to configure duplicate pack extensions: {0}.{1}.{2} contains {3} duplicateRef=Duplicate ref: {0} duplicateRefAttribute=Duplicate ref attribute: {0} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java index 311d9c22aa..e31533aaa2 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java @@ -315,6 +315,8 @@ public class JGitText extends TranslationBundle { /***/ public String downloadCancelled; /***/ public String downloadCancelledDuringIndexing; /***/ public String duplicateAdvertisementsOf; + /***/ public String duplicateCacheTablesGiven; + /***/ public String duplicatePackExtensionsForCacheTables; /***/ public String duplicatePackExtensionsSet; /***/ public String duplicateRef; /***/ public String duplicateRefAttribute; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java index fa86701de8..20f4666373 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java @@ -263,6 +263,21 @@ public class DfsBlockCacheConfig { return packExtCacheConfigurations; } + /** + * Set the list of pack ext cache configs. + * + * Made visible for testing. + * + * @param packExtCacheConfigurations + * the list of pack ext cache configs to set. + * @return {@code this} + */ + DfsBlockCacheConfig setPackExtCacheConfigurations( + List packExtCacheConfigurations) { + this.packExtCacheConfigurations = packExtCacheConfigurations; + return this; + } + /** * Update properties by setting fields from the configuration. *

@@ -435,7 +450,15 @@ public class DfsBlockCacheConfig { // Configuration for the cache instance. private final DfsBlockCacheConfig packExtCacheConfiguration; - private DfsBlockCachePackExtConfig(EnumSet packExts, + /** + * Made visible for testing. + * + * @param packExts + * Set of {@link PackExt}s associated to this cache config. + * @param packExtCacheConfiguration + * {@link DfsBlockCacheConfig} for this cache config. + */ + DfsBlockCachePackExtConfig(EnumSet packExts, DfsBlockCacheConfig packExtCacheConfiguration) { this.packExts = packExts; this.packExtCacheConfiguration = packExtCacheConfiguration; @@ -475,6 +498,5 @@ public class DfsBlockCacheConfig { return new DfsBlockCachePackExtConfig(EnumSet.copyOf(packExts), dfsBlockCacheConfig); } - } } \ No newline at end of file diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/PackExtBlockCacheTable.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/PackExtBlockCacheTable.java new file mode 100644 index 0000000000..858f731b70 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/PackExtBlockCacheTable.java @@ -0,0 +1,289 @@ +/* + * Copyright (c) 2024, Google LLC and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.storage.dfs; + +import java.io.IOException; +import java.text.MessageFormat; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.internal.storage.dfs.DfsBlockCache.ReadableChannelSupplier; +import org.eclipse.jgit.internal.storage.dfs.DfsBlockCache.Ref; +import org.eclipse.jgit.internal.storage.dfs.DfsBlockCache.RefLoader; +import org.eclipse.jgit.internal.storage.dfs.DfsBlockCacheConfig.DfsBlockCachePackExtConfig; +import org.eclipse.jgit.internal.storage.pack.PackExt; + +/** + * A table that holds multiple cache tables accessed by {@link PackExt} types. + * + *

+ * Allows the separation of entries from different {@link PackExt} types to + * limit churn in cache caused by entries of differing sizes. + *

+ * Separating these tables enables the fine-tuning of cache tables per extension + * type. + */ +class PackExtBlockCacheTable implements DfsBlockCacheTable { + private final DfsBlockCacheTable defaultBlockCacheTable; + + // Holds the unique tables backing the extBlockCacheTables values. + private final List blockCacheTableList; + + // Holds the mapping of PackExt to DfsBlockCacheTables. + // The relation between the size of extBlockCacheTables entries and + // blockCacheTableList entries is: + // blockCacheTableList.size() <= extBlockCacheTables.size() + private final Map extBlockCacheTables; + + /** + * Builds the PackExtBlockCacheTable from a list of + * {@link DfsBlockCachePackExtConfig}s. + * + * @param cacheConfig + * {@link DfsBlockCacheConfig} containing + * {@link DfsBlockCachePackExtConfig}s used to configure + * PackExtBlockCacheTable. The {@link DfsBlockCacheConfig} holds + * the configuration for the default cache table. + * @return the cache table built from the given configs. + * @throws IllegalArgumentException + * when no {@link DfsBlockCachePackExtConfig} exists in the + * {@link DfsBlockCacheConfig}. + */ + static PackExtBlockCacheTable fromBlockCacheConfigs( + DfsBlockCacheConfig cacheConfig) { + DfsBlockCacheTable defaultTable = new ClockBlockCacheTable(cacheConfig); + Map packExtBlockCacheTables = new HashMap<>(); + List packExtConfigs = cacheConfig + .getPackExtCacheConfigurations(); + if (packExtConfigs == null || packExtConfigs.size() == 0) { + throw new IllegalArgumentException( + JGitText.get().noPackExtConfigurationGiven); + } + for (DfsBlockCachePackExtConfig packExtCacheConfig : packExtConfigs) { + DfsBlockCacheTable table = new ClockBlockCacheTable( + packExtCacheConfig.getPackExtCacheConfiguration()); + for (PackExt packExt : packExtCacheConfig.getPackExts()) { + if (packExtBlockCacheTables.containsKey(packExt)) { + throw new IllegalArgumentException(MessageFormat.format( + JGitText.get().duplicatePackExtensionsForCacheTables, + packExt)); + } + packExtBlockCacheTables.put(packExt, table); + } + } + return fromCacheTables(defaultTable, packExtBlockCacheTables); + } + + /** + * Creates a new PackExtBlockCacheTable from the combination of a default + * {@link DfsBlockCacheTable} and a map of {@link PackExt}s to + * {@link DfsBlockCacheTable}s. + *

+ * This method allows for the PackExtBlockCacheTable to handle a mapping of + * {@link PackExt}s to arbitrarily defined {@link DfsBlockCacheTable} + * implementations. This is especially useful for users wishing to implement + * custom cache tables. + *

+ * This is currently made visible for testing. + * + * @param defaultBlockCacheTable + * the default table used when a handling a {@link PackExt} type + * that does not map to a {@link DfsBlockCacheTable} mapped by + * packExtsCacheTablePairs. + * @param packExtBlockCacheTables + * the mapping of {@link PackExt}s to + * {@link DfsBlockCacheTable}s. A single + * {@link DfsBlockCacheTable} can be defined for multiple + * {@link PackExt}s in a many-to-one relationship. + * @return the PackExtBlockCacheTable created from the + * defaultBlockCacheTable and packExtsCacheTablePairs mapping. + * @throws IllegalArgumentException + * when a {@link PackExt} is defined for multiple + * {@link DfsBlockCacheTable}s. + */ + static PackExtBlockCacheTable fromCacheTables( + DfsBlockCacheTable defaultBlockCacheTable, + Map packExtBlockCacheTables) { + Set blockCacheTables = new HashSet<>(); + blockCacheTables.add(defaultBlockCacheTable); + blockCacheTables.addAll(packExtBlockCacheTables.values()); + return new PackExtBlockCacheTable(defaultBlockCacheTable, + List.copyOf(blockCacheTables), packExtBlockCacheTables); + } + + private PackExtBlockCacheTable(DfsBlockCacheTable defaultBlockCacheTable, + List blockCacheTableList, + Map extBlockCacheTables) { + this.defaultBlockCacheTable = defaultBlockCacheTable; + this.blockCacheTableList = blockCacheTableList; + this.extBlockCacheTables = extBlockCacheTables; + } + + @Override + public boolean hasBlock0(DfsStreamKey key) { + return getTable(key).hasBlock0(key); + } + + @Override + public DfsBlock getOrLoad(BlockBasedFile file, long position, + DfsReader dfsReader, ReadableChannelSupplier fileChannel) + throws IOException { + return getTable(file.ext).getOrLoad(file, position, dfsReader, + fileChannel); + } + + @Override + public Ref getOrLoadRef(DfsStreamKey key, long position, + RefLoader loader) throws IOException { + return getTable(key).getOrLoadRef(key, position, loader); + } + + @Override + public void put(DfsBlock v) { + getTable(v.stream).put(v); + } + + @Override + public Ref put(DfsStreamKey key, long pos, long size, T v) { + return getTable(key).put(key, pos, size, v); + } + + @Override + public Ref putRef(DfsStreamKey key, long size, T v) { + return getTable(key).putRef(key, size, v); + } + + @Override + public boolean contains(DfsStreamKey key, long position) { + return getTable(key).contains(key, position); + } + + @Override + public T get(DfsStreamKey key, long position) { + return getTable(key).get(key, position); + } + + @Override + public BlockCacheStats getBlockCacheStats() { + return new CacheStats(blockCacheTableList.stream() + .map(DfsBlockCacheTable::getBlockCacheStats) + .collect(Collectors.toList())); + } + + private DfsBlockCacheTable getTable(PackExt packExt) { + return extBlockCacheTables.getOrDefault(packExt, + defaultBlockCacheTable); + } + + private DfsBlockCacheTable getTable(DfsStreamKey key) { + return extBlockCacheTables.getOrDefault(getPackExt(key), + defaultBlockCacheTable); + } + + private static PackExt getPackExt(DfsStreamKey key) { + return PackExt.values()[key.packExtPos]; + } + + private static class CacheStats implements BlockCacheStats { + private final List blockCacheStats; + + private CacheStats(List blockCacheStats) { + this.blockCacheStats = blockCacheStats; + } + + @Override + public long[] getCurrentSize() { + long[] sums = emptyPackStats(); + for (BlockCacheStats blockCacheStatsEntry : blockCacheStats) { + sums = add(sums, blockCacheStatsEntry.getCurrentSize()); + } + return sums; + } + + @Override + public long[] getHitCount() { + long[] sums = emptyPackStats(); + for (BlockCacheStats blockCacheStatsEntry : blockCacheStats) { + sums = add(sums, blockCacheStatsEntry.getHitCount()); + } + return sums; + } + + @Override + public long[] getMissCount() { + long[] sums = emptyPackStats(); + for (BlockCacheStats blockCacheStatsEntry : blockCacheStats) { + sums = add(sums, blockCacheStatsEntry.getMissCount()); + } + return sums; + } + + @Override + public long[] getTotalRequestCount() { + long[] sums = emptyPackStats(); + for (BlockCacheStats blockCacheStatsEntry : blockCacheStats) { + sums = add(sums, blockCacheStatsEntry.getTotalRequestCount()); + } + return sums; + } + + @Override + public long[] getHitRatio() { + long[] hit = getHitCount(); + long[] miss = getMissCount(); + long[] ratio = new long[Math.max(hit.length, miss.length)]; + for (int i = 0; i < ratio.length; i++) { + if (i >= hit.length) { + ratio[i] = 0; + } else if (i >= miss.length) { + ratio[i] = 100; + } else { + long total = hit[i] + miss[i]; + ratio[i] = total == 0 ? 0 : hit[i] * 100 / total; + } + } + return ratio; + } + + @Override + public long[] getEvictions() { + long[] sums = emptyPackStats(); + for (BlockCacheStats blockCacheStatsEntry : blockCacheStats) { + sums = add(sums, blockCacheStatsEntry.getEvictions()); + } + return sums; + } + + private static long[] emptyPackStats() { + return new long[PackExt.values().length]; + } + + private static long[] add(long[] first, long[] second) { + long[] sums = new long[Integer.max(first.length, second.length)]; + int i; + for (i = 0; i < Integer.min(first.length, second.length); i++) { + sums[i] = first[i] + second[i]; + } + for (int j = i; j < first.length; j++) { + sums[j] = first[i]; + } + for (int j = i; j < second.length; j++) { + sums[j] = second[i]; + } + return sums; + } + } +} -- cgit v1.2.3 From 31e534193ff531e3aad1f115b16d8bda97bfc9e9 Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Wed, 10 Apr 2024 13:52:56 -0700 Subject: DfsPackFile: Abstract the loading of pack indexes DfsPackFile assumes that the indexes are stored in file streams and their references need to be cached in DFS. This doesn't allow us to experiment other storage options, like key-value databases. In these experiments not all indexes are together in the same storage. Define an interface per index to load it, so implementors can focus on the specifics of their index. Put them together in the IndexFactory interface. The implementation of the IndexFactory chooses the right combination of storages. At the moment we do this only for primary and reverse indexes. Following changes can do the same for other indexes. Change-Id: Icf790d8ba64b34dbe984759f1c6d8ec702554149 --- .../jgit/internal/storage/dfs/DfsPackFile.java | 253 +++++++++++++++------ 1 file changed, 183 insertions(+), 70 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java index cdd10618e0..845feab5ac 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java @@ -107,6 +107,53 @@ public final class DfsPackFile extends BlockBasedFile { /** Lock for {@link #corruptObjects}. */ private final Object corruptObjectsLock = new Object(); + private final IndexFactory indexFactory; + + /** + * Returns the indexes for this pack. + *

+ * We define indexes in different sub interfaces to allow implementing the + * indexes over different combinations of backends. + *

+ * Implementations decide if/how to cache the indexes. The calling + * DfsPackFile will keep the reference to the index as long as it needs it. + */ + public interface IndexFactory { + /** + * Take care of loading the primary and reverse indexes for this pack. + */ + interface PackIndexes { + /** + * Load the primary index for the pack. + * + * @param ctx + * reader to find the raw bytes + * @return a primary index + * @throws IOException + * a problem finding/parsing the index + */ + PackIndex index(DfsReader ctx) throws IOException; + + /** + * Load the reverse index of the pack + * + * @param ctx + * reader to find the raw bytes + * @return the reverse index of the pack + * @throws IOException + * a problem finding/parsing the reverse index + */ + PackReverseIndex reverseIndex(DfsReader ctx) throws IOException; + } + + /** + * Returns a provider of the primary and reverse indexes of this pack + * + * @return an implementation of the {@link PackIndexes} interface + */ + PackIndexes getPackIndexes(); + } + /** * Construct a reader for an existing, packfile. * @@ -116,7 +163,8 @@ public final class DfsPackFile extends BlockBasedFile { * description of the pack within the DFS. */ DfsPackFile(DfsBlockCache cache, DfsPackDescription desc) { - this(cache, desc, DEFAULT_BITMAP_LOADER); + this(cache, desc, DEFAULT_BITMAP_LOADER, + new CachedStreamIndexFactory(cache, desc)); } /** @@ -128,9 +176,11 @@ public final class DfsPackFile extends BlockBasedFile { * description of the pack within the DFS * @param bitmapLoader * loader to get the bitmaps of this pack (if any) + * @param indexFactory + * an IndexFactory to get references to the indexes of this pack */ public DfsPackFile(DfsBlockCache cache, DfsPackDescription desc, - PackBitmapIndexLoader bitmapLoader) { + PackBitmapIndexLoader bitmapLoader, IndexFactory indexFactory) { super(cache, desc, PACK); int bs = desc.getBlockSize(PACK); @@ -142,6 +192,7 @@ public final class DfsPackFile extends BlockBasedFile { length = sz > 0 ? sz : -1; this.bitmapLoader = bitmapLoader; + this.indexFactory = indexFactory; } /** @@ -196,22 +247,7 @@ public final class DfsPackFile extends BlockBasedFile { Repository.getGlobalListenerList() .dispatch(new BeforeDfsPackIndexLoadedEvent(this)); try { - DfsStreamKey idxKey = desc.getStreamKey(INDEX); - // Keep the value parsed in the loader, in case the Ref<> is - // nullified in ClockBlockCacheTable#reserveSpace - // before we read its value. - AtomicReference loadedRef = new AtomicReference<>(null); - DfsBlockCache.Ref cachedRef = cache.getOrLoadRef(idxKey, - REF_POSITION, () -> { - RefWithSize idx = loadPackIndex(ctx); - loadedRef.set(idx.ref); - return new DfsBlockCache.Ref<>(idxKey, REF_POSITION, - idx.size, idx.ref); - }); - if (loadedRef.get() == null) { - ctx.stats.idxCacheHit++; - } - index = cachedRef.get() != null ? cachedRef.get() : loadedRef.get(); + index = indexFactory.getPackIndexes().index(ctx); if (index == null) { throw new IOException( "Couldn't get a reference to the primary index"); //$NON-NLS-1$ @@ -328,26 +364,7 @@ public final class DfsPackFile extends BlockBasedFile { return reverseIndex; } - PackIndex idx = idx(ctx); - DfsStreamKey revKey = desc.getStreamKey(REVERSE_INDEX); - // Keep the value parsed in the loader, in case the Ref<> is - // nullified in ClockBlockCacheTable#reserveSpace - // before we read its value. - AtomicReference loadedRef = new AtomicReference<>( - null); - DfsBlockCache.Ref cachedRef = cache - .getOrLoadRef(revKey, REF_POSITION, () -> { - RefWithSize ridx = loadReverseIdx(ctx, - idx); - loadedRef.set(ridx.ref); - return new DfsBlockCache.Ref<>(revKey, REF_POSITION, - ridx.size, ridx.ref); - }); - if (loadedRef.get() == null) { - ctx.stats.ridxCacheHit++; - } - reverseIndex = cachedRef.get() != null ? cachedRef.get() - : loadedRef.get(); + reverseIndex = indexFactory.getPackIndexes().reverseIndex(ctx); if (reverseIndex == null) { throw new IOException( "Couldn't get a reference to the reverse index"); //$NON-NLS-1$ @@ -1227,38 +1244,6 @@ public final class DfsPackFile extends BlockBasedFile { } } - private RefWithSize loadPackIndex(DfsReader ctx) - throws IOException { - try { - ctx.stats.readIdx++; - long start = System.nanoTime(); - try (ReadableChannel rc = ctx.db.openFile(desc, INDEX)) { - PackIndex idx = PackIndex.read(alignTo8kBlocks(rc)); - ctx.stats.readIdxBytes += rc.position(); - return new RefWithSize<>(idx, idx.getObjectCount() * REC_SIZE); - } finally { - ctx.stats.readIdxMicros += elapsedMicros(start); - } - } catch (EOFException e) { - throw new IOException(MessageFormat.format( - DfsText.get().shortReadOfIndex, - desc.getFileName(INDEX)), e); - } catch (IOException e) { - throw new IOException(MessageFormat.format( - DfsText.get().cannotReadIndex, - desc.getFileName(INDEX)), e); - } - } - - private static RefWithSize loadReverseIdx(DfsReader ctx, - PackIndex idx) { - ctx.stats.readReverseIdx++; - long start = System.nanoTime(); - PackReverseIndex revidx = PackReverseIndexFactory.computeFromIndex(idx); - ctx.stats.readReverseIdxMicros += elapsedMicros(start); - return new RefWithSize<>(revidx, idx.getObjectCount() * 8); - } - private DfsBlockCache.Ref loadObjectSizeIndex( DfsReader ctx, DfsStreamKey objectSizeIndexKey) throws IOException { ctx.stats.readObjectSizeIndex++; @@ -1457,6 +1442,134 @@ public final class DfsPackFile extends BlockBasedFile { } } + /** + * An index factory backed by Dfs streams and references cached in + * DfsBlockCache + */ + public static final class CachedStreamIndexFactory implements IndexFactory { + private final CachedStreamPackIndexes indexes; + + /** + * An index factory + * + * @param cache + * DFS block cache to use for the references + * @param desc + * This factory loads indexes for this package + */ + public CachedStreamIndexFactory(DfsBlockCache cache, + DfsPackDescription desc) { + this.indexes = new CachedStreamPackIndexes(cache, desc); + } + + @Override + public PackIndexes getPackIndexes() { + return indexes; + } + } + + /** + * Load primary and reverse index from Dfs streams and cache the references + * in DfsBlockCache. + */ + public static final class CachedStreamPackIndexes implements IndexFactory.PackIndexes { + private final DfsBlockCache cache; + + private final DfsPackDescription desc; + + /** + * An index factory + * + * @param cache + * DFS block cache to use for the references + * @param desc This factory loads indexes for this package + */ + public CachedStreamPackIndexes(DfsBlockCache cache, + DfsPackDescription desc) { + this.cache = cache; + this.desc = desc; + } + + @Override + public PackIndex index(DfsReader ctx) throws IOException { + DfsStreamKey idxKey = desc.getStreamKey(INDEX); + // Keep the value parsed in the loader, in case the Ref<> is + // nullified in ClockBlockCacheTable#reserveSpace + // before we read its value. + AtomicReference loadedRef = new AtomicReference<>(null); + DfsBlockCache.Ref cachedRef = cache.getOrLoadRef(idxKey, + REF_POSITION, () -> { + RefWithSize idx = loadPackIndex(ctx, desc); + loadedRef.set(idx.ref); + return new DfsBlockCache.Ref<>(idxKey, REF_POSITION, + idx.size, idx.ref); + }); + if (loadedRef.get() == null) { + ctx.stats.idxCacheHit++; + } + return cachedRef.get() != null ? cachedRef.get() : loadedRef.get(); + } + + private static RefWithSize loadPackIndex(DfsReader ctx, + DfsPackDescription desc) throws IOException { + try { + ctx.stats.readIdx++; + long start = System.nanoTime(); + try (ReadableChannel rc = ctx.db.openFile(desc, INDEX)) { + PackIndex idx = PackIndex.read(alignTo8kBlocks(rc)); + ctx.stats.readIdxBytes += rc.position(); + return new RefWithSize<>(idx, + idx.getObjectCount() * REC_SIZE); + } finally { + ctx.stats.readIdxMicros += elapsedMicros(start); + } + } catch (EOFException e) { + throw new IOException( + MessageFormat.format(DfsText.get().shortReadOfIndex, + desc.getFileName(INDEX)), + e); + } catch (IOException e) { + throw new IOException( + MessageFormat.format(DfsText.get().cannotReadIndex, + desc.getFileName(INDEX)), + e); + } + } + + @Override + public PackReverseIndex reverseIndex(DfsReader ctx) throws IOException { + PackIndex idx = index(ctx); + DfsStreamKey revKey = desc.getStreamKey(REVERSE_INDEX); + // Keep the value parsed in the loader, in case the Ref<> is + // nullified in ClockBlockCacheTable#reserveSpace + // before we read its value. + AtomicReference loadedRef = new AtomicReference<>( + null); + DfsBlockCache.Ref cachedRef = cache + .getOrLoadRef(revKey, REF_POSITION, () -> { + RefWithSize ridx = loadReverseIdx(ctx, + idx); + loadedRef.set(ridx.ref); + return new DfsBlockCache.Ref<>(revKey, REF_POSITION, + ridx.size, ridx.ref); + }); + if (loadedRef.get() == null) { + ctx.stats.ridxCacheHit++; + } + return cachedRef.get() != null ? cachedRef.get() : loadedRef.get(); + } + + private static RefWithSize loadReverseIdx( + DfsReader ctx, PackIndex idx) { + ctx.stats.readReverseIdx++; + long start = System.nanoTime(); + PackReverseIndex revidx = PackReverseIndexFactory + .computeFromIndex(idx); + ctx.stats.readReverseIdxMicros += elapsedMicros(start); + return new RefWithSize<>(revidx, idx.getObjectCount() * 8); + } + } + private static final class RefWithSize { final V ref; final long size; -- cgit v1.2.3 From 81199f02fb49f79ef17b2daa3efac00d16040766 Mon Sep 17 00:00:00 2001 From: granny Date: Tue, 2 Jul 2024 15:15:37 -0700 Subject: Lib: Fix ssh value for gpg.format throwing an IllegalArgumentException Git version 2.34 and later supports signing commits and tags with SSH keys. This means gpg.format now supports "ssh" as a value. Change-Id: Iee1e5a68a816bec149a17a73a6916d2884a54163 --- .../tst/org/eclipse/jgit/lib/GpgConfigTest.java | 10 ++++++++++ org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgConfig.java | 4 +++- 2 files changed, 13 insertions(+), 1 deletion(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/GpgConfigTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/GpgConfigTest.java index 32f6766d47..5c2b190777 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/GpgConfigTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/GpgConfigTest.java @@ -95,6 +95,16 @@ public class GpgConfigTest { assertEquals(GpgConfig.GpgFormat.X509, new GpgConfig(c).getKeyFormat()); } + @Test + public void testGetKeyFormat_ssh() throws Exception { + Config c = parse("" // + + "[gpg]\n" // + + " format = ssh\n" // + ); + + assertEquals(GpgConfig.GpgFormat.SSH, new GpgConfig(c).getKeyFormat()); + } + @Test public void testGetSigningKey() throws Exception { Config c = parse("" // diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgConfig.java index 427a235f3b..4b0c07983f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgConfig.java @@ -24,7 +24,9 @@ public class GpgConfig { /** Value for openpgp */ OPENPGP("openpgp"), //$NON-NLS-1$ /** Value for x509 */ - X509("x509"); //$NON-NLS-1$ + X509("x509"), //$NON-NLS-1$ + /** Value for ssh */ + SSH("ssh"); //$NON-NLS-1$ private final String configValue; -- cgit v1.2.3 From d82bea9d8bbcb438459f756f3d57068997aeb782 Mon Sep 17 00:00:00 2001 From: Sergey Zakharov Date: Sun, 21 Jul 2024 16:06:19 +0300 Subject: ObjectWalk: Remove duplicated word "the" in class documentation Change-Id: I0ca0b5fc65b5cafc768a053b1de40aea9f14231c --- org.eclipse.jgit/src/org/eclipse/jgit/revwalk/ObjectWalk.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/ObjectWalk.java b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/ObjectWalk.java index 82671d9abb..385f073a77 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/ObjectWalk.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/ObjectWalk.java @@ -43,7 +43,7 @@ import org.eclipse.jgit.util.RawParseUtils; * Tree and blob objects reachable from interesting commits are automatically * scheduled for inclusion in the results of {@link #nextObject()}, returning * each object exactly once. Objects are sorted and returned according to the - * the commits that reference them and the order they appear within a tree. + * commits that reference them and the order they appear within a tree. * Ordering can be affected by changing the * {@link org.eclipse.jgit.revwalk.RevSort} used to order the commits that are * returned first. -- cgit v1.2.3 From 9b73ec4d12d65f3366e7cc5c3533d84d8c442a67 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 9 Aug 2024 11:53:01 +0200 Subject: Fix "Comparison of narrow type with wide type in loop condition" This issue was detected by a GitHub CodeQL security scan run on JGit source code. Description of the error raised by the security scan: "In a loop condition, comparison of a value of a narrow type with a value of a wide type may always evaluate to true if the wider value is sufficiently large (or small). This is because the narrower value may overflow. This can lead to an infinite loop." Fix this by using type `long` for the local variable `done`. Change-Id: Ibd4f71299e3f2e40d4331227bd143569a4264d8c --- org.eclipse.jgit/src/org/eclipse/jgit/transport/PackParser.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/PackParser.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/PackParser.java index 715cbb48fb..cfceed00e8 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/PackParser.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/PackParser.java @@ -531,7 +531,7 @@ public abstract class PackParser { receiving.beginTask(JGitText.get().receivingObjects, (int) expectedObjectCount); try { - for (int done = 0; done < expectedObjectCount; done++) { + for (long done = 0; done < expectedObjectCount; done++) { indexOneObject(); receiving.update(1); if (receiving.isCancelled()) -- cgit v1.2.3 From a5e123349c921bd6850e8d11d00ea142422dddba Mon Sep 17 00:00:00 2001 From: Thomas Wolf Date: Sat, 10 Aug 2024 19:27:07 +0200 Subject: ConfigConstants: Add missing @since 7.0 Change-Id: I1ea31c1f0735b7c8fd09fbedc413d613e4baa803 Signed-off-by: Thomas Wolf --- org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java index f4e43c9f53..4be0cceedc 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java @@ -77,7 +77,11 @@ public final class ConfigConstants { /** The "dfs" section */ public static final String CONFIG_DFS_SECTION = "dfs"; - /** The dfs cache subsection prefix */ + /** + * The dfs cache subsection prefix. + * + * @since 7.0 + */ public static final String CONFIG_DFS_CACHE_PREFIX = "dfs."; /** -- cgit v1.2.3 From 888cd56583b23b88f714ac3f0596e23c690c1a74 Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Tue, 20 Aug 2024 11:56:41 -0700 Subject: DfsReaderIoStats: Order fields and methods consistently As fields and getters were added, we didn't respect the (hard to see) existing order. Reorder with the following criteria: Methods: xCacheHits() (for all indexes in index order), xCount() (same), xBytes() (same), xMicros() (same). Index order: primary, reverse, bitmap, commit-graph, object-size Change-Id: I28f1d8121070d4357d566e3683947a26ceb3ba04 --- .../internal/storage/dfs/DfsReaderIoStats.java | 68 +++++++++++----------- 1 file changed, 34 insertions(+), 34 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReaderIoStats.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReaderIoStats.java index adb4673ae4..1e5e439598 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReaderIoStats.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReaderIoStats.java @@ -40,12 +40,12 @@ public class DfsReaderIoStats { /** Total number of complete pack indexes read into memory. */ long readIdx; - /** Total number of complete bitmap indexes read into memory. */ - long readBitmap; - /** Total number of reverse indexes added into memory. */ long readReverseIdx; + /** Total number of complete bitmap indexes read into memory. */ + long readBitmap; + /** Total number of complete commit graphs read into memory. */ long readCommitGraph; @@ -55,6 +55,9 @@ public class DfsReaderIoStats { /** Total number of bytes read from pack indexes. */ long readIdxBytes; + /** Total number of bytes read from bitmap indexes. */ + long readBitmapIdxBytes; + /** Total number of bytes read from commit graphs. */ long readCommitGraphBytes; @@ -67,18 +70,15 @@ public class DfsReaderIoStats { /** Total microseconds spent creating reverse indexes. */ long readReverseIdxMicros; + /** Total microseconds spent reading bitmap indexes. */ + long readBitmapIdxMicros; + /** Total microseconds spent creating commit graphs. */ long readCommitGraphMicros; /** Total microseconds spent creating object size indexes */ long readObjectSizeIndexMicros; - /** Total number of bytes read from bitmap indexes. */ - long readBitmapIdxBytes; - - /** Total microseconds spent reading bitmap indexes. */ - long readBitmapIdxMicros; - /** Total number of block cache hits. */ long blockCacheHit; @@ -195,21 +195,21 @@ public class DfsReaderIoStats { } /** - * Get total number of times the commit graph read into memory. + * Get total number of complete bitmap indexes read into memory. * - * @return total number of commit graph read into memory. + * @return total number of complete bitmap indexes read into memory. */ - public long getReadCommitGraphCount() { - return stats.readCommitGraph; + public long getReadBitmapIndexCount() { + return stats.readBitmap; } /** - * Get total number of complete bitmap indexes read into memory. + * Get total number of times the commit graph read into memory. * - * @return total number of complete bitmap indexes read into memory. + * @return total number of commit graph read into memory. */ - public long getReadBitmapIndexCount() { - return stats.readBitmap; + public long getReadCommitGraphCount() { + return stats.readCommitGraph; } /** @@ -230,6 +230,15 @@ public class DfsReaderIoStats { return stats.readIdxBytes; } + /** + * Get total number of bytes read from bitmap indexes. + * + * @return total number of bytes read from bitmap indexes. + */ + public long getReadBitmapIndexBytes() { + return stats.readBitmapIdxBytes; + } + /** * Get total number of bytes read from commit graphs. * @@ -258,30 +267,21 @@ public class DfsReaderIoStats { } /** - * Get total microseconds spent reading commit graphs. - * - * @return total microseconds spent reading commit graphs. - */ - public long getReadCommitGraphMicros() { - return stats.readCommitGraphMicros; - } - - /** - * Get total number of bytes read from bitmap indexes. + * Get total microseconds spent reading bitmap indexes. * - * @return total number of bytes read from bitmap indexes. + * @return total microseconds spent reading bitmap indexes. */ - public long getReadBitmapIndexBytes() { - return stats.readBitmapIdxBytes; + public long getReadBitmapIndexMicros() { + return stats.readBitmapIdxMicros; } /** - * Get total microseconds spent reading bitmap indexes. + * Get total microseconds spent reading commit graphs. * - * @return total microseconds spent reading bitmap indexes. + * @return total microseconds spent reading commit graphs. */ - public long getReadBitmapIndexMicros() { - return stats.readBitmapIdxMicros; + public long getReadCommitGraphMicros() { + return stats.readCommitGraphMicros; } /** -- cgit v1.2.3 From 81067f18e6ee9ac36602926ac0e47945a18e8e3c Mon Sep 17 00:00:00 2001 From: Thomas Wolf Date: Tue, 20 Aug 2024 22:40:10 +0200 Subject: GpgConfig: Add missing @since Change-Id: Ie56e7d8f2defe10a87565056a1763288d5b1e1a6 Signed-off-by: Thomas Wolf --- org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgConfig.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgConfig.java index 4b0c07983f..f5064df061 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgConfig.java @@ -25,7 +25,11 @@ public class GpgConfig { OPENPGP("openpgp"), //$NON-NLS-1$ /** Value for x509 */ X509("x509"), //$NON-NLS-1$ - /** Value for ssh */ + /** + * Value for ssh. + * + * @since 7.0 + */ SSH("ssh"); //$NON-NLS-1$ private final String configValue; -- cgit v1.2.3 From d66175d7dca30be7ab96af0e3998f795a174b891 Mon Sep 17 00:00:00 2001 From: Kaushik Lingarkar Date: Tue, 20 Aug 2024 15:50:20 -0700 Subject: LockFile: Retry lock creation if parent dirs were removed In the small window between creation of the lock file's parent dirs and the lock file itself, the parent dirs may be cleaned by an external process packing refs in the repository. When this scenario occurs, retry creating the lock file (along with its parent dirs). Change-Id: Id7ec60c3f7f373b59f1dc8de6b8fa6df6bdf2570 Signed-off-by: Kaushik Lingarkar --- .../eclipse/jgit/internal/storage/file/LockFile.java | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LockFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LockFile.java index a2d8bd0140..1983541e4a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LockFile.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LockFile.java @@ -24,6 +24,7 @@ import java.nio.ByteBuffer; import java.nio.channels.Channels; import java.nio.channels.FileChannel; import java.nio.file.Files; +import java.nio.file.NoSuchFileException; import java.nio.file.StandardCopyOption; import java.nio.file.attribute.FileTime; import java.text.MessageFormat; @@ -141,9 +142,8 @@ public class LockFile { throw new IllegalStateException( MessageFormat.format(JGitText.get().lockAlreadyHeld, ref)); } - FileUtils.mkdirs(lck.getParentFile(), true); try { - token = FS.DETECTED.createNewFileAtomic(lck); + token = createLockFileWithRetry(); } catch (IOException e) { LOG.error(JGitText.get().failedCreateLockFile, lck, e); throw e; @@ -160,6 +160,19 @@ public class LockFile { return obtainedLock; } + private FS.LockToken createLockFileWithRetry() throws IOException { + try { + return createLockFile(); + } catch (NoSuchFileException e) { + return createLockFile(); + } + } + + private FS.LockToken createLockFile() throws IOException { + FileUtils.mkdirs(lck.getParentFile(), true); + return FS.DETECTED.createNewFileAtomic(lck); + } + /** * Try to establish the lock for appending. * -- cgit v1.2.3 From a78e6eaef63754902f46dffe657a783403c44bfe Mon Sep 17 00:00:00 2001 From: Thomas Wolf Date: Tue, 20 Aug 2024 22:41:45 +0200 Subject: Signing: refactor interfaces This is a big API-breaking change cleaning up the signing interfaces. Initially, these interfaces were GPG/OpenPGP-specific. When EGit added new signers and signature verifiers that called an external GPG executable, they were found inadequate and were extended to be able to pass in the GpgConfig to get access to the "gpg.program" setting. With the introduction of X.509 S/MIME signing, it was discovered that the interfaces were still not quite adequate, and the "Gpg" prefix on the class names were confusing. Since 7.0 is a major version bump, I'm taking this chance to overhaul these interfaces from ground up. For signing, there is a new Signer interface. With it goes a SignerFactory SPI interface, and a final Signers class managing the currently set signers. By default, signers for the different signature types are created from the signer factories, which are discovered via the ServiceLoader. External code can install its own signers, overriding the default factories. For signature verification, exactly the same mechanism is used. This simplifies the setup of signers and signature verifiers, and makes it all more regular. Signer instances just get a byte[] to sign and don't have to worry about ObjectBuilders at all. SignatureVerifier instances also just get the data and signature as byte[] and don't have to worry about extracting the signature from a commit or tag, or about what kind of signature it is. Both Signers and SignatureVerifiers always get passed the Repository and the GpgConfig. The repository will be needed in an implementation for SSH signatures because gpg.ssh.* configs may need to be loaded explicitly, and some of those values need the current workspace location. For signature verification, there is exactly one place in core JGit in SignatureVerifiers that extracts signatures, determines the signature type, and then calls the right signature verifier. Change RevTag to recognize all signature types known in git (GPG, X509, and SSH). Change-Id: I26d2731e7baebb38976c87b7f328b63a239760d5 Signed-off-by: Thomas Wolf --- org.eclipse.jgit.gpg.bc/META-INF/MANIFEST.MF | 5 +- org.eclipse.jgit.gpg.bc/pom.xml | 2 +- ...rg.eclipse.jgit.lib.GpgSignatureVerifierFactory | 1 - .../services/org.eclipse.jgit.lib.GpgSigner | 1 - .../org.eclipse.jgit.lib.SignatureVerifierFactory | 1 + .../services/org.eclipse.jgit.lib.SignerFactory | 1 + .../jgit/gpg/bc/BouncyCastleGpgSignerFactory.java | 34 --- .../internal/BouncyCastleGpgSignatureVerifier.java | 138 ++---------- .../BouncyCastleGpgSignatureVerifierFactory.java | 23 +- .../gpg/bc/internal/BouncyCastleGpgSigner.java | 127 ++++------- .../bc/internal/BouncyCastleGpgSignerFactory.java | 31 +++ .../src/org/eclipse/jgit/pgm/Log.java | 24 +-- .../src/org/eclipse/jgit/pgm/Show.java | 27 +-- .../src/org/eclipse/jgit/pgm/Tag.java | 5 +- .../jgit/pgm/internal/VerificationUtils.java | 2 +- .../org/eclipse/jgit/api/CommitCommandTest.java | 73 +++++-- .../org/eclipse/jgit/internal/JGitText.properties | 3 +- .../src/org/eclipse/jgit/api/CommitCommand.java | 47 ++-- .../src/org/eclipse/jgit/api/TagCommand.java | 45 ++-- .../org/eclipse/jgit/api/VerificationResult.java | 5 +- .../eclipse/jgit/api/VerifySignatureCommand.java | 57 +---- .../src/org/eclipse/jgit/internal/JGitText.java | 3 +- .../jgit/lib/AbstractGpgSignatureVerifier.java | 71 ------ .../src/org/eclipse/jgit/lib/Constants.java | 14 ++ .../src/org/eclipse/jgit/lib/GpgConfig.java | 24 +-- .../src/org/eclipse/jgit/lib/GpgObjectSigner.java | 95 -------- .../org/eclipse/jgit/lib/GpgSignatureVerifier.java | 184 ---------------- .../jgit/lib/GpgSignatureVerifierFactory.java | 92 -------- .../src/org/eclipse/jgit/lib/GpgSigner.java | 141 ------------ .../org/eclipse/jgit/lib/SignatureVerifier.java | 106 +++++++++ .../eclipse/jgit/lib/SignatureVerifierFactory.java | 37 ++++ .../org/eclipse/jgit/lib/SignatureVerifiers.java | 239 +++++++++++++++++++++ .../src/org/eclipse/jgit/lib/Signer.java | 139 ++++++++++++ .../src/org/eclipse/jgit/lib/SignerFactory.java | 37 ++++ .../src/org/eclipse/jgit/lib/Signers.java | 101 +++++++++ .../src/org/eclipse/jgit/revwalk/RevTag.java | 27 ++- .../src/org/eclipse/jgit/util/SignatureUtils.java | 26 +-- 37 files changed, 944 insertions(+), 1044 deletions(-) delete mode 100644 org.eclipse.jgit.gpg.bc/resources/META-INF/services/org.eclipse.jgit.lib.GpgSignatureVerifierFactory delete mode 100644 org.eclipse.jgit.gpg.bc/resources/META-INF/services/org.eclipse.jgit.lib.GpgSigner create mode 100644 org.eclipse.jgit.gpg.bc/resources/META-INF/services/org.eclipse.jgit.lib.SignatureVerifierFactory create mode 100644 org.eclipse.jgit.gpg.bc/resources/META-INF/services/org.eclipse.jgit.lib.SignerFactory delete mode 100644 org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/BouncyCastleGpgSignerFactory.java create mode 100644 org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/BouncyCastleGpgSignerFactory.java delete mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/lib/AbstractGpgSignatureVerifier.java delete mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgObjectSigner.java delete mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgSignatureVerifier.java delete mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgSignatureVerifierFactory.java delete mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgSigner.java create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/lib/SignatureVerifier.java create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/lib/SignatureVerifierFactory.java create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/lib/SignatureVerifiers.java create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/lib/Signer.java create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/lib/SignerFactory.java create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/lib/Signers.java (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.gpg.bc/META-INF/MANIFEST.MF b/org.eclipse.jgit.gpg.bc/META-INF/MANIFEST.MF index 9749ac1111..7a62d1d232 100644 --- a/org.eclipse.jgit.gpg.bc/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit.gpg.bc/META-INF/MANIFEST.MF @@ -28,9 +28,6 @@ Import-Package: org.bouncycastle.asn1;version="[1.69.0,2.0.0)", org.bouncycastle.util;version="[1.69.0,2.0.0)", org.bouncycastle.util.encoders;version="[1.69.0,2.0.0)", org.bouncycastle.util.io;version="[1.69.0,2.0.0)", - org.eclipse.jgit.annotations;version="[7.0.0,7.1.0)", - org.eclipse.jgit.api.errors;version="[7.0.0,7.1.0)", org.slf4j;version="[1.7.0,3.0.0)" -Export-Package: org.eclipse.jgit.gpg.bc;version="7.0.0", - org.eclipse.jgit.gpg.bc.internal;version="7.0.0";x-friends:="org.eclipse.jgit.gpg.bc.test", +Export-Package: org.eclipse.jgit.gpg.bc.internal;version="7.0.0";x-friends:="org.eclipse.jgit.gpg.bc.test", org.eclipse.jgit.gpg.bc.internal.keys;version="7.0.0";x-friends:="org.eclipse.jgit.gpg.bc.test" diff --git a/org.eclipse.jgit.gpg.bc/pom.xml b/org.eclipse.jgit.gpg.bc/pom.xml index f4dce68fab..ad907d3809 100644 --- a/org.eclipse.jgit.gpg.bc/pom.xml +++ b/org.eclipse.jgit.gpg.bc/pom.xml @@ -160,7 +160,7 @@ false false false - false + true true false diff --git a/org.eclipse.jgit.gpg.bc/resources/META-INF/services/org.eclipse.jgit.lib.GpgSignatureVerifierFactory b/org.eclipse.jgit.gpg.bc/resources/META-INF/services/org.eclipse.jgit.lib.GpgSignatureVerifierFactory deleted file mode 100644 index 17ab30fba7..0000000000 --- a/org.eclipse.jgit.gpg.bc/resources/META-INF/services/org.eclipse.jgit.lib.GpgSignatureVerifierFactory +++ /dev/null @@ -1 +0,0 @@ -org.eclipse.jgit.gpg.bc.internal.BouncyCastleGpgSignatureVerifierFactory \ No newline at end of file diff --git a/org.eclipse.jgit.gpg.bc/resources/META-INF/services/org.eclipse.jgit.lib.GpgSigner b/org.eclipse.jgit.gpg.bc/resources/META-INF/services/org.eclipse.jgit.lib.GpgSigner deleted file mode 100644 index 6752b64ddf..0000000000 --- a/org.eclipse.jgit.gpg.bc/resources/META-INF/services/org.eclipse.jgit.lib.GpgSigner +++ /dev/null @@ -1 +0,0 @@ -org.eclipse.jgit.gpg.bc.internal.BouncyCastleGpgSigner diff --git a/org.eclipse.jgit.gpg.bc/resources/META-INF/services/org.eclipse.jgit.lib.SignatureVerifierFactory b/org.eclipse.jgit.gpg.bc/resources/META-INF/services/org.eclipse.jgit.lib.SignatureVerifierFactory new file mode 100644 index 0000000000..17ab30fba7 --- /dev/null +++ b/org.eclipse.jgit.gpg.bc/resources/META-INF/services/org.eclipse.jgit.lib.SignatureVerifierFactory @@ -0,0 +1 @@ +org.eclipse.jgit.gpg.bc.internal.BouncyCastleGpgSignatureVerifierFactory \ No newline at end of file diff --git a/org.eclipse.jgit.gpg.bc/resources/META-INF/services/org.eclipse.jgit.lib.SignerFactory b/org.eclipse.jgit.gpg.bc/resources/META-INF/services/org.eclipse.jgit.lib.SignerFactory new file mode 100644 index 0000000000..c0b214db39 --- /dev/null +++ b/org.eclipse.jgit.gpg.bc/resources/META-INF/services/org.eclipse.jgit.lib.SignerFactory @@ -0,0 +1 @@ +org.eclipse.jgit.gpg.bc.internal.BouncyCastleGpgSignerFactory diff --git a/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/BouncyCastleGpgSignerFactory.java b/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/BouncyCastleGpgSignerFactory.java deleted file mode 100644 index fdd1a2b11a..0000000000 --- a/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/BouncyCastleGpgSignerFactory.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (C) 2021 Thomas Wolf and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ -package org.eclipse.jgit.gpg.bc; - -import org.eclipse.jgit.gpg.bc.internal.BouncyCastleGpgSigner; -import org.eclipse.jgit.lib.GpgSigner; - -/** - * Factory for creating a {@link GpgSigner} based on Bouncy Castle. - * - * @since 5.11 - */ -public final class BouncyCastleGpgSignerFactory { - - private BouncyCastleGpgSignerFactory() { - // No instantiation - } - - /** - * Creates a new {@link GpgSigner}. - * - * @return the {@link GpgSigner} - */ - public static GpgSigner create() { - return new BouncyCastleGpgSigner(); - } -} diff --git a/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/BouncyCastleGpgSignatureVerifier.java b/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/BouncyCastleGpgSignatureVerifier.java index 3378bb3969..5a3d43ba54 100644 --- a/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/BouncyCastleGpgSignatureVerifier.java +++ b/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/BouncyCastleGpgSignatureVerifier.java @@ -12,7 +12,6 @@ package org.eclipse.jgit.gpg.bc.internal; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; -import java.security.Security; import java.text.MessageFormat; import java.time.Instant; import java.util.Date; @@ -20,7 +19,6 @@ import java.util.List; import java.util.Locale; import org.bouncycastle.bcpg.sig.IssuerFingerprint; -import org.bouncycastle.jce.provider.BouncyCastleProvider; import org.bouncycastle.openpgp.PGPCompressedData; import org.bouncycastle.openpgp.PGPException; import org.bouncycastle.openpgp.PGPPublicKey; @@ -31,33 +29,20 @@ import org.bouncycastle.openpgp.PGPUtil; import org.bouncycastle.openpgp.jcajce.JcaPGPObjectFactory; import org.bouncycastle.openpgp.operator.jcajce.JcaPGPContentVerifierBuilderProvider; import org.bouncycastle.util.encoders.Hex; -import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.api.errors.JGitInternalException; -import org.eclipse.jgit.lib.AbstractGpgSignatureVerifier; import org.eclipse.jgit.lib.GpgConfig; -import org.eclipse.jgit.lib.GpgSignatureVerifier; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.lib.SignatureVerifier; import org.eclipse.jgit.util.LRUMap; import org.eclipse.jgit.util.StringUtils; /** - * A {@link GpgSignatureVerifier} to verify GPG signatures using BouncyCastle. + * A {@link SignatureVerifier} to verify GPG signatures using BouncyCastle. */ public class BouncyCastleGpgSignatureVerifier - extends AbstractGpgSignatureVerifier { + implements SignatureVerifier { - private static void registerBouncyCastleProviderIfNecessary() { - if (Security.getProvider(BouncyCastleProvider.PROVIDER_NAME) == null) { - Security.addProvider(new BouncyCastleProvider()); - } - } - - /** - * Creates a new instance and registers the BouncyCastle security provider - * if needed. - */ - public BouncyCastleGpgSignatureVerifier() { - registerBouncyCastleProviderIfNecessary(); - } + private static final String NAME = "bc"; //$NON-NLS-1$ // To support more efficient signature verification of multiple objects we // cache public keys once found in a LRU cache. @@ -70,7 +55,7 @@ public class BouncyCastleGpgSignatureVerifier @Override public String getName() { - return "bc"; //$NON-NLS-1$ + return NAME; } static PGPSignature parseSignature(InputStream in) @@ -90,9 +75,8 @@ public class BouncyCastleGpgSignatureVerifier } @Override - public SignatureVerification verify(@NonNull GpgConfig config, byte[] data, - byte[] signatureData) - throws IOException { + public SignatureVerification verify(Repository repository, GpgConfig config, + byte[] data, byte[] signatureData) throws IOException { PGPSignature signature = null; String fingerprint = null; String signer = null; @@ -127,14 +111,15 @@ public class BouncyCastleGpgSignatureVerifier } Date signatureCreatedAt = signature.getCreationTime(); if (fingerprint == null && signer == null && keyId == null) { - return new VerificationResult(signatureCreatedAt, null, null, null, - false, false, TrustLevel.UNKNOWN, + return new SignatureVerification(NAME, signatureCreatedAt, + null, null, null, false, false, TrustLevel.UNKNOWN, BCText.get().signatureNoKeyInfo); } if (fingerprint != null && keyId != null && !fingerprint.endsWith(keyId)) { - return new VerificationResult(signatureCreatedAt, signer, fingerprint, - signer, false, false, TrustLevel.UNKNOWN, + return new SignatureVerification(NAME, signatureCreatedAt, + signer, fingerprint, signer, false, false, + TrustLevel.UNKNOWN, MessageFormat.format(BCText.get().signatureInconsistent, keyId, fingerprint)); } @@ -175,15 +160,16 @@ public class BouncyCastleGpgSignatureVerifier bySigner.put(signer, NO_KEY); } } - return new VerificationResult(signatureCreatedAt, signer, - fingerprint, signer, false, false, TrustLevel.UNKNOWN, - BCText.get().signatureNoPublicKey); + return new SignatureVerification(NAME, signatureCreatedAt, + signer, fingerprint, signer, false, false, + TrustLevel.UNKNOWN, BCText.get().signatureNoPublicKey); } if (fingerprint != null && !publicKey.isExactMatch()) { // We did find _some_ signing key for the signer, but it doesn't // match the given fingerprint. - return new VerificationResult(signatureCreatedAt, signer, - fingerprint, signer, false, false, TrustLevel.UNKNOWN, + return new SignatureVerification(NAME, signatureCreatedAt, + signer, fingerprint, signer, false, false, + TrustLevel.UNKNOWN, MessageFormat.format(BCText.get().signatureNoSigningKey, fingerprint)); } @@ -229,8 +215,7 @@ public class BouncyCastleGpgSignatureVerifier boolean verified = false; try { signature.init( - new JcaPGPContentVerifierBuilderProvider() - .setProvider(BouncyCastleProvider.PROVIDER_NAME), + new JcaPGPContentVerifierBuilderProvider(), pubKey); signature.update(data); verified = signature.verify(); @@ -238,15 +223,8 @@ public class BouncyCastleGpgSignatureVerifier throw new JGitInternalException( BCText.get().signatureVerificationError, e); } - return new VerificationResult(signatureCreatedAt, signer, fingerprint, user, - verified, expired, trust, null); - } - - @Override - public SignatureVerification verify(byte[] data, byte[] signatureData) - throws IOException { - throw new UnsupportedOperationException( - "Call verify(GpgConfig, byte[], byte[]) instead."); //$NON-NLS-1$ + return new SignatureVerification(NAME, signatureCreatedAt, signer, + fingerprint, user, verified, expired, trust, null); } private TrustLevel parseGpgTrustPacket(byte[] packet) { @@ -282,76 +260,4 @@ public class BouncyCastleGpgSignatureVerifier byFingerprint.clear(); bySigner.clear(); } - - private static class VerificationResult implements SignatureVerification { - - private final Date creationDate; - - private final String signer; - - private final String keyUser; - - private final String fingerprint; - - private final boolean verified; - - private final boolean expired; - - private final @NonNull TrustLevel trustLevel; - - private final String message; - - public VerificationResult(Date creationDate, String signer, - String fingerprint, String user, boolean verified, - boolean expired, @NonNull TrustLevel trust, String message) { - this.creationDate = creationDate; - this.signer = signer; - this.fingerprint = fingerprint; - this.keyUser = user; - this.verified = verified; - this.expired = expired; - this.trustLevel = trust; - this.message = message; - } - - @Override - public Date getCreationDate() { - return creationDate; - } - - @Override - public String getSigner() { - return signer; - } - - @Override - public String getKeyUser() { - return keyUser; - } - - @Override - public String getKeyFingerprint() { - return fingerprint; - } - - @Override - public boolean isExpired() { - return expired; - } - - @Override - public TrustLevel getTrustLevel() { - return trustLevel; - } - - @Override - public String getMessage() { - return message; - } - - @Override - public boolean getVerified() { - return verified; - } - } } diff --git a/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/BouncyCastleGpgSignatureVerifierFactory.java b/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/BouncyCastleGpgSignatureVerifierFactory.java index ae82b758a6..566ad1bf91 100644 --- a/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/BouncyCastleGpgSignatureVerifierFactory.java +++ b/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/BouncyCastleGpgSignatureVerifierFactory.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2021, Thomas Wolf and others + * Copyright (C) 2021, 2024 Thomas Wolf and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -9,20 +9,27 @@ */ package org.eclipse.jgit.gpg.bc.internal; -import org.eclipse.jgit.lib.GpgSignatureVerifier; -import org.eclipse.jgit.lib.GpgSignatureVerifierFactory; +import org.eclipse.jgit.lib.GpgConfig.GpgFormat; +import org.eclipse.jgit.lib.SignatureVerifier; +import org.eclipse.jgit.lib.SignatureVerifierFactory; /** - * A {@link GpgSignatureVerifierFactory} that creates - * {@link GpgSignatureVerifier} instances that verify GPG signatures using - * BouncyCastle and that do cache public keys. + * A {@link SignatureVerifierFactory} that creates {@link SignatureVerifier} + * instances that verify GPG signatures using BouncyCastle and that do cache + * public keys. */ public final class BouncyCastleGpgSignatureVerifierFactory - extends GpgSignatureVerifierFactory { + implements SignatureVerifierFactory { @Override - public GpgSignatureVerifier getVerifier() { + public GpgFormat getType() { + return GpgFormat.OPENPGP; + } + + @Override + public SignatureVerifier create() { return new BouncyCastleGpgSignatureVerifier(); } + } diff --git a/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/BouncyCastleGpgSigner.java b/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/BouncyCastleGpgSigner.java index 763b7f7526..1d187a5db2 100644 --- a/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/BouncyCastleGpgSigner.java +++ b/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/BouncyCastleGpgSigner.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018, 2021, Salesforce and others + * Copyright (C) 2018, 2024, Salesforce and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -14,13 +14,11 @@ import java.io.IOException; import java.net.URISyntaxException; import java.security.NoSuchAlgorithmException; import java.security.NoSuchProviderException; -import java.security.Security; import java.util.Iterator; import org.bouncycastle.bcpg.ArmoredOutputStream; import org.bouncycastle.bcpg.BCPGOutputStream; import org.bouncycastle.bcpg.HashAlgorithmTags; -import org.bouncycastle.jce.provider.BouncyCastleProvider; import org.bouncycastle.openpgp.PGPException; import org.bouncycastle.openpgp.PGPPrivateKey; import org.bouncycastle.openpgp.PGPPublicKey; @@ -30,79 +28,23 @@ import org.bouncycastle.openpgp.PGPSignatureGenerator; import org.bouncycastle.openpgp.PGPSignatureSubpacketGenerator; import org.bouncycastle.openpgp.operator.jcajce.JcaPGPContentSignerBuilder; import org.bouncycastle.openpgp.operator.jcajce.JcePBESecretKeyDecryptorBuilder; -import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.api.errors.CanceledException; import org.eclipse.jgit.api.errors.JGitInternalException; import org.eclipse.jgit.api.errors.UnsupportedSigningFormatException; import org.eclipse.jgit.errors.UnsupportedCredentialItem; -import org.eclipse.jgit.internal.JGitText; -import org.eclipse.jgit.lib.CommitBuilder; import org.eclipse.jgit.lib.GpgConfig; -import org.eclipse.jgit.lib.GpgObjectSigner; import org.eclipse.jgit.lib.GpgSignature; -import org.eclipse.jgit.lib.GpgSigner; -import org.eclipse.jgit.lib.ObjectBuilder; import org.eclipse.jgit.lib.PersonIdent; -import org.eclipse.jgit.lib.GpgConfig.GpgFormat; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.lib.Signer; import org.eclipse.jgit.transport.CredentialsProvider; import org.eclipse.jgit.util.StringUtils; /** * GPG Signer using the BouncyCastle library. */ -public class BouncyCastleGpgSigner extends GpgSigner - implements GpgObjectSigner { - - private static void registerBouncyCastleProviderIfNecessary() { - if (Security.getProvider(BouncyCastleProvider.PROVIDER_NAME) == null) { - Security.addProvider(new BouncyCastleProvider()); - } - } - - /** - * Create a new instance. - *

- * The BounceCastleProvider will be registered if necessary. - *

- */ - public BouncyCastleGpgSigner() { - registerBouncyCastleProviderIfNecessary(); - } - - @Override - public boolean canLocateSigningKey(@Nullable String gpgSigningKey, - PersonIdent committer, CredentialsProvider credentialsProvider) - throws CanceledException { - try { - return canLocateSigningKey(gpgSigningKey, committer, - credentialsProvider, null); - } catch (UnsupportedSigningFormatException e) { - // Cannot occur with a null config - return false; - } - } - - @Override - public boolean canLocateSigningKey(@Nullable String gpgSigningKey, - PersonIdent committer, CredentialsProvider credentialsProvider, - GpgConfig config) - throws CanceledException, UnsupportedSigningFormatException { - if (config != null && config.getKeyFormat() != GpgFormat.OPENPGP) { - throw new UnsupportedSigningFormatException( - JGitText.get().onlyOpenPgpSupportedForSigning); - } - try (BouncyCastleGpgKeyPassphrasePrompt passphrasePrompt = new BouncyCastleGpgKeyPassphrasePrompt( - credentialsProvider)) { - BouncyCastleGpgKey gpgKey = locateSigningKey(gpgSigningKey, - committer, passphrasePrompt); - return gpgKey != null; - } catch (CanceledException e) { - throw e; - } catch (Exception e) { - return false; - } - } +public class BouncyCastleGpgSigner implements Signer { private BouncyCastleGpgKey locateSigningKey(@Nullable String gpgSigningKey, PersonIdent committer, @@ -121,38 +63,24 @@ public class BouncyCastleGpgSigner extends GpgSigner } @Override - public void sign(@NonNull CommitBuilder commit, - @Nullable String gpgSigningKey, @NonNull PersonIdent committer, - CredentialsProvider credentialsProvider) throws CanceledException { - try { - signObject(commit, gpgSigningKey, committer, credentialsProvider, - null); - } catch (UnsupportedSigningFormatException e) { - // Cannot occur with a null config - } - } - - @Override - public void signObject(@NonNull ObjectBuilder object, - @Nullable String gpgSigningKey, @NonNull PersonIdent committer, - CredentialsProvider credentialsProvider, GpgConfig config) - throws CanceledException, UnsupportedSigningFormatException { - if (config != null && config.getKeyFormat() != GpgFormat.OPENPGP) { - throw new UnsupportedSigningFormatException( - JGitText.get().onlyOpenPgpSupportedForSigning); + public GpgSignature sign(Repository repository, GpgConfig config, + byte[] data, PersonIdent committer, String signingKey, + CredentialsProvider credentialsProvider) throws CanceledException, + IOException, UnsupportedSigningFormatException { + String gpgSigningKey = signingKey; + if (gpgSigningKey == null) { + gpgSigningKey = config.getSigningKey(); } try (BouncyCastleGpgKeyPassphrasePrompt passphrasePrompt = new BouncyCastleGpgKeyPassphrasePrompt( credentialsProvider)) { BouncyCastleGpgKey gpgKey = locateSigningKey(gpgSigningKey, - committer, - passphrasePrompt); + committer, passphrasePrompt); PGPSecretKey secretKey = gpgKey.getSecretKey(); if (secretKey == null) { throw new JGitInternalException( BCText.get().unableToSignCommitNoSecretKey); } - JcePBESecretKeyDecryptorBuilder decryptorBuilder = new JcePBESecretKeyDecryptorBuilder() - .setProvider(BouncyCastleProvider.PROVIDER_NAME); + JcePBESecretKeyDecryptorBuilder decryptorBuilder = new JcePBESecretKeyDecryptorBuilder(); PGPPrivateKey privateKey = null; if (!passphrasePrompt.hasPassphrase()) { // Either the key is not encrypted, or it was read from the @@ -177,8 +105,7 @@ public class BouncyCastleGpgSigner extends GpgSigner PGPSignatureGenerator signatureGenerator = new PGPSignatureGenerator( new JcaPGPContentSignerBuilder( publicKey.getAlgorithm(), - HashAlgorithmTags.SHA256).setProvider( - BouncyCastleProvider.PROVIDER_NAME)); + HashAlgorithmTags.SHA256)); signatureGenerator.init(PGPSignature.BINARY_DOCUMENT, privateKey); PGPSignatureSubpacketGenerator subpackets = new PGPSignatureSubpacketGenerator(); subpackets.setIssuerFingerprint(false, publicKey); @@ -202,16 +129,36 @@ public class BouncyCastleGpgSigner extends GpgSigner ByteArrayOutputStream buffer = new ByteArrayOutputStream(); try (BCPGOutputStream out = new BCPGOutputStream( new ArmoredOutputStream(buffer))) { - signatureGenerator.update(object.build()); + signatureGenerator.update(data); signatureGenerator.generate().encode(out); } - object.setGpgSignature(new GpgSignature(buffer.toByteArray())); - } catch (PGPException | IOException | NoSuchAlgorithmException + return new GpgSignature(buffer.toByteArray()); + } catch (PGPException | NoSuchAlgorithmException | NoSuchProviderException | URISyntaxException e) { throw new JGitInternalException(e.getMessage(), e); } } + @Override + public boolean canLocateSigningKey(Repository repository, GpgConfig config, + PersonIdent committer, String signingKey, + CredentialsProvider credentialsProvider) throws CanceledException { + String gpgSigningKey = signingKey; + if (gpgSigningKey == null) { + gpgSigningKey = config.getSigningKey(); + } + try (BouncyCastleGpgKeyPassphrasePrompt passphrasePrompt = new BouncyCastleGpgKeyPassphrasePrompt( + credentialsProvider)) { + BouncyCastleGpgKey gpgKey = locateSigningKey(gpgSigningKey, + committer, passphrasePrompt); + return gpgKey != null; + } catch (CanceledException e) { + throw e; + } catch (Exception e) { + return false; + } + } + static String extractSignerId(String pgpUserId) { int from = pgpUserId.indexOf('<'); if (from >= 0) { diff --git a/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/BouncyCastleGpgSignerFactory.java b/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/BouncyCastleGpgSignerFactory.java new file mode 100644 index 0000000000..92ab65d7e4 --- /dev/null +++ b/org.eclipse.jgit.gpg.bc/src/org/eclipse/jgit/gpg/bc/internal/BouncyCastleGpgSignerFactory.java @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2021, 2024 Thomas Wolf and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.gpg.bc.internal; + +import org.eclipse.jgit.lib.GpgConfig.GpgFormat; +import org.eclipse.jgit.lib.Signer; +import org.eclipse.jgit.lib.SignerFactory; + +/** + * Factory for creating a {@link Signer} for OPENPGP signatures based on Bouncy + * Castle. + */ +public final class BouncyCastleGpgSignerFactory implements SignerFactory { + + @Override + public GpgFormat getType() { + return GpgFormat.OPENPGP; + } + + @Override + public Signer create() { + return new BouncyCastleGpgSigner(); + } +} diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Log.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Log.java index 852a4b377b..958e566986 100644 --- a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Log.java +++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Log.java @@ -32,13 +32,12 @@ import org.eclipse.jgit.errors.LargeObjectException; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.GpgConfig; -import org.eclipse.jgit.lib.GpgSignatureVerifier; -import org.eclipse.jgit.lib.GpgSignatureVerifier.SignatureVerification; -import org.eclipse.jgit.lib.GpgSignatureVerifierFactory; +import org.eclipse.jgit.lib.SignatureVerifier.SignatureVerification; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.PersonIdent; import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.lib.SignatureVerifiers; import org.eclipse.jgit.notes.NoteMap; import org.eclipse.jgit.pgm.internal.CLIText; import org.eclipse.jgit.pgm.internal.VerificationUtils; @@ -174,8 +173,6 @@ class Log extends RevWalkTextBuiltin { // END -- Options shared with Diff - private GpgSignatureVerifier verifier; - private GpgConfig config; Log() { @@ -227,9 +224,6 @@ class Log extends RevWalkTextBuiltin { throw die(e.getMessage(), e); } finally { diffFmt.close(); - if (verifier != null) { - verifier.clear(); - } } } @@ -293,21 +287,13 @@ class Log extends RevWalkTextBuiltin { if (c.getRawGpgSignature() == null) { return; } - if (verifier == null) { - GpgSignatureVerifierFactory factory = GpgSignatureVerifierFactory - .getDefault(); - if (factory == null) { - throw die(CLIText.get().logNoSignatureVerifier, null); - } - verifier = factory.getVerifier(); - } - SignatureVerification verification = verifier.verifySignature(c, - config); + SignatureVerification verification = SignatureVerifiers.verify(db, + config, c); if (verification == null) { return; } VerificationUtils.writeVerification(outw, verification, - verifier.getName(), c.getCommitterIdent()); + verification.verifierName(), c.getCommitterIdent()); } /** diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Show.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Show.java index 4feb090032..1576792234 100644 --- a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Show.java +++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Show.java @@ -30,12 +30,11 @@ import org.eclipse.jgit.errors.RevisionSyntaxException; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.FileMode; import org.eclipse.jgit.lib.GpgConfig; -import org.eclipse.jgit.lib.GpgSignatureVerifier; -import org.eclipse.jgit.lib.GpgSignatureVerifierFactory; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.PersonIdent; import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.lib.GpgSignatureVerifier.SignatureVerification; +import org.eclipse.jgit.lib.SignatureVerifier.SignatureVerification; +import org.eclipse.jgit.lib.SignatureVerifiers; import org.eclipse.jgit.pgm.internal.CLIText; import org.eclipse.jgit.pgm.internal.VerificationUtils; import org.eclipse.jgit.pgm.opt.PathTreeFilterHandler; @@ -335,23 +334,13 @@ class Show extends TextBuiltin { if (c.getRawGpgSignature() == null) { return; } - GpgSignatureVerifierFactory factory = GpgSignatureVerifierFactory - .getDefault(); - if (factory == null) { - throw die(CLIText.get().logNoSignatureVerifier, null); - } - GpgSignatureVerifier verifier = factory.getVerifier(); GpgConfig config = new GpgConfig(db.getConfig()); - try { - SignatureVerification verification = verifier.verifySignature(c, - config); - if (verification == null) { - return; - } - VerificationUtils.writeVerification(outw, verification, - verifier.getName(), c.getCommitterIdent()); - } finally { - verifier.clear(); + SignatureVerification verification = SignatureVerifiers.verify(db, + config, c); + if (verification == null) { + throw die(CLIText.get().logNoSignatureVerifier, null); } + VerificationUtils.writeVerification(outw, verification, + verification.verifierName(), c.getCommitterIdent()); } } diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Tag.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Tag.java index 4ea67ab92c..6be30c9447 100644 --- a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Tag.java +++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Tag.java @@ -27,10 +27,10 @@ import org.eclipse.jgit.api.VerifySignatureCommand; import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.api.errors.RefAlreadyExistsException; import org.eclipse.jgit.lib.Constants; -import org.eclipse.jgit.lib.GpgSignatureVerifier.SignatureVerification; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.lib.SignatureVerifier.SignatureVerification; import org.eclipse.jgit.pgm.internal.CLIText; import org.eclipse.jgit.pgm.internal.VerificationUtils; import org.eclipse.jgit.revwalk.RevCommit; @@ -106,7 +106,8 @@ class Tag extends TextBuiltin { if (error != null) { throw die(error.getMessage(), error); } - writeVerification(verifySig.getVerifier().getName(), + writeVerification( + verification.getVerification().verifierName(), (RevTag) verification.getObject(), verification.getVerification()); } diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/internal/VerificationUtils.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/internal/VerificationUtils.java index c1f8a86a8c..64ee602620 100644 --- a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/internal/VerificationUtils.java +++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/internal/VerificationUtils.java @@ -11,7 +11,7 @@ package org.eclipse.jgit.pgm.internal; import java.io.IOException; -import org.eclipse.jgit.lib.GpgSignatureVerifier.SignatureVerification; +import org.eclipse.jgit.lib.SignatureVerifier.SignatureVerification; import org.eclipse.jgit.lib.PersonIdent; import org.eclipse.jgit.util.GitDateFormatter; import org.eclipse.jgit.util.SignatureUtils; diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CommitCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CommitCommandTest.java index 35de73e204..e74e234297 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CommitCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CommitCommandTest.java @@ -27,6 +27,7 @@ import java.util.concurrent.atomic.AtomicInteger; import org.eclipse.jgit.api.CherryPickResult.CherryPickStatus; import org.eclipse.jgit.api.errors.CanceledException; import org.eclipse.jgit.api.errors.EmptyCommitException; +import org.eclipse.jgit.api.errors.UnsupportedSigningFormatException; import org.eclipse.jgit.api.errors.WrongRepositoryStateException; import org.eclipse.jgit.diff.DiffEntry; import org.eclipse.jgit.dircache.DirCache; @@ -34,19 +35,23 @@ import org.eclipse.jgit.dircache.DirCacheBuilder; import org.eclipse.jgit.dircache.DirCacheEntry; import org.eclipse.jgit.junit.RepositoryTestCase; import org.eclipse.jgit.junit.time.TimeUtil; -import org.eclipse.jgit.lib.CommitBuilder; +import org.eclipse.jgit.lib.CommitConfig.CleanupMode; import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.FileMode; -import org.eclipse.jgit.lib.GpgSigner; +import org.eclipse.jgit.lib.GpgConfig; +import org.eclipse.jgit.lib.GpgConfig.GpgFormat; +import org.eclipse.jgit.lib.GpgSignature; +import org.eclipse.jgit.lib.ObjectBuilder; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.PersonIdent; import org.eclipse.jgit.lib.RefUpdate; import org.eclipse.jgit.lib.RefUpdate.Result; import org.eclipse.jgit.lib.ReflogEntry; import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.lib.Signer; +import org.eclipse.jgit.lib.Signers; import org.eclipse.jgit.lib.StoredConfig; -import org.eclipse.jgit.lib.CommitConfig.CleanupMode; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.storage.file.FileBasedConfig; import org.eclipse.jgit.submodule.SubmoduleWalk; @@ -839,21 +844,39 @@ public class CommitCommandTest extends RepositoryTestCase { String[] signingKey = new String[1]; PersonIdent[] signingCommitters = new PersonIdent[1]; AtomicInteger callCount = new AtomicInteger(); - GpgSigner.setDefault(new GpgSigner() { + // Since GpgFormat defaults to OpenPGP just set a new signer for + // that. + Signers.set(GpgFormat.OPENPGP, new Signer() { + @Override - public void sign(CommitBuilder commit, String gpgSigningKey, - PersonIdent signingCommitter, CredentialsProvider credentialsProvider) { - signingKey[0] = gpgSigningKey; + public void signObject(Repository repo, GpgConfig config, + ObjectBuilder builder, PersonIdent signingCommitter, + String signingKeySpec, + CredentialsProvider credentialsProvider) + throws CanceledException, + UnsupportedSigningFormatException { + signingKey[0] = signingKeySpec; signingCommitters[0] = signingCommitter; callCount.incrementAndGet(); } @Override - public boolean canLocateSigningKey(String gpgSigningKey, - PersonIdent signingCommitter, + public GpgSignature sign(Repository repo, GpgConfig config, + byte[] data, PersonIdent signingCommitter, + String signingKeySpec, + CredentialsProvider credentialsProvider) + throws CanceledException, + UnsupportedSigningFormatException { + throw new CanceledException("Unexpected call"); + } + + @Override + public boolean canLocateSigningKey(Repository repo, + GpgConfig config, PersonIdent signingCommitter, + String signingKeySpec, CredentialsProvider credentialsProvider) throws CanceledException { - return false; + throw new CanceledException("Unexpected call"); } }); @@ -904,19 +927,37 @@ public class CommitCommandTest extends RepositoryTestCase { git.add().addFilepattern("file1").call(); AtomicInteger callCount = new AtomicInteger(); - GpgSigner.setDefault(new GpgSigner() { + // Since GpgFormat defaults to OpenPGP just set a new signer for + // that. + Signers.set(GpgFormat.OPENPGP, new Signer() { + @Override - public void sign(CommitBuilder commit, String gpgSigningKey, - PersonIdent signingCommitter, CredentialsProvider credentialsProvider) { + public void signObject(Repository repo, GpgConfig config, + ObjectBuilder builder, PersonIdent signingCommitter, + String signingKeySpec, + CredentialsProvider credentialsProvider) + throws CanceledException, + UnsupportedSigningFormatException { callCount.incrementAndGet(); } @Override - public boolean canLocateSigningKey(String gpgSigningKey, - PersonIdent signingCommitter, + public GpgSignature sign(Repository repo, GpgConfig config, + byte[] data, PersonIdent signingCommitter, + String signingKeySpec, + CredentialsProvider credentialsProvider) + throws CanceledException, + UnsupportedSigningFormatException { + throw new CanceledException("Unexpected call"); + } + + @Override + public boolean canLocateSigningKey(Repository repo, + GpgConfig config, PersonIdent signingCommitter, + String signingKeySpec, CredentialsProvider credentialsProvider) throws CanceledException { - return false; + throw new CanceledException("Unexpected call"); } }); diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties index c9f7336609..11435b8ce4 100644 --- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties +++ b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties @@ -576,7 +576,6 @@ obtainingCommitsForCherryPick=Obtaining commits that need to be cherry-picked oldIdMustNotBeNull=Expected old ID must not be null onlyOneFetchSupported=Only one fetch supported onlyOneOperationCallPerConnectionIsSupported=Only one operation call per connection is supported. -onlyOpenPgpSupportedForSigning=OpenPGP is the only supported signing option with JGit at this time (gpg.format must be set to openpgp). openFilesMustBeAtLeast1=Open files must be >= 1 openingConnection=Opening connection operationCanceled=Operation {0} was canceled @@ -723,6 +722,8 @@ shortSkipOfBlock=Short skip of block. shutdownCleanup=Cleanup {} during JVM shutdown shutdownCleanupFailed=Cleanup during JVM shutdown failed shutdownCleanupListenerFailed=Cleanup of {0} during JVM shutdown failed +signatureServiceConflict={0} conflict for type {1}. Already registered is {2}; additional factory {3} is ignored. +signatureTypeUnknown=No signer for {0} signatures. Use another signature type for git config gpg.format, or do not sign. signatureVerificationError=Signature verification failed signatureVerificationUnavailable=No signature verifier registered signedTagMessageNoLf=A non-empty message of a signed tag must end in LF. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/CommitCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/CommitCommand.java index a1a2cc09d2..a7d409c3f5 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/CommitCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/CommitCommand.java @@ -51,9 +51,6 @@ import org.eclipse.jgit.lib.CommitConfig.CleanupMode; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.FileMode; import org.eclipse.jgit.lib.GpgConfig; -import org.eclipse.jgit.lib.GpgConfig.GpgFormat; -import org.eclipse.jgit.lib.GpgObjectSigner; -import org.eclipse.jgit.lib.GpgSigner; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectInserter; import org.eclipse.jgit.lib.PersonIdent; @@ -62,6 +59,8 @@ import org.eclipse.jgit.lib.RefUpdate; import org.eclipse.jgit.lib.RefUpdate.Result; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.lib.RepositoryState; +import org.eclipse.jgit.lib.Signer; +import org.eclipse.jgit.lib.Signers; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.RevObject; import org.eclipse.jgit.revwalk.RevTag; @@ -129,7 +128,7 @@ public class CommitCommand extends GitCommand { private String signingKey; - private GpgSigner gpgSigner; + private Signer signer; private GpgConfig gpgConfig; @@ -319,30 +318,22 @@ public class CommitCommand extends GitCommand { } } - private void sign(CommitBuilder commit) throws ServiceUnavailableException, - CanceledException, UnsupportedSigningFormatException { - if (gpgSigner == null) { - gpgSigner = GpgSigner.getDefault(); - if (gpgSigner == null) { - throw new ServiceUnavailableException( - JGitText.get().signingServiceUnavailable); + private void sign(CommitBuilder commit) + throws CanceledException, IOException, + UnsupportedSigningFormatException { + if (signer == null) { + signer = Signers.get(gpgConfig.getKeyFormat()); + if (signer == null) { + throw new UnsupportedSigningFormatException(MessageFormat + .format(JGitText.get().signatureTypeUnknown, + gpgConfig.getKeyFormat().toConfigValue())); } } if (signingKey == null) { signingKey = gpgConfig.getSigningKey(); } - if (gpgSigner instanceof GpgObjectSigner) { - ((GpgObjectSigner) gpgSigner).signObject(commit, - signingKey, committer, credentialsProvider, - gpgConfig); - } else { - if (gpgConfig.getKeyFormat() != GpgFormat.OPENPGP) { - throw new UnsupportedSigningFormatException(JGitText - .get().onlyOpenPgpSupportedForSigning); - } - gpgSigner.sign(commit, signingKey, committer, - credentialsProvider); - } + signer.signObject(repo, gpgConfig, commit, committer, signingKey, + credentialsProvider); } private void updateRef(RepositoryState state, ObjectId headId, @@ -1097,22 +1088,22 @@ public class CommitCommand extends GitCommand { } /** - * Sets the {@link GpgSigner} to use if the commit is to be signed. + * Sets the {@link Signer} to use if the commit is to be signed. * * @param signer * to use; if {@code null}, the default signer will be used * @return {@code this} - * @since 5.11 + * @since 7.0 */ - public CommitCommand setGpgSigner(GpgSigner signer) { + public CommitCommand setSigner(Signer signer) { checkCallable(); - this.gpgSigner = signer; + this.signer = signer; return this; } /** * Sets an external {@link GpgConfig} to use. Whether it will be used is at - * the discretion of the {@link #setGpgSigner(GpgSigner)}. + * the discretion of the {@link #setSigner(Signer)}. * * @param config * to set; if {@code null}, the config will be loaded from the diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/TagCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/TagCommand.java index 3edaf5e748..cc8589fa1c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/TagCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/TagCommand.java @@ -18,14 +18,11 @@ import org.eclipse.jgit.api.errors.InvalidTagNameException; import org.eclipse.jgit.api.errors.JGitInternalException; import org.eclipse.jgit.api.errors.NoHeadException; import org.eclipse.jgit.api.errors.RefAlreadyExistsException; -import org.eclipse.jgit.api.errors.ServiceUnavailableException; import org.eclipse.jgit.api.errors.UnsupportedSigningFormatException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.GpgConfig; import org.eclipse.jgit.lib.GpgConfig.GpgFormat; -import org.eclipse.jgit.lib.GpgObjectSigner; -import org.eclipse.jgit.lib.GpgSigner; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectInserter; import org.eclipse.jgit.lib.PersonIdent; @@ -33,6 +30,8 @@ import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.RefUpdate; import org.eclipse.jgit.lib.RefUpdate.Result; import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.lib.Signer; +import org.eclipse.jgit.lib.Signers; import org.eclipse.jgit.lib.TagBuilder; import org.eclipse.jgit.revwalk.RevObject; import org.eclipse.jgit.revwalk.RevWalk; @@ -79,7 +78,7 @@ public class TagCommand extends GitCommand { private GpgConfig gpgConfig; - private GpgObjectSigner gpgSigner; + private Signer signer; private CredentialsProvider credentialsProvider; @@ -133,9 +132,9 @@ public class TagCommand extends GitCommand { newTag.setTagger(tagger); newTag.setObjectId(id); - if (gpgSigner != null) { - gpgSigner.signObject(newTag, signingKey, tagger, - credentialsProvider, gpgConfig); + if (signer != null) { + signer.signObject(repo, gpgConfig, newTag, tagger, signingKey, + credentialsProvider); } // write the tag object @@ -196,15 +195,12 @@ public class TagCommand extends GitCommand { * * @throws InvalidTagNameException * if the tag name is null or invalid - * @throws ServiceUnavailableException - * if the tag should be signed but no signer can be found * @throws UnsupportedSigningFormatException * if the tag should be signed but {@code gpg.format} is not * {@link GpgFormat#OPENPGP} */ private void processOptions() - throws InvalidTagNameException, ServiceUnavailableException, - UnsupportedSigningFormatException { + throws InvalidTagNameException, UnsupportedSigningFormatException { if (name == null || !Repository.isValidRefName(Constants.R_TAGS + name)) { throw new InvalidTagNameException( @@ -230,16 +226,15 @@ public class TagCommand extends GitCommand { doSign = gpgConfig.isSignAnnotated(); } if (doSign) { - if (signingKey == null) { - signingKey = gpgConfig.getSigningKey(); - } - if (gpgSigner == null) { - GpgSigner signer = GpgSigner.getDefault(); - if (!(signer instanceof GpgObjectSigner)) { - throw new ServiceUnavailableException( - JGitText.get().signingServiceUnavailable); + if (signer == null) { + signer = Signers.get(gpgConfig.getKeyFormat()); + if (signer == null) { + throw new UnsupportedSigningFormatException( + MessageFormat.format( + JGitText.get().signatureTypeUnknown, + gpgConfig.getKeyFormat() + .toConfigValue())); } - gpgSigner = (GpgObjectSigner) signer; } // The message of a signed tag must end in a newline because // the signature will be appended. @@ -326,22 +321,22 @@ public class TagCommand extends GitCommand { } /** - * Sets the {@link GpgSigner} to use if the commit is to be signed. + * Sets the {@link Signer} to use if the commit is to be signed. * * @param signer * to use; if {@code null}, the default signer will be used * @return {@code this} - * @since 5.11 + * @since 7.0 */ - public TagCommand setGpgSigner(GpgObjectSigner signer) { + public TagCommand setSigner(Signer signer) { checkCallable(); - this.gpgSigner = signer; + this.signer = signer; return this; } /** * Sets an external {@link GpgConfig} to use. Whether it will be used is at - * the discretion of the {@link #setGpgSigner(GpgObjectSigner)}. + * the discretion of the {@link #setSigner(Signer)}. * * @param config * to set; if {@code null}, the config will be loaded from the diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/VerificationResult.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/VerificationResult.java index 21cddf75b7..f5f4b06e45 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/VerificationResult.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/VerificationResult.java @@ -9,7 +9,7 @@ */ package org.eclipse.jgit.api; -import org.eclipse.jgit.lib.GpgSignatureVerifier; +import org.eclipse.jgit.lib.SignatureVerifier; import org.eclipse.jgit.revwalk.RevObject; /** @@ -34,8 +34,9 @@ public interface VerificationResult { * Retrieves the signature verification result. * * @return the result, or {@code null} if none was computed + * @since 7.0 */ - GpgSignatureVerifier.SignatureVerification getVerification(); + SignatureVerifier.SignatureVerification getVerification(); /** * Retrieves the git object of which the signature was verified. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/VerifySignatureCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/VerifySignatureCommand.java index 6a2a44ea2d..487ff04323 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/VerifySignatureCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/VerifySignatureCommand.java @@ -25,11 +25,10 @@ import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.GpgConfig; -import org.eclipse.jgit.lib.GpgSignatureVerifier; -import org.eclipse.jgit.lib.GpgSignatureVerifier.SignatureVerification; -import org.eclipse.jgit.lib.GpgSignatureVerifierFactory; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.lib.SignatureVerifier.SignatureVerification; +import org.eclipse.jgit.lib.SignatureVerifiers; import org.eclipse.jgit.revwalk.RevObject; import org.eclipse.jgit.revwalk.RevWalk; @@ -65,12 +64,8 @@ public class VerifySignatureCommand extends GitCommand result = new HashMap<>(); - if (verifier == null) { - GpgSignatureVerifierFactory factory = GpgSignatureVerifierFactory - .getDefault(); - if (factory == null) { - throw new ServiceUnavailableException( - JGitText.get().signatureVerificationUnavailable); - } - verifier = factory.getVerifier(); - ownVerifier = true; - } if (config == null) { config = new GpgConfig(repo.getConfig()); } @@ -239,10 +196,6 @@ public class VerifySignatureCommand extends GitCommand and others -* -* This program and the accompanying materials are made available under the -* terms of the Eclipse Distribution License v. 1.0 which is available at -* https://www.eclipse.org/org/documents/edl-v10.php. -* -* SPDX-License-Identifier: BSD-3-Clause -*/ -package org.eclipse.jgit.lib; - -import java.io.IOException; -import java.util.Arrays; - -import org.eclipse.jgit.revwalk.RevCommit; -import org.eclipse.jgit.revwalk.RevObject; -import org.eclipse.jgit.revwalk.RevTag; -import org.eclipse.jgit.util.RawParseUtils; - -/** - * Provides a base implementation of - * {@link GpgSignatureVerifier#verifySignature(RevObject, GpgConfig)}. - * - * @since 6.9 - */ -public abstract class AbstractGpgSignatureVerifier - implements GpgSignatureVerifier { - - @Override - public SignatureVerification verifySignature(RevObject object, - GpgConfig config) throws IOException { - if (object instanceof RevCommit) { - RevCommit commit = (RevCommit) object; - byte[] signatureData = commit.getRawGpgSignature(); - if (signatureData == null) { - return null; - } - byte[] raw = commit.getRawBuffer(); - // Now remove the GPG signature - byte[] header = { 'g', 'p', 'g', 's', 'i', 'g' }; - int start = RawParseUtils.headerStart(header, raw, 0); - if (start < 0) { - return null; - } - int end = RawParseUtils.nextLfSkippingSplitLines(raw, start); - // start is at the beginning of the header's content - start -= header.length + 1; - // end is on the terminating LF; we need to skip that, too - if (end < raw.length) { - end++; - } - byte[] data = new byte[raw.length - (end - start)]; - System.arraycopy(raw, 0, data, 0, start); - System.arraycopy(raw, end, data, start, raw.length - end); - return verify(config, data, signatureData); - } else if (object instanceof RevTag) { - RevTag tag = (RevTag) object; - byte[] signatureData = tag.getRawGpgSignature(); - if (signatureData == null) { - return null; - } - byte[] raw = tag.getRawBuffer(); - // The signature is just tacked onto the end of the message, which - // is last in the buffer. - byte[] data = Arrays.copyOfRange(raw, 0, - raw.length - signatureData.length); - return verify(config, data, signatureData); - } - return null; - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java index b9c90bda30..8599bf7dbc 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java @@ -552,6 +552,20 @@ public final class Constants { */ public static final String GPG_SIGNATURE_PREFIX = "-----BEGIN PGP SIGNATURE-----"; //$NON-NLS-1$ + /** + * Prefix of a CMS signature (X.509, S/MIME). + * + * @since 7.0 + */ + public static final String CMS_SIGNATURE_PREFIX = "-----BEGIN SIGNED MESSAGE-----"; //$NON-NLS-1$ + + /** + * Prefix of an SSH signature. + * + * @since 7.0 + */ + public static final String SSH_SIGNATURE_PREFIX = "-----BEGIN SSH SIGNATURE-----"; //$NON-NLS-1$ + /** * Create a new digest function for objects. * diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgConfig.java index f5064df061..fb5c904215 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgConfig.java @@ -61,27 +61,6 @@ public class GpgConfig { private final boolean forceAnnotated; - /** - * Create a {@link GpgConfig} with the given parameters and default - * {@code true} for signing commits and {@code false} for tags. - * - * @param keySpec - * to use - * @param format - * to use - * @param gpgProgram - * to use - * @since 5.11 - */ - public GpgConfig(String keySpec, GpgFormat format, String gpgProgram) { - keyFormat = format; - signingKey = keySpec; - program = gpgProgram; - signCommits = true; - signAllTags = false; - forceAnnotated = false; - } - /** * Create a new GPG config that reads the configuration from config. * @@ -97,10 +76,11 @@ public class GpgConfig { String exe = config.getString(ConfigConstants.CONFIG_GPG_SECTION, keyFormat.toConfigValue(), ConfigConstants.CONFIG_KEY_PROGRAM); - if (exe == null) { + if (exe == null && GpgFormat.OPENPGP.equals(keyFormat)) { exe = config.getString(ConfigConstants.CONFIG_GPG_SECTION, null, ConfigConstants.CONFIG_KEY_PROGRAM); } + program = exe; signCommits = config.getBoolean(ConfigConstants.CONFIG_COMMIT_SECTION, ConfigConstants.CONFIG_KEY_GPGSIGN, false); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgObjectSigner.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgObjectSigner.java deleted file mode 100644 index 074f46567b..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgObjectSigner.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright (C) 2020 Thomas Wolf and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ -package org.eclipse.jgit.lib; - -import org.eclipse.jgit.annotations.NonNull; -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.api.errors.CanceledException; -import org.eclipse.jgit.api.errors.UnsupportedSigningFormatException; -import org.eclipse.jgit.transport.CredentialsProvider; - -/** - * Creates GPG signatures for Git objects. - * - * @since 5.11 - */ -public interface GpgObjectSigner { - - /** - * Signs the specified object. - * - *

- * Implementors should obtain the payload for signing from the specified - * object via {@link ObjectBuilder#build()} and create a proper - * {@link GpgSignature}. The generated signature must be set on the - * specified {@code object} (see - * {@link ObjectBuilder#setGpgSignature(GpgSignature)}). - *

- *

- * Any existing signature on the object must be discarded prior obtaining - * the payload via {@link ObjectBuilder#build()}. - *

- * - * @param object - * the object to sign (must not be {@code null} and must be - * complete to allow proper calculation of payload) - * @param gpgSigningKey - * the signing key to locate (passed as is to the GPG signing - * tool as is; eg., value of user.signingkey) - * @param committer - * the signing identity (to help with key lookup in case signing - * key is not specified) - * @param credentialsProvider - * provider to use when querying for signing key credentials (eg. - * passphrase) - * @param config - * GPG settings from the git config - * @throws CanceledException - * when signing was canceled (eg., user aborted when entering - * passphrase) - * @throws UnsupportedSigningFormatException - * if a config is given and the wanted key format is not - * supported - */ - void signObject(@NonNull ObjectBuilder object, - @Nullable String gpgSigningKey, @NonNull PersonIdent committer, - CredentialsProvider credentialsProvider, GpgConfig config) - throws CanceledException, UnsupportedSigningFormatException; - - /** - * Indicates if a signing key is available for the specified committer - * and/or signing key. - * - * @param gpgSigningKey - * the signing key to locate (passed as is to the GPG signing - * tool as is; eg., value of user.signingkey) - * @param committer - * the signing identity (to help with key lookup in case signing - * key is not specified) - * @param credentialsProvider - * provider to use when querying for signing key credentials (eg. - * passphrase) - * @param config - * GPG settings from the git config - * @return true if a signing key is available, - * false otherwise - * @throws CanceledException - * when signing was canceled (eg., user aborted when entering - * passphrase) - * @throws UnsupportedSigningFormatException - * if a config is given and the wanted key format is not - * supported - */ - public abstract boolean canLocateSigningKey(@Nullable String gpgSigningKey, - @NonNull PersonIdent committer, - CredentialsProvider credentialsProvider, GpgConfig config) - throws CanceledException, UnsupportedSigningFormatException; - -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgSignatureVerifier.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgSignatureVerifier.java deleted file mode 100644 index 91c9bab5a4..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgSignatureVerifier.java +++ /dev/null @@ -1,184 +0,0 @@ -/* - * Copyright (C) 2021, 2024 Thomas Wolf and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ -package org.eclipse.jgit.lib; - -import java.io.IOException; -import java.util.Date; - -import org.eclipse.jgit.annotations.NonNull; -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.api.errors.JGitInternalException; -import org.eclipse.jgit.revwalk.RevObject; - -/** - * A {@code GpgSignatureVerifier} can verify GPG signatures on git commits and - * tags. - * - * @since 5.11 - */ -public interface GpgSignatureVerifier { - - /** - * Verifies the signature on a signed commit or tag. - * - * @param object - * to verify - * @param config - * the {@link GpgConfig} to use - * @return a {@link SignatureVerification} describing the outcome of the - * verification, or {@code null} if the object was not signed - * @throws IOException - * if an error occurs getting a public key - * @throws org.eclipse.jgit.api.errors.JGitInternalException - * if signature verification fails - */ - @Nullable - SignatureVerification verifySignature(@NonNull RevObject object, - @NonNull GpgConfig config) throws IOException; - - /** - * Verifies a given signature for given data. - * - * @param config - * the {@link GpgConfig} - * @param data - * the signature is for - * @param signatureData - * the ASCII-armored signature - * @return a {@link SignatureVerification} describing the outcome - * @throws IOException - * if the signature cannot be parsed - * @throws JGitInternalException - * if signature verification fails - * @since 6.9 - */ - default SignatureVerification verify(@NonNull GpgConfig config, byte[] data, - byte[] signatureData) throws IOException { - // Default implementation for backwards compatibility; override as - // appropriate - return verify(data, signatureData); - } - - /** - * Verifies a given signature for given data. - * - * @param data - * the signature is for - * @param signatureData - * the ASCII-armored signature - * @return a {@link SignatureVerification} describing the outcome - * @throws IOException - * if the signature cannot be parsed - * @throws JGitInternalException - * if signature verification fails - * @deprecated since 6.9, use {@link #verify(GpgConfig, byte[], byte[])} - * instead - */ - @Deprecated - public SignatureVerification verify(byte[] data, byte[] signatureData) - throws IOException; - - /** - * Retrieves the name of this verifier. This should be a short string - * identifying the engine that verified the signature, like "gpg" if GPG is - * used, or "bc" for a BouncyCastle implementation. - * - * @return the name - */ - @NonNull - String getName(); - - /** - * A {@link GpgSignatureVerifier} may cache public keys to speed up - * verifying signatures on multiple objects. This clears this cache, if any. - */ - void clear(); - - /** - * A {@code SignatureVerification} returns data about a (positively or - * negatively) verified signature. - */ - interface SignatureVerification { - - // Data about the signature. - - @NonNull - Date getCreationDate(); - - // Data from the signature used to find a public key. - - /** - * Obtains the signer as stored in the signature, if known. - * - * @return the signer, or {@code null} if unknown - */ - String getSigner(); - - /** - * Obtains the short or long fingerprint of the public key as stored in - * the signature, if known. - * - * @return the fingerprint, or {@code null} if unknown - */ - String getKeyFingerprint(); - - // Some information about the found public key. - - /** - * Obtains the OpenPGP user ID associated with the key. - * - * @return the user id, or {@code null} if unknown - */ - String getKeyUser(); - - /** - * Tells whether the public key used for this signature verification was - * expired when the signature was created. - * - * @return {@code true} if the key was expired already, {@code false} - * otherwise - */ - boolean isExpired(); - - /** - * Obtains the trust level of the public key used to verify the - * signature. - * - * @return the trust level - */ - @NonNull - TrustLevel getTrustLevel(); - - // The verification result. - - /** - * Tells whether the signature verification was successful. - * - * @return {@code true} if the signature was verified successfully; - * {@code false} if not. - */ - boolean getVerified(); - - /** - * Obtains a human-readable message giving additional information about - * the outcome of the verification. - * - * @return the message, or {@code null} if none set. - */ - String getMessage(); - } - - /** - * The owner's trust in a public key. - */ - enum TrustLevel { - UNKNOWN, NEVER, MARGINAL, FULL, ULTIMATE - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgSignatureVerifierFactory.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgSignatureVerifierFactory.java deleted file mode 100644 index 59775c475b..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgSignatureVerifierFactory.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright (C) 2021, 2022 Thomas Wolf and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ -package org.eclipse.jgit.lib; - -import java.util.Iterator; -import java.util.ServiceConfigurationError; -import java.util.ServiceLoader; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * A {@code GpgSignatureVerifierFactory} creates {@link GpgSignatureVerifier} instances. - * - * @since 5.11 - */ -public abstract class GpgSignatureVerifierFactory { - - private static final Logger LOG = LoggerFactory - .getLogger(GpgSignatureVerifierFactory.class); - - private static class DefaultFactory { - - private static volatile GpgSignatureVerifierFactory defaultFactory = loadDefault(); - - private static GpgSignatureVerifierFactory loadDefault() { - try { - ServiceLoader loader = ServiceLoader - .load(GpgSignatureVerifierFactory.class); - Iterator iter = loader.iterator(); - if (iter.hasNext()) { - return iter.next(); - } - } catch (ServiceConfigurationError e) { - LOG.error(e.getMessage(), e); - } - return null; - } - - private DefaultFactory() { - // No instantiation - } - - public static GpgSignatureVerifierFactory getDefault() { - return defaultFactory; - } - - /** - * Sets the default factory. - * - * @param factory - * the new default factory - */ - public static void setDefault(GpgSignatureVerifierFactory factory) { - defaultFactory = factory; - } - } - - /** - * Retrieves the default factory. - * - * @return the default factory or {@code null} if none set - */ - public static GpgSignatureVerifierFactory getDefault() { - return DefaultFactory.getDefault(); - } - - /** - * Sets the default factory. - * - * @param factory - * the new default factory - */ - public static void setDefault(GpgSignatureVerifierFactory factory) { - DefaultFactory.setDefault(factory); - } - - /** - * Creates a new {@link GpgSignatureVerifier}. - * - * @return the new {@link GpgSignatureVerifier} - */ - public abstract GpgSignatureVerifier getVerifier(); - -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgSigner.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgSigner.java deleted file mode 100644 index b25a61b506..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgSigner.java +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Copyright (C) 2018, 2022 Salesforce and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ -package org.eclipse.jgit.lib; - -import java.util.Iterator; -import java.util.ServiceConfigurationError; -import java.util.ServiceLoader; - -import org.eclipse.jgit.annotations.NonNull; -import org.eclipse.jgit.annotations.Nullable; -import org.eclipse.jgit.api.errors.CanceledException; -import org.eclipse.jgit.transport.CredentialsProvider; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Creates GPG signatures for Git objects. - * - * @since 5.3 - */ -public abstract class GpgSigner { - - private static final Logger LOG = LoggerFactory.getLogger(GpgSigner.class); - - private static class DefaultSigner { - - private static volatile GpgSigner defaultSigner = loadGpgSigner(); - - private static GpgSigner loadGpgSigner() { - try { - ServiceLoader loader = ServiceLoader - .load(GpgSigner.class); - Iterator iter = loader.iterator(); - if (iter.hasNext()) { - return iter.next(); - } - } catch (ServiceConfigurationError e) { - LOG.error(e.getMessage(), e); - } - return null; - } - - private DefaultSigner() { - // No instantiation - } - - public static GpgSigner getDefault() { - return defaultSigner; - } - - public static void setDefault(GpgSigner signer) { - defaultSigner = signer; - } - } - - /** - * Get the default signer, or null. - * - * @return the default signer, or null. - */ - public static GpgSigner getDefault() { - return DefaultSigner.getDefault(); - } - - /** - * Set the default signer. - * - * @param signer - * the new default signer, may be null to select no - * default. - */ - public static void setDefault(GpgSigner signer) { - DefaultSigner.setDefault(signer); - } - - /** - * Signs the specified commit. - * - *

- * Implementors should obtain the payload for signing from the specified - * commit via {@link CommitBuilder#build()} and create a proper - * {@link GpgSignature}. The generated signature must be set on the - * specified {@code commit} (see - * {@link CommitBuilder#setGpgSignature(GpgSignature)}). - *

- *

- * Any existing signature on the commit must be discarded prior obtaining - * the payload via {@link CommitBuilder#build()}. - *

- * - * @param commit - * the commit to sign (must not be null and must be - * complete to allow proper calculation of payload) - * @param gpgSigningKey - * the signing key to locate (passed as is to the GPG signing - * tool as is; eg., value of user.signingkey) - * @param committer - * the signing identity (to help with key lookup in case signing - * key is not specified) - * @param credentialsProvider - * provider to use when querying for signing key credentials (eg. - * passphrase) - * @throws CanceledException - * when signing was canceled (eg., user aborted when entering - * passphrase) - */ - public abstract void sign(@NonNull CommitBuilder commit, - @Nullable String gpgSigningKey, @NonNull PersonIdent committer, - CredentialsProvider credentialsProvider) throws CanceledException; - - /** - * Indicates if a signing key is available for the specified committer - * and/or signing key. - * - * @param gpgSigningKey - * the signing key to locate (passed as is to the GPG signing - * tool as is; eg., value of user.signingkey) - * @param committer - * the signing identity (to help with key lookup in case signing - * key is not specified) - * @param credentialsProvider - * provider to use when querying for signing key credentials (eg. - * passphrase) - * @return true if a signing key is available, - * false otherwise - * @throws CanceledException - * when signing was canceled (eg., user aborted when entering - * passphrase) - */ - public abstract boolean canLocateSigningKey(@Nullable String gpgSigningKey, - @NonNull PersonIdent committer, - CredentialsProvider credentialsProvider) throws CanceledException; - -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/SignatureVerifier.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/SignatureVerifier.java new file mode 100644 index 0000000000..2ce2708cb9 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/SignatureVerifier.java @@ -0,0 +1,106 @@ +/* + * Copyright (C) 2021, 2024 Thomas Wolf and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lib; + +import java.io.IOException; +import java.util.Date; + +import org.eclipse.jgit.annotations.NonNull; +import org.eclipse.jgit.api.errors.JGitInternalException; + +/** + * A {@code SignatureVerifier} can verify signatures on git commits and tags. + * + * @since 7.0 + */ +public interface SignatureVerifier { + + /** + * Verifies a given signature for given data. + * + * @param repository + * the {@link Repository} the data comes from. + * @param config + * the {@link GpgConfig} + * @param data + * the signature is for + * @param signatureData + * the ASCII-armored signature + * @return a {@link SignatureVerification} describing the outcome + * @throws IOException + * if the signature cannot be parsed + * @throws JGitInternalException + * if signature verification fails + */ + SignatureVerification verify(@NonNull Repository repository, + @NonNull GpgConfig config, byte[] data, byte[] signatureData) + throws IOException; + + /** + * Retrieves the name of this verifier. This should be a short string + * identifying the engine that verified the signature, like "gpg" if GPG is + * used, or "bc" for a BouncyCastle implementation. + * + * @return the name + */ + @NonNull + String getName(); + + /** + * A {@link SignatureVerifier} may cache public keys to speed up + * verifying signatures on multiple objects. This clears this cache, if any. + */ + void clear(); + + /** + * A {@code SignatureVerification} returns data about a (positively or + * negatively) verified signature. + * + * @param verifierName + * the name of the verifier that created this verification result + * @param creationDate + * date and time the signature was created + * @param signer + * the signer as stored in the signature, or {@code null} if + * unknown + * @param keyFingerprint + * fingerprint of the public key, or {@code null} if unknown + * @param keyUser + * user associated with the key, or {@code null} if unknown + * @param verified + * whether the signature verification was successful + * @param expired + * whether the public key used for this signature verification + * was expired when the signature was created + * @param trustLevel + * the trust level of the public key used to verify the signature + * @param message + * human-readable message giving additional information about the + * outcome of the verification, possibly {@code null} + */ + record SignatureVerification( + String verifierName, + Date creationDate, + String signer, + String keyFingerprint, + String keyUser, + boolean verified, + boolean expired, + @NonNull TrustLevel trustLevel, + String message) { + } + + /** + * The owner's trust in a public key. + */ + enum TrustLevel { + UNKNOWN, NEVER, MARGINAL, FULL, ULTIMATE + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/SignatureVerifierFactory.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/SignatureVerifierFactory.java new file mode 100644 index 0000000000..7844aba3bd --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/SignatureVerifierFactory.java @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2024 Thomas Wolf and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lib; + +import org.eclipse.jgit.annotations.NonNull; + +/** + * A factory for {@link SignatureVerifier}s. + * + * @since 7.0 + */ +public interface SignatureVerifierFactory { + + /** + * Tells what kind of {@link SignatureVerifier} this factory creates. + * + * @return the {@link GpgConfig.GpgFormat} of the signer + */ + @NonNull + GpgConfig.GpgFormat getType(); + + /** + * Creates a new instance of a {@link SignatureVerifier} that can produce + * signatures of type {@link #getType()}. + * + * @return a new {@link SignatureVerifier} + */ + @NonNull + SignatureVerifier create(); +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/SignatureVerifiers.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/SignatureVerifiers.java new file mode 100644 index 0000000000..01c8422b66 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/SignatureVerifiers.java @@ -0,0 +1,239 @@ +/* + * Copyright (C) 2024 Thomas Wolf and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lib; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.text.MessageFormat; +import java.util.Arrays; +import java.util.EnumMap; +import java.util.Map; +import java.util.ServiceConfigurationError; +import java.util.ServiceLoader; +import java.util.concurrent.ConcurrentHashMap; + +import org.eclipse.jgit.annotations.NonNull; +import org.eclipse.jgit.annotations.Nullable; +import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevObject; +import org.eclipse.jgit.revwalk.RevTag; +import org.eclipse.jgit.util.RawParseUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Manages the available signers. + * + * @since 7.0 + */ +public final class SignatureVerifiers { + + private static final Logger LOG = LoggerFactory.getLogger(SignatureVerifiers.class); + + private static final byte[] PGP_PREFIX = Constants.GPG_SIGNATURE_PREFIX + .getBytes(StandardCharsets.US_ASCII); + + private static final byte[] X509_PREFIX = Constants.CMS_SIGNATURE_PREFIX + .getBytes(StandardCharsets.US_ASCII); + + private static final byte[] SSH_PREFIX = Constants.SSH_SIGNATURE_PREFIX + .getBytes(StandardCharsets.US_ASCII); + + private static final Map FACTORIES = loadSignatureVerifiers(); + + private static final Map VERIFIERS = new ConcurrentHashMap<>(); + + private static Map loadSignatureVerifiers() { + Map result = new EnumMap<>( + GpgConfig.GpgFormat.class); + try { + for (SignatureVerifierFactory factory : ServiceLoader + .load(SignatureVerifierFactory.class)) { + GpgConfig.GpgFormat format = factory.getType(); + SignatureVerifierFactory existing = result.get(format); + if (existing != null) { + LOG.warn("{}", //$NON-NLS-1$ + MessageFormat.format( + JGitText.get().signatureServiceConflict, + "SignatureVerifierFactory", format, //$NON-NLS-1$ + existing.getClass().getCanonicalName(), + factory.getClass().getCanonicalName())); + } else { + result.put(format, factory); + } + } + } catch (ServiceConfigurationError e) { + LOG.error(e.getMessage(), e); + } + return result; + } + + private SignatureVerifiers() { + // No instantiation + } + + /** + * Retrieves a {@link Signer} that can produce signatures of the given type + * {@code format}. + * + * @param format + * {@link GpgConfig.GpgFormat} the signer must support + * @return a {@link Signer}, or {@code null} if none is available + */ + public static SignatureVerifier get(@NonNull GpgConfig.GpgFormat format) { + return VERIFIERS.computeIfAbsent(format, f -> { + SignatureVerifierFactory factory = FACTORIES.get(format); + if (factory == null) { + return null; + } + return factory.create(); + }); + } + + /** + * Sets a specific signature verifier to use for a specific signature type. + * + * @param format + * signature type to set the {@code verifier} for + * @param verifier + * the {@link SignatureVerifier} to use for signatures of type + * {@code format}; if {@code null}, a default implementation, if + * available, may be used. + */ + public static void set(@NonNull GpgConfig.GpgFormat format, + SignatureVerifier verifier) { + SignatureVerifier previous; + if (verifier == null) { + previous = VERIFIERS.remove(format); + } else { + previous = VERIFIERS.put(format, verifier); + } + if (previous != null) { + previous.clear(); + } + } + + /** + * Verifies the signature on a signed commit or tag. + * + * @param repository + * the {@link Repository} the object is from + * @param config + * the {@link GpgConfig} to use + * @param object + * to verify + * @return a {@link SignatureVerifier.SignatureVerification} describing the + * outcome of the verification, or {@code null} if the object does + * not have a signature of a known type + * @throws IOException + * if an error occurs getting a public key + * @throws org.eclipse.jgit.api.errors.JGitInternalException + * if signature verification fails + */ + @Nullable + public static SignatureVerifier.SignatureVerification verify( + @NonNull Repository repository, @NonNull GpgConfig config, + @NonNull RevObject object) throws IOException { + if (object instanceof RevCommit) { + RevCommit commit = (RevCommit) object; + byte[] signatureData = commit.getRawGpgSignature(); + if (signatureData == null) { + return null; + } + byte[] raw = commit.getRawBuffer(); + // Now remove the GPG signature + byte[] header = { 'g', 'p', 'g', 's', 'i', 'g' }; + int start = RawParseUtils.headerStart(header, raw, 0); + if (start < 0) { + return null; + } + int end = RawParseUtils.nextLfSkippingSplitLines(raw, start); + // start is at the beginning of the header's content + start -= header.length + 1; + // end is on the terminating LF; we need to skip that, too + if (end < raw.length) { + end++; + } + byte[] data = new byte[raw.length - (end - start)]; + System.arraycopy(raw, 0, data, 0, start); + System.arraycopy(raw, end, data, start, raw.length - end); + return verify(repository, config, data, signatureData); + } else if (object instanceof RevTag) { + RevTag tag = (RevTag) object; + byte[] signatureData = tag.getRawGpgSignature(); + if (signatureData == null) { + return null; + } + byte[] raw = tag.getRawBuffer(); + // The signature is just tacked onto the end of the message, which + // is last in the buffer. + byte[] data = Arrays.copyOfRange(raw, 0, + raw.length - signatureData.length); + return verify(repository, config, data, signatureData); + } + return null; + } + + /** + * Verifies a given signature for some give data. + * + * @param repository + * the {@link Repository} the object is from + * @param config + * the {@link GpgConfig} to use + * @param data + * to verify the signature of + * @param signature + * the given signature of the {@code data} + * @return a {@link SignatureVerifier.SignatureVerification} describing the + * outcome of the verification, or {@code null} if the signature + * type is unknown + * @throws IOException + * if an error occurs getting a public key + * @throws org.eclipse.jgit.api.errors.JGitInternalException + * if signature verification fails + */ + @Nullable + public static SignatureVerifier.SignatureVerification verify( + @NonNull Repository repository, @NonNull GpgConfig config, + byte[] data, byte[] signature) throws IOException { + GpgConfig.GpgFormat format = getFormat(signature); + if (format == null) { + return null; + } + SignatureVerifier verifier = get(format); + if (verifier == null) { + return null; + } + return verifier.verify(repository, config, data, signature); + } + + /** + * Determines the type of a given signature. + * + * @param signature + * to get the type of + * @return the signature type, or {@code null} if unknown + */ + @Nullable + public static GpgConfig.GpgFormat getFormat(byte[] signature) { + if (RawParseUtils.match(signature, 0, PGP_PREFIX) > 0) { + return GpgConfig.GpgFormat.OPENPGP; + } + if (RawParseUtils.match(signature, 0, X509_PREFIX) > 0) { + return GpgConfig.GpgFormat.X509; + } + if (RawParseUtils.match(signature, 0, SSH_PREFIX) > 0) { + return GpgConfig.GpgFormat.SSH; + } + return null; + } +} \ No newline at end of file diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Signer.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Signer.java new file mode 100644 index 0000000000..3bb7464d08 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Signer.java @@ -0,0 +1,139 @@ +/* + * Copyright (C) 2024 Thomas Wolf and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lib; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; + +import org.eclipse.jgit.annotations.NonNull; +import org.eclipse.jgit.api.errors.CanceledException; +import org.eclipse.jgit.api.errors.JGitInternalException; +import org.eclipse.jgit.api.errors.UnsupportedSigningFormatException; +import org.eclipse.jgit.transport.CredentialsProvider; + +/** + * Creates signatures for Git objects. + * + * @since 7.0 + */ +public interface Signer { + + /** + * Signs the specified object. + * + *

+ * Implementors should obtain the payload for signing from the specified + * object via {@link ObjectBuilder#build()} and create a proper + * {@link GpgSignature}. The generated signature is set on the specified + * {@code object} (see {@link ObjectBuilder#setGpgSignature(GpgSignature)}). + *

+ *

+ * Any existing signature on the object must be discarded prior obtaining + * the payload via {@link ObjectBuilder#build()}. + *

+ * + * @param repository + * {@link Repository} the object belongs to + * @param config + * GPG settings from the git config + * @param object + * the object to sign (must not be {@code null} and must be + * complete to allow proper calculation of payload) + * @param committer + * the signing identity (to help with key lookup in case signing + * key is not specified) + * @param signingKey + * if non-{@code null} overrides the signing key from the config + * @param credentialsProvider + * provider to use when querying for signing key credentials (eg. + * passphrase) + * @throws CanceledException + * when signing was canceled (eg., user aborted when entering + * passphrase) + * @throws IOException + * if an I/O error occurs + * @throws UnsupportedSigningFormatException + * if a config is given and the wanted key format is not + * supported + */ + default void signObject(@NonNull Repository repository, + @NonNull GpgConfig config, @NonNull ObjectBuilder object, + @NonNull PersonIdent committer, String signingKey, + CredentialsProvider credentialsProvider) + throws CanceledException, IOException, + UnsupportedSigningFormatException { + try { + object.setGpgSignature(sign(repository, config, object.build(), + committer, signingKey, credentialsProvider)); + } catch (UnsupportedEncodingException e) { + throw new JGitInternalException(e.getMessage(), e); + } + } + + /** + * Signs arbitrary data. + * + * @param repository + * {@link Repository} the signature is created in + * @param config + * GPG settings from the git config + * @param data + * the data to sign + * @param committer + * the signing identity (to help with key lookup in case signing + * key is not specified) + * @param signingKey + * if non-{@code null} overrides the signing key from the config + * @param credentialsProvider + * provider to use when querying for signing key credentials (eg. + * passphrase) + * @return the signature for {@code data} + * @throws CanceledException + * when signing was canceled (eg., user aborted when entering + * passphrase) + * @throws IOException + * if an I/O error occurs + * @throws UnsupportedSigningFormatException + * if a config is given and the wanted key format is not + * supported + */ + GpgSignature sign(@NonNull Repository repository, @NonNull GpgConfig config, + byte[] data, @NonNull PersonIdent committer, String signingKey, + CredentialsProvider credentialsProvider) throws CanceledException, + IOException, UnsupportedSigningFormatException; + + /** + * Indicates if a signing key is available for the specified committer + * and/or signing key. + * + * @param repository + * the current {@link Repository} + * @param config + * GPG settings from the git config + * @param committer + * the signing identity (to help with key lookup in case signing + * key is not specified) + * @param signingKey + * if non-{@code null} overrides the signing key from the config + * @param credentialsProvider + * provider to use when querying for signing key credentials (eg. + * passphrase) + * @return {@code true} if a signing key is available, {@code false} + * otherwise + * @throws CanceledException + * when signing was canceled (eg., user aborted when entering + * passphrase) + */ + boolean canLocateSigningKey(@NonNull Repository repository, + @NonNull GpgConfig config, @NonNull PersonIdent committer, + String signingKey, CredentialsProvider credentialsProvider) + throws CanceledException; + +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/SignerFactory.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/SignerFactory.java new file mode 100644 index 0000000000..125d25e3b7 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/SignerFactory.java @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2024 Thomas Wolf and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lib; + +import org.eclipse.jgit.annotations.NonNull; + +/** + * A factory for {@link Signer}s. + * + * @since 7.0 + */ +public interface SignerFactory { + + /** + * Tells what kind of {@link Signer} this factory creates. + * + * @return the {@link GpgConfig.GpgFormat} of the signer + */ + @NonNull + GpgConfig.GpgFormat getType(); + + /** + * Creates a new instance of a {@link Signer} that can produce signatures of + * type {@link #getType()}. + * + * @return a new {@link Signer} + */ + @NonNull + Signer create(); +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Signers.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Signers.java new file mode 100644 index 0000000000..7771b07841 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Signers.java @@ -0,0 +1,101 @@ +/* + * Copyright (C) 2024 Thomas Wolf and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lib; + +import java.text.MessageFormat; +import java.util.EnumMap; +import java.util.Map; +import java.util.ServiceConfigurationError; +import java.util.ServiceLoader; +import java.util.concurrent.ConcurrentHashMap; + +import org.eclipse.jgit.annotations.NonNull; +import org.eclipse.jgit.internal.JGitText; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Manages the available signers. + * + * @since 7.0 + */ +public final class Signers { + + private static final Logger LOG = LoggerFactory.getLogger(Signers.class); + + private static final Map SIGNER_FACTORIES = loadSigners(); + + private static final Map SIGNERS = new ConcurrentHashMap<>(); + + private static Map loadSigners() { + Map result = new EnumMap<>( + GpgConfig.GpgFormat.class); + try { + for (SignerFactory factory : ServiceLoader + .load(SignerFactory.class)) { + GpgConfig.GpgFormat format = factory.getType(); + SignerFactory existing = result.get(format); + if (existing != null) { + LOG.warn("{}", //$NON-NLS-1$ + MessageFormat.format( + JGitText.get().signatureServiceConflict, + "SignerFactory", format, //$NON-NLS-1$ + existing.getClass().getCanonicalName(), + factory.getClass().getCanonicalName())); + } else { + result.put(format, factory); + } + } + } catch (ServiceConfigurationError e) { + LOG.error(e.getMessage(), e); + } + return result; + } + + private Signers() { + // No instantiation + } + + /** + * Retrieves a {@link Signer} that can produce signatures of the given type + * {@code format}. + * + * @param format + * {@link GpgConfig.GpgFormat} the signer must support + * @return a {@link Signer}, or {@code null} if none is available + */ + public static Signer get(@NonNull GpgConfig.GpgFormat format) { + return SIGNERS.computeIfAbsent(format, f -> { + SignerFactory factory = SIGNER_FACTORIES.get(format); + if (factory == null) { + return null; + } + return factory.create(); + }); + } + + /** + * Sets a specific signer to use for a specific signature type. + * + * @param format + * signature type to set the {@code signer} for + * @param signer + * the {@link Signer} to use for signatures of type + * {@code format}; if {@code null}, a default implementation, if + * available, may be used. + */ + public static void set(@NonNull GpgConfig.GpgFormat format, Signer signer) { + if (signer == null) { + SIGNERS.remove(format); + } else { + SIGNERS.put(format, signer); + } + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevTag.java b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevTag.java index 5b50c2afd7..0737a78085 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevTag.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevTag.java @@ -13,7 +13,6 @@ package org.eclipse.jgit.revwalk; import static java.nio.charset.StandardCharsets.UTF_8; -import static org.eclipse.jgit.lib.Constants.GPG_SIGNATURE_PREFIX; import java.io.IOException; import java.nio.charset.Charset; @@ -39,8 +38,17 @@ import org.eclipse.jgit.util.StringUtils; */ public class RevTag extends RevObject { - private static final byte[] hSignature = Constants - .encodeASCII(GPG_SIGNATURE_PREFIX); + private static final byte[] SIGNATURE_START = Constants + .encodeASCII("-----BEGIN"); //$NON-NLS-1$ + + private static final byte[] GPG_SIGNATURE_START = Constants + .encodeASCII(Constants.GPG_SIGNATURE_PREFIX); + + private static final byte[] CMS_SIGNATURE_START = Constants + .encodeASCII(Constants.CMS_SIGNATURE_PREFIX); + + private static final byte[] SSH_SIGNATURE_START = Constants + .encodeASCII(Constants.SSH_SIGNATURE_PREFIX); /** * Parse an annotated tag from its canonical format. @@ -209,20 +217,27 @@ public class RevTag extends RevObject { return msgB; } // Find the last signature start and return the rest - int start = nextStart(hSignature, raw, msgB); + int start = nextStart(SIGNATURE_START, raw, msgB); if (start < 0) { return start; } int next = RawParseUtils.nextLF(raw, start); while (next < raw.length) { - int newStart = nextStart(hSignature, raw, next); + int newStart = nextStart(SIGNATURE_START, raw, next); if (newStart < 0) { break; } start = newStart; next = RawParseUtils.nextLF(raw, start); } - return start; + // SIGNATURE_START is just a prefix. Check that it is one of the known + // full signature start tags. + if (RawParseUtils.match(raw, start, GPG_SIGNATURE_START) > 0 + || RawParseUtils.match(raw, start, CMS_SIGNATURE_START) > 0 + || RawParseUtils.match(raw, start, SSH_SIGNATURE_START) > 0) { + return start; + } + return -1; } /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/SignatureUtils.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/SignatureUtils.java index cf06172c17..90524db20a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/SignatureUtils.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/SignatureUtils.java @@ -13,8 +13,8 @@ import java.text.MessageFormat; import java.util.Locale; import org.eclipse.jgit.internal.JGitText; -import org.eclipse.jgit.lib.GpgSignatureVerifier.SignatureVerification; -import org.eclipse.jgit.lib.GpgSignatureVerifier.TrustLevel; +import org.eclipse.jgit.lib.SignatureVerifier.SignatureVerification; +import org.eclipse.jgit.lib.SignatureVerifier.TrustLevel; import org.eclipse.jgit.lib.PersonIdent; /** @@ -39,29 +39,31 @@ public final class SignatureUtils { * to use for dates * @return a textual representation of the {@link SignatureVerification}, * using LF as line separator + * + * @since 7.0 */ public static String toString(SignatureVerification verification, PersonIdent creator, GitDateFormatter formatter) { StringBuilder result = new StringBuilder(); // Use the creator's timezone for the signature date PersonIdent dateId = new PersonIdent(creator, - verification.getCreationDate()); + verification.creationDate()); result.append(MessageFormat.format(JGitText.get().verifySignatureMade, formatter.formatDate(dateId))); result.append('\n'); result.append(MessageFormat.format( JGitText.get().verifySignatureKey, - verification.getKeyFingerprint().toUpperCase(Locale.ROOT))); + verification.keyFingerprint().toUpperCase(Locale.ROOT))); result.append('\n'); - if (!StringUtils.isEmptyOrNull(verification.getSigner())) { + if (!StringUtils.isEmptyOrNull(verification.signer())) { result.append( MessageFormat.format(JGitText.get().verifySignatureIssuer, - verification.getSigner())); + verification.signer())); result.append('\n'); } String msg; - if (verification.getVerified()) { - if (verification.isExpired()) { + if (verification.verified()) { + if (verification.expired()) { msg = JGitText.get().verifySignatureExpired; } else { msg = JGitText.get().verifySignatureGood; @@ -69,14 +71,14 @@ public final class SignatureUtils { } else { msg = JGitText.get().verifySignatureBad; } - result.append(MessageFormat.format(msg, verification.getKeyUser())); - if (!TrustLevel.UNKNOWN.equals(verification.getTrustLevel())) { + result.append(MessageFormat.format(msg, verification.keyUser())); + if (!TrustLevel.UNKNOWN.equals(verification.trustLevel())) { result.append(' ' + MessageFormat .format(JGitText.get().verifySignatureTrust, verification - .getTrustLevel().name().toLowerCase(Locale.ROOT))); + .trustLevel().name().toLowerCase(Locale.ROOT))); } result.append('\n'); - msg = verification.getMessage(); + msg = verification.message(); if (!StringUtils.isEmptyOrNull(msg)) { result.append(msg); result.append('\n'); -- cgit v1.2.3 From 692ccfc0c29d53afc7a0b82f41efcd999ed217b0 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Tue, 27 Aug 2024 01:31:44 +0200 Subject: AmazonS3: Ensure SAXParserFactory sets valid/expected input params Change Ie8a9d411fc19e8b7bf86c0b4df0b02153a0e9444 broke setting valid/expected input parameters for the XML parser. This can be fixed by calling SaxParserFactory#setNamespaceAware, see [1]. Also see earlier fix in [2]. [1] https://stackoverflow.com/questions/24891323/namespace-handling-with-sax-in-java [2] I05e993032ab3a6afb78634290b578ebc73cf1cbd Bug: jgit-87 Change-Id: Id4e9eebac8d9de81e5d48a608066db3cc862e15c --- org.eclipse.jgit/src/org/eclipse/jgit/transport/AmazonS3.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/AmazonS3.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/AmazonS3.java index b873925316..aaf9f8a08a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/AmazonS3.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/AmazonS3.java @@ -757,8 +757,10 @@ public class AmazonS3 { final XMLReader xr; try { - xr = SAXParserFactory.newInstance().newSAXParser() - .getXMLReader(); + SAXParserFactory saxParserFactory = SAXParserFactory + .newInstance(); + saxParserFactory.setNamespaceAware(true); + xr = saxParserFactory.newSAXParser().getXMLReader(); } catch (SAXException | ParserConfigurationException e) { throw new IOException( JGitText.get().noXMLParserAvailable, e); -- cgit v1.2.3 From 6e15524470d9b822a10644cee47e8475eea9f049 Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Tue, 20 Aug 2024 12:54:04 -0700 Subject: DfsReaderIoStats: getters to object size index micros/bytes These properties of the stats object don't have a getter and it is required to export those values in logs. Change-Id: I7f91a38ee4d02668aff1cbc8424ea669cdb1d2f7 --- .../jgit/internal/storage/dfs/DfsReaderIoStats.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReaderIoStats.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReaderIoStats.java index 1e5e439598..fcfa3e0dee 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReaderIoStats.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReaderIoStats.java @@ -248,6 +248,15 @@ public class DfsReaderIoStats { return stats.readCommitGraphBytes; } + /** + * Get total number of bytes read from object size indexes. + * + * @return total number of bytes read from object size indexes. + */ + public long getObjectSizeIndexBytes() { + return stats.readObjectSizeIndexBytes; + } + /** * Get total microseconds spent reading pack indexes. * @@ -284,6 +293,15 @@ public class DfsReaderIoStats { return stats.readCommitGraphMicros; } + /** + * Get total microseconds spent reading object size indexes. + * + * @return total microseconds spent reading object size indexes. + */ + public long getReadObjectSizeIndexMicros() { + return stats.readObjectSizeIndexMicros; + } + /** * Get total number of block cache hits. * -- cgit v1.2.3 From c318c8a435f584ac00d1cb24292da0f8a956707d Mon Sep 17 00:00:00 2001 From: Sam Delmerico Date: Tue, 27 Aug 2024 13:27:35 -0700 Subject: DfsPackFile: re-add metrics for bitmap index loads The calculations for readBitmapIdxBytes and readBitmapIdxMicros metrics were unintentionally removed in https://gerrithub.io/c/eclipse-jgit/jgit/+/1177262, so this change adds those metrics back. Change-Id: I7ac7769acd092680933a27b2b3e70dd67690cfbf --- .../src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java | 3 +++ 1 file changed, 3 insertions(+) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java index 845feab5ac..48ed47a77c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java @@ -1280,9 +1280,12 @@ public final class DfsPackFile extends BlockBasedFile { private DfsBlockCache.Ref loadBitmapIndex(DfsReader ctx, DfsStreamKey bitmapKey) throws IOException { ctx.stats.readBitmap++; + long start = System.nanoTime(); PackBitmapIndexLoader.LoadResult result = bitmapLoader .loadPackBitmapIndex(ctx, this); bitmapIndex = result.bitmapIndex; + ctx.stats.readBitmapIdxBytes += result.bytesRead; + ctx.stats.readBitmapIdxMicros += elapsedMicros(start); return new DfsBlockCache.Ref<>(bitmapKey, REF_POSITION, result.bytesRead, result.bitmapIndex); } -- cgit v1.2.3 From 9627ee84ed4086cee51131cfd980aef0dbf33f9c Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Wed, 28 Aug 2024 16:08:04 +0200 Subject: Suppress non-externalized string warnings for texts which don't need to be translatable. Change-Id: Iacac5df9525ddbc9942b7c440450b6aad4864135 --- .../src/org/eclipse/jgit/gitrepo/BareSuperprojectWriter.java | 2 +- org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoCommand.java | 3 ++- .../org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/BareSuperprojectWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/BareSuperprojectWriter.java index d191e23399..e511a68d2e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/BareSuperprojectWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/BareSuperprojectWriter.java @@ -157,7 +157,7 @@ class BareSuperprojectWriter { if (ObjectId.isId(proj.getRevision())) { objectId = ObjectId.fromString(proj.getRevision()); if (config.recordRemoteBranch && proj.getUpstream() != null) { - cfg.setString("submodule", name, "ref", proj.getUpstream()); + cfg.setString("submodule", name, "ref", proj.getUpstream()); //$NON-NLS-1$//$NON-NLS-2$ } } else { objectId = callback.sha1(url, proj.getRevision()); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoCommand.java index 9979664ceb..d30e8d63fd 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoCommand.java @@ -255,7 +255,8 @@ public class RepoCommand extends GitCommand { @SuppressWarnings("serial") static class ManifestErrorException extends GitAPIException { ManifestErrorException(Throwable cause) { - super(RepoText.get().invalidManifest + " " + cause.getMessage(), cause); + super(RepoText.get().invalidManifest + " " + cause.getMessage(), //$NON-NLS-1$ + cause); } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java index 20f4666373..fa68b979fb 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java @@ -344,7 +344,7 @@ public class DfsBlockCacheConfig { if (packExtsDuplicates.size() > 0) { String duplicatePackExts = packExtsDuplicates.stream() .map(PackExt::toString) - .collect(Collectors.joining(",")); + .collect(Collectors.joining(",")); //$NON-NLS-1$ throw new IllegalArgumentException(MessageFormat.format( JGitText.get().duplicatePackExtensionsSet, CONFIG_CORE_SECTION, subSection, @@ -480,7 +480,7 @@ public class DfsBlockCacheConfig { throw new IllegalArgumentException( JGitText.get().noPackExtGivenForConfiguration); } - String[] extensions = packExtensions.split(" ", -1); + String[] extensions = packExtensions.split(" ", -1); //$NON-NLS-1$ Set packExts = new HashSet<>(extensions.length); for (String extension : extensions) { try { -- cgit v1.2.3 From 07f5e817982cdfcb38db7c10f37c6e4964a0a666 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Wed, 28 Aug 2024 16:13:42 +0200 Subject: PackObjectSizeIndexV1: fix boxing warnings Change-Id: I25e6194fb8bf09dcac1613cec8349c3893a4f81a --- .../eclipse/jgit/internal/storage/file/PackObjectSizeIndexV1.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackObjectSizeIndexV1.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackObjectSizeIndexV1.java index e172f141f5..9957f54fbf 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackObjectSizeIndexV1.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackObjectSizeIndexV1.java @@ -149,7 +149,8 @@ class PackObjectSizeIndexV1 implements PackObjectSizeIndex { byte[] data = in.readNBytes(expectedBytes); if (data.length < expectedBytes) { throw new IOException(MessageFormat - .format(JGitText.get().unableToReadFullArray, ints)); + .format(JGitText.get().unableToReadFullArray, + Integer.valueOf(ints))); } return new IntArray(data); } @@ -217,7 +218,8 @@ class PackObjectSizeIndexV1 implements PackObjectSizeIndex { byte[] data = in.readNBytes(longs * LONG_SIZE); if (data.length < longs * LONG_SIZE) { throw new IOException(MessageFormat - .format(JGitText.get().unableToReadFullArray, longs)); + .format(JGitText.get().unableToReadFullArray, + Integer.valueOf(longs))); } return new LongArray(data); } -- cgit v1.2.3 From cef6e8a9affcdea0b2f628d1a5d94365ac528e2b Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Thu, 29 Aug 2024 15:13:47 -0700 Subject: ObjectId: Add method to read an ObjectId from a ByteBuffer Some storages return data in a convenient ByteBuffer wrapper, but there is no straigh-forward method to read ObjectIds from it. Add ObjectId#fromRaw(ByteBuffer) to read object ids from byte buffers. Change-Id: Ia3b244005e4d9a613294f5ad9dab3b8e7bc3d7df --- .../tst/org/eclipse/jgit/lib/ObjectIdTest.java | 13 +++++++++++++ org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectId.java | 17 +++++++++++++++++ 2 files changed, 30 insertions(+) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/ObjectIdTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/ObjectIdTest.java index 21032c341f..d6f0b038d2 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/ObjectIdTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/ObjectIdTest.java @@ -16,6 +16,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import java.nio.ByteBuffer; import java.util.Locale; import org.eclipse.jgit.errors.InvalidObjectIdException; @@ -153,4 +154,16 @@ public class ObjectIdTest { assertEquals(ObjectId.fromRaw(exp).name(), id.name()); } } + + @Test + public void test_toFromByteBuffer_raw() { + ObjectId oid = ObjectId + .fromString("ff00eedd003713bb1bb26b808ec9312548e73946"); + ByteBuffer anObject = ByteBuffer.allocate(Constants.OBJECT_ID_LENGTH); + oid.copyRawTo(anObject); + anObject.flip(); + + ObjectId actual = ObjectId.fromRaw(anObject); + assertEquals(oid.name(), actual.name()); + } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectId.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectId.java index 1c31263e33..1b455b974d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectId.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectId.java @@ -15,6 +15,7 @@ import java.io.IOException; import java.io.ObjectInputStream; import java.io.ObjectOutputStream; import java.io.Serializable; +import java.nio.ByteBuffer; import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.errors.InvalidObjectIdException; @@ -151,6 +152,22 @@ public class ObjectId extends AnyObjectId implements Serializable { return new ObjectId(a, b, c, d, e); } + /** + * Convert an ObjectId from raw binary representation + * + * @param bb + * a bytebuffer with the objectid encoded as 5 consecutive ints. + * This is the reverse of {@link ObjectId#copyRawTo(ByteBuffer)} + * + * @return the converted object id. + * + * @since 7.0 + */ + public static final ObjectId fromRaw(ByteBuffer bb) { + return new ObjectId(bb.getInt(), bb.getInt(), bb.getInt(), bb.getInt(), + bb.getInt()); + } + /** * Convert an ObjectId from raw binary representation. * -- cgit v1.2.3 From ff1592ef7b406d096b2fd822745a9a29d7831ef5 Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Thu, 29 Aug 2024 15:40:49 -0700 Subject: UploadPack: Remove @Deprecated classes and methods The coming 7.0 release is a chance to clean up these deprecated classes and methods. Change-Id: I23d263c9244c1e0096eb7c96c290213c9e668a03 --- .../src/org/eclipse/jgit/transport/UploadPack.java | 61 +--------------------- 1 file changed, 1 insertion(+), 60 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java index 9318871520..ec51c885c6 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java @@ -30,11 +30,11 @@ import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_MULTI_ACK_D import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_NO_DONE; import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_NO_PROGRESS; import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_OFS_DELTA; +import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_SESSION_ID; import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_SHALLOW; import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_SIDEBAND_ALL; import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_SIDE_BAND; import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_SIDE_BAND_64K; -import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_SESSION_ID; import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_THIN_PACK; import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_WAIT_FOR_DONE; import static org.eclipse.jgit.transport.GitProtocolConstants.PACKET_ACK; @@ -80,7 +80,6 @@ import org.eclipse.jgit.errors.PackProtocolException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.pack.CachedPackUriProvider; import org.eclipse.jgit.internal.storage.pack.PackWriter; -import org.eclipse.jgit.internal.transport.parser.FirstWant; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.NullProgressMonitor; import org.eclipse.jgit.lib.ObjectId; @@ -172,52 +171,6 @@ public class UploadPack implements Closeable { throws PackProtocolException, IOException; } - /** - * Data in the first line of a want-list, the line itself plus options. - * - * @deprecated Use {@link FirstWant} instead - */ - @Deprecated - public static class FirstLine { - - private final FirstWant firstWant; - - /** - * @param line - * line from the client. - */ - public FirstLine(String line) { - try { - firstWant = FirstWant.fromLine(line); - } catch (PackProtocolException e) { - throw new UncheckedIOException(e); - } - } - - /** - * Get non-capabilities part of the line - * - * @return non-capabilities part of the line. - */ - public String getLine() { - return firstWant.getLine(); - } - - /** - * Get capabilities parsed from the line - * - * @return capabilities parsed from the line. - */ - public Set getOptions() { - if (firstWant.getAgent() != null) { - Set caps = new HashSet<>(firstWant.getCapabilities()); - caps.add(OPTION_AGENT + '=' + firstWant.getAgent()); - return caps; - } - return firstWant.getCapabilities(); - } - } - /* * {@link java.util.function.Consumer} doesn't allow throwing checked * exceptions. Define our own to propagate IOExceptions. @@ -1692,18 +1645,6 @@ public class UploadPack implements Closeable { return currentRequest.getDepth(); } - /** - * Deprecated synonym for {@code getFilterSpec().getBlobLimit()}. - * - * @return filter blob limit requested by the client, or -1 if no limit - * @since 5.3 - * @deprecated Use {@link #getFilterSpec()} instead - */ - @Deprecated - public final long getFilterBlobLimit() { - return getFilterSpec().getBlobLimit(); - } - /** * Returns the filter spec for the current request. Valid only after * calling recvWants(). This may be a no-op filter spec, but it won't be -- cgit v1.2.3 From 8438808f9d98c90fcae10567405ade34d38319cf Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Thu, 29 Aug 2024 15:43:22 -0700 Subject: ReachabilityChecker: Remove @Deprecated method Cleaning up before 7.0 release. Change-Id: I620c3a485785b3d1efc950317fe552b8465a479e --- .../eclipse/jgit/revwalk/ReachabilityChecker.java | 34 ---------------------- 1 file changed, 34 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/ReachabilityChecker.java b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/ReachabilityChecker.java index 1a869a0703..5afb669a15 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/ReachabilityChecker.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/ReachabilityChecker.java @@ -26,40 +26,6 @@ import org.eclipse.jgit.errors.MissingObjectException; * @since 5.4 */ public interface ReachabilityChecker { - - /** - * Check if all targets are reachable from the {@code starters} commits. - *

- * Caller should parse the objectIds (preferably with - * {@code walk.parseCommit()} and handle missing/incorrect type objects - * before calling this method. - * - * @param targets - * commits to reach. - * @param starters - * known starting points. - * @return An unreachable target if at least one of the targets is - * unreachable. An empty optional if all targets are reachable from - * the starters. - * - * @throws MissingObjectException - * if any of the incoming objects doesn't exist in the - * repository. - * @throws IncorrectObjectTypeException - * if any of the incoming objects is not a commit or a tag. - * @throws IOException - * if any of the underlying indexes or readers can not be - * opened. - * - * @deprecated see {{@link #areAllReachable(Collection, Stream)} - */ - @Deprecated - default Optional areAllReachable(Collection targets, - Collection starters) throws MissingObjectException, - IncorrectObjectTypeException, IOException { - return areAllReachable(targets, starters.stream()); - } - /** * Check if all targets are reachable from the {@code starters} commits. *

-- cgit v1.2.3 From e5cc3151aea90d9af958c260fcb9a7a3cfc2ca6d Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Thu, 29 Aug 2024 15:45:15 -0700 Subject: RepoCommand.RemoteReader: Remove @Deprecated method Cleaning up before 7.0 release. Change-Id: I83db715a8170a54957dd5bcf38df5b0911a12107 --- .../src/org/eclipse/jgit/gitrepo/RepoCommand.java | 26 ---------------------- 1 file changed, 26 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoCommand.java index 9979664ceb..c80a58a7c8 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/RepoCommand.java @@ -110,32 +110,6 @@ public class RepoCommand extends GitCommand { @Nullable public ObjectId sha1(String uri, String ref) throws GitAPIException; - /** - * Read a file from a remote repository. - * - * @param uri - * The URI of the remote repository - * @param ref - * The ref (branch/tag/etc.) to read - * @param path - * The relative path (inside the repo) to the file to read - * @return the file content. - * @throws GitAPIException - * If the ref have an invalid or ambiguous name, or it does - * not exist in the repository, - * @throws IOException - * If the object does not exist or is too large - * @since 3.5 - * - * @deprecated Use {@link #readFileWithMode(String, String, String)} - * instead - */ - @Deprecated - public default byte[] readFile(String uri, String ref, String path) - throws GitAPIException, IOException { - return readFileWithMode(uri, ref, path).getContents(); - } - /** * Read contents and mode (i.e. permissions) of the file from a remote * repository. -- cgit v1.2.3 From 45e28716cbef951301a917f2a4019cc7c8ac041c Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Thu, 29 Aug 2024 15:47:15 -0700 Subject: walks: Remove deprecated #createReachabilityChecker() method Change-Id: If9375ac49fa82e988e0708d89f0f929495e6e7cc --- .../src/org/eclipse/jgit/revwalk/ObjectWalk.java | 23 ---------------------- .../src/org/eclipse/jgit/revwalk/RevWalk.java | 19 +----------------- 2 files changed, 1 insertion(+), 41 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/ObjectWalk.java b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/ObjectWalk.java index 385f073a77..7c763bc9b8 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/ObjectWalk.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/ObjectWalk.java @@ -163,29 +163,6 @@ public class ObjectWalk extends RevWalk { pathBuf = new byte[256]; } - /** - * Create an object reachability checker that will use bitmaps if possible. - * - * This reachability checker accepts any object as target. For checks - * exclusively between commits, see - * {@link RevWalk#createReachabilityChecker()}. - * - * @return an object reachability checker, using bitmaps if possible. - * - * @throws IOException - * when the index fails to load. - * - * @since 5.8 - * @deprecated use - * {@code ObjectReader#createObjectReachabilityChecker(ObjectWalk)} - * instead. - */ - @Deprecated - public final ObjectReachabilityChecker createObjectReachabilityChecker() - throws IOException { - return reader.createObjectReachabilityChecker(this); - } - /** * Mark an object or commit to start graph traversal from. *

diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevWalk.java b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevWalk.java index 76c14e9c26..9f0e28d0ce 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevWalk.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevWalk.java @@ -31,9 +31,9 @@ import org.eclipse.jgit.errors.LargeObjectException; import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.errors.RevWalkException; import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.internal.storage.commitgraph.CommitGraph; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.AsyncObjectLoaderQueue; -import org.eclipse.jgit.internal.storage.commitgraph.CommitGraph; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.MutableObjectId; import org.eclipse.jgit.lib.NullProgressMonitor; @@ -278,23 +278,6 @@ public class RevWalk implements Iterable, AutoCloseable { return reader; } - /** - * Get a reachability checker for commits over this revwalk. - * - * @return the most efficient reachability checker for this repository. - * @throws IOException - * if it cannot open any of the underlying indices. - * - * @since 5.4 - * @deprecated use {@code ObjectReader#createReachabilityChecker(RevWalk)} - * instead. - */ - @Deprecated - public final ReachabilityChecker createReachabilityChecker() - throws IOException { - return reader.createReachabilityChecker(this); - } - /** * {@inheritDoc} *

-- cgit v1.2.3 From 0757ca6bfab80a20d3b832f66ed9a7b359d0d915 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 30 Aug 2024 16:38:55 +0200 Subject: Remove deprecated CheckoutCommand#setForce method Change-Id: Ie0b59ec21fc8dbe18242bbb52cb794e5a02c1ed5 --- .../src/org/eclipse/jgit/api/CheckoutCommand.java | 18 ------------------ 1 file changed, 18 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/CheckoutCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/CheckoutCommand.java index c133219d4d..32c242f271 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/CheckoutCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/CheckoutCommand.java @@ -641,24 +641,6 @@ public class CheckoutCommand extends GitCommand { return this; } - /** - * Specify to force the ref update in case of a branch switch. - * - * @param force - * if true and the branch with the given name - * already exists, the start-point of an existing branch will be - * set to a new start-point; if false, the existing branch will - * not be changed - * @return this instance - * @deprecated this method was badly named comparing its semantics to native - * git's checkout --force option, use - * {@link #setForceRefUpdate(boolean)} instead - */ - @Deprecated - public CheckoutCommand setForce(boolean force) { - return setForceRefUpdate(force); - } - /** * Specify to force the ref update in case of a branch switch. * -- cgit v1.2.3 From 373fbb4882f7d415c95dad23aba9e01ef2d3da9f Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 30 Aug 2024 16:40:21 +0200 Subject: Remove deprecated RemoteRemoveCommand#setName method Change-Id: I2bd2a213b4642283b11f787ffcbab29916e2c5e7 --- .../src/org/eclipse/jgit/api/RemoteRemoveCommand.java | 12 ------------ 1 file changed, 12 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/RemoteRemoveCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/RemoteRemoveCommand.java index 553fc2e7a0..ad553f07a9 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/RemoteRemoveCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/RemoteRemoveCommand.java @@ -46,18 +46,6 @@ public class RemoteRemoveCommand extends GitCommand { super(repo); } - /** - * The name of the remote to remove. - * - * @param name - * a remote name - * @deprecated use {@link #setRemoteName} instead - */ - @Deprecated - public void setName(String name) { - this.remoteName = name; - } - /** * The name of the remote to remove. * -- cgit v1.2.3 From 30274153eaeeaeb7639810114219d05a3a1cc267 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 30 Aug 2024 16:41:15 +0200 Subject: Remove deprecated RemoteSetUrlCommand#setName method Change-Id: I769e999b5990f83719e3c638da9dfd0be4c5b826 --- .../src/org/eclipse/jgit/api/RemoteSetUrlCommand.java | 12 ------------ 1 file changed, 12 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/RemoteSetUrlCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/RemoteSetUrlCommand.java index e3d01861fc..281e81026b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/RemoteSetUrlCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/RemoteSetUrlCommand.java @@ -68,18 +68,6 @@ public class RemoteSetUrlCommand extends GitCommand { super(repo); } - /** - * The name of the remote to change the URL for. - * - * @param name - * a remote name - * @deprecated use {@link #setRemoteName} instead - */ - @Deprecated - public void setName(String name) { - this.remoteName = name; - } - /** * The name of the remote to change the URL for. * -- cgit v1.2.3 From b4243e4317982bd7c6ac23f32f46353522cdcd57 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 30 Aug 2024 16:42:59 +0200 Subject: Remove deprecated RemoteSetUrlCommand#setPush method Change-Id: I4026531148a7560179dae89c05424f256e6600d2 --- .../src/org/eclipse/jgit/api/RemoteSetUrlCommand.java | 17 ----------------- 1 file changed, 17 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/RemoteSetUrlCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/RemoteSetUrlCommand.java index 281e81026b..b09742cba9 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/RemoteSetUrlCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/RemoteSetUrlCommand.java @@ -106,23 +106,6 @@ public class RemoteSetUrlCommand extends GitCommand { return this; } - /** - * Whether to change the push URL of the remote instead of the fetch URL. - * - * @param push - * true to set the push url, false to - * set the fetch url - * @deprecated use {@link #setUriType} instead - */ - @Deprecated - public void setPush(boolean push) { - if (push) { - setUriType(UriType.PUSH); - } else { - setUriType(UriType.FETCH); - } - } - /** * Whether to change the push URL of the remote instead of the fetch URL. * -- cgit v1.2.3 From 8c81cfc9c5718b79c3af47517780952c62200d0c Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 30 Aug 2024 16:44:16 +0200 Subject: Remove deprecated RemoteSetUrlCommand#setUri method Change-Id: Ib9041be0d88dd837aa68eda2b00f3aa9ebc54c27 --- .../src/org/eclipse/jgit/api/RemoteSetUrlCommand.java | 12 ------------ 1 file changed, 12 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/RemoteSetUrlCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/RemoteSetUrlCommand.java index b09742cba9..68ddce361f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/RemoteSetUrlCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/RemoteSetUrlCommand.java @@ -81,18 +81,6 @@ public class RemoteSetUrlCommand extends GitCommand { return this; } - /** - * The new URL for the remote. - * - * @param uri - * an URL for the remote - * @deprecated use {@link #setRemoteUri} instead - */ - @Deprecated - public void setUri(URIish uri) { - this.remoteUri = uri; - } - /** * The new URL for the remote. * -- cgit v1.2.3 From ee02be9c8a59d85d04b8a3d6573e26f0e3ad5a85 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 30 Aug 2024 16:45:34 +0200 Subject: Remove deprecated StashApplyCommand#setApplyIndex method Change-Id: I2183285f6ccba1b62a318e711f2451df5d083ca1 --- .../src/org/eclipse/jgit/api/StashApplyCommand.java | 12 ------------ 1 file changed, 12 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/StashApplyCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/StashApplyCommand.java index e4157286f1..1eac1ecd83 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/StashApplyCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/StashApplyCommand.java @@ -260,18 +260,6 @@ public class StashApplyCommand extends GitCommand { } } - /** - * Whether to restore the index state - * - * @param applyIndex - * true (default) if the command should restore the index state - * @deprecated use {@link #setRestoreIndex} instead - */ - @Deprecated - public void setApplyIndex(boolean applyIndex) { - this.restoreIndex = applyIndex; - } - /** * Whether to restore the index state * -- cgit v1.2.3 From 724626a9cc183ca3c80852c25c272d9c837a2ac5 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 30 Aug 2024 16:46:26 +0200 Subject: Remove deprecated StashApplyCommand#setApplyUntracked method Change-Id: I23e3ed3259edad3a22b59cca938f041b39ebf0cb --- .../src/org/eclipse/jgit/api/StashApplyCommand.java | 13 ------------- 1 file changed, 13 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/StashApplyCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/StashApplyCommand.java index 1eac1ecd83..b0b715e06b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/StashApplyCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/StashApplyCommand.java @@ -304,19 +304,6 @@ public class StashApplyCommand extends GitCommand { return this; } - /** - * Whether the command should restore untracked files - * - * @param applyUntracked - * true (default) if the command should restore untracked files - * @since 3.4 - * @deprecated use {@link #setRestoreUntracked} instead - */ - @Deprecated - public void setApplyUntracked(boolean applyUntracked) { - this.restoreUntracked = applyUntracked; - } - /** * Whether the command should restore untracked files * -- cgit v1.2.3 From a00b7c392922952122033fafb18d28556991423b Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 30 Aug 2024 17:07:01 +0200 Subject: Remove deprecated DirCacheCheckout#checkoutEntry methods Change-Id: I28d00d7a70af0fbb76e237dd77b929508720ecdb --- .../jgit/treewalk/FileTreeIteratorTest.java | 11 +- .../eclipse/jgit/dircache/DirCacheCheckout.java | 121 --------------------- 2 files changed, 7 insertions(+), 125 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/treewalk/FileTreeIteratorTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/treewalk/FileTreeIteratorTest.java index e463e9070a..7b9e70d4da 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/treewalk/FileTreeIteratorTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/treewalk/FileTreeIteratorTest.java @@ -25,8 +25,9 @@ import java.time.Instant; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.ResetCommand.ResetType; +import org.eclipse.jgit.dircache.Checkout; import org.eclipse.jgit.dircache.DirCache; -import org.eclipse.jgit.dircache.DirCacheCheckout; +import org.eclipse.jgit.dircache.DirCacheCheckout.CheckoutMetadata; import org.eclipse.jgit.dircache.DirCacheEditor; import org.eclipse.jgit.dircache.DirCacheEditor.PathEdit; import org.eclipse.jgit.dircache.DirCacheEntry; @@ -38,6 +39,7 @@ import org.eclipse.jgit.junit.JGitTestUtil; import org.eclipse.jgit.junit.RepositoryTestCase; import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.CoreConfig.EolStreamType; import org.eclipse.jgit.lib.FileMode; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectInserter; @@ -303,11 +305,12 @@ public class FileTreeIteratorTest extends RepositoryTestCase { DirCacheEntry dce = db.readDirCache().getEntry("symlink"); dce.setFileMode(FileMode.SYMLINK); try (ObjectReader objectReader = db.newObjectReader()) { + Checkout checkout = new Checkout(db).setRecursiveDeletion(false); + checkout.checkout(dce, + new CheckoutMetadata(EolStreamType.DIRECT, null), + objectReader, null); WorkingTreeOptions options = db.getConfig() .get(WorkingTreeOptions.KEY); - DirCacheCheckout.checkoutEntry(db, dce, objectReader, false, null, - options); - FileTreeIterator fti = new FileTreeIterator(trash, db.getFS(), options); while (!fti.getEntryPathString().equals("symlink")) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java index fa0a82fd58..4f78404f48 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java @@ -1400,127 +1400,6 @@ public class DirCacheCheckout { } } - /** - * Updates the file in the working tree with content and mode from an entry - * in the index. The new content is first written to a new temporary file in - * the same directory as the real file. Then that new file is renamed to the - * final filename. - * - *

- * Note: if the entry path on local file system exists as a non-empty - * directory, and the target entry type is a link or file, the checkout will - * fail with {@link java.io.IOException} since existing non-empty directory - * cannot be renamed to file or link without deleting it recursively. - *

- * - * @param repo - * repository managing the destination work tree. - * @param entry - * the entry containing new mode and content - * @param or - * object reader to use for checkout - * @throws java.io.IOException - * if an IO error occurred - * @since 3.6 - * @deprecated since 5.1, use - * {@link #checkoutEntry(Repository, DirCacheEntry, ObjectReader, boolean, CheckoutMetadata, WorkingTreeOptions)} - * instead - */ - @Deprecated - public static void checkoutEntry(Repository repo, DirCacheEntry entry, - ObjectReader or) throws IOException { - checkoutEntry(repo, entry, or, false, null, null); - } - - - /** - * Updates the file in the working tree with content and mode from an entry - * in the index. The new content is first written to a new temporary file in - * the same directory as the real file. Then that new file is renamed to the - * final filename. - * - *

- * Note: if the entry path on local file system exists as a file, it - * will be deleted and if it exists as a directory, it will be deleted - * recursively, independently if has any content. - *

- * - * @param repo - * repository managing the destination work tree. - * @param entry - * the entry containing new mode and content - * @param or - * object reader to use for checkout - * @param deleteRecursive - * true to recursively delete final path if it exists on the file - * system - * @param checkoutMetadata - * containing - *
    - *
  • smudgeFilterCommand to be run for smudging the entry to be - * checked out
    • - *
  • eolStreamType used for stream conversion
    • - *
- * @throws java.io.IOException - * if an IO error occurred - * @since 4.2 - * @deprecated since 6.3, use - * {@link #checkoutEntry(Repository, DirCacheEntry, ObjectReader, boolean, CheckoutMetadata, WorkingTreeOptions)} - * instead - */ - @Deprecated - public static void checkoutEntry(Repository repo, DirCacheEntry entry, - ObjectReader or, boolean deleteRecursive, - CheckoutMetadata checkoutMetadata) throws IOException { - checkoutEntry(repo, entry, or, deleteRecursive, checkoutMetadata, null); - } - - /** - * Updates the file in the working tree with content and mode from an entry - * in the index. The new content is first written to a new temporary file in - * the same directory as the real file. Then that new file is renamed to the - * final filename. - * - *

- * Note: if the entry path on local file system exists as a file, it - * will be deleted and if it exists as a directory, it will be deleted - * recursively, independently if has any content. - *

- * - * @param repo - * repository managing the destination work tree. - * @param entry - * the entry containing new mode and content - * @param or - * object reader to use for checkout - * @param deleteRecursive - * true to recursively delete final path if it exists on the file - * system - * @param checkoutMetadata - * containing - *
    - *
  • smudgeFilterCommand to be run for smudging the entry to be - * checked out
    • - *
  • eolStreamType used for stream conversion
    • - *
- * @param options - * {@link WorkingTreeOptions} that are effective; if {@code null} - * they are loaded from the repository config - * @throws java.io.IOException - * if an IO error occurred - * @since 6.3 - * @deprecated since 6.6.1; use {@link Checkout} instead - */ - @Deprecated - public static void checkoutEntry(Repository repo, DirCacheEntry entry, - ObjectReader or, boolean deleteRecursive, - CheckoutMetadata checkoutMetadata, WorkingTreeOptions options) - throws IOException { - Checkout checkout = new Checkout(repo, options) - .setRecursiveDeletion(deleteRecursive); - checkout.checkout(entry, checkoutMetadata, or, null); - } - /** * Return filtered content for a specific object (blob). EOL handling and * smudge-filter handling are applied in the same way as it would be done -- cgit v1.2.3 From f1a4afe37ac95b76a6fda5d86e330f633f681579 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 30 Aug 2024 17:10:33 +0200 Subject: DirCacheEntry: remove deprecated methods - boolean mightBeRacilyClean(int, int) - long getLastModified() - void setLastModified(long) Change-Id: I428fbb2109e13e80b8655622531c10e55a922a6f --- .../org/eclipse/jgit/dircache/DirCacheEntry.java | 50 ---------------------- 1 file changed, 50 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheEntry.java b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheEntry.java index c5e1e4e765..5a22938694 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheEntry.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheEntry.java @@ -386,28 +386,6 @@ public class DirCacheEntry { } } - /** - * Is it possible for this entry to be accidentally assumed clean? - *

- * The "racy git" problem happens when a work file can be updated faster - * than the filesystem records file modification timestamps. It is possible - * for an application to edit a work file, update the index, then edit it - * again before the filesystem will give the work file a new modification - * timestamp. This method tests to see if file was written out at the same - * time as the index. - * - * @param smudge_s - * seconds component of the index's last modified time. - * @param smudge_ns - * nanoseconds component of the index's last modified time. - * @return true if extra careful checks should be used. - * @deprecated use {@link #mightBeRacilyClean(Instant)} instead - */ - @Deprecated - public final boolean mightBeRacilyClean(int smudge_s, int smudge_ns) { - return mightBeRacilyClean(Instant.ofEpochSecond(smudge_s, smudge_ns)); - } - /** * Is it possible for this entry to be accidentally assumed clean? *

@@ -652,22 +630,6 @@ public class DirCacheEntry { encodeTS(P_CTIME, when); } - /** - * Get the cached last modification date of this file, in milliseconds. - *

- * One of the indicators that the file has been modified by an application - * changing the working tree is if the last modification time for the file - * differs from the time stored in this entry. - * - * @return last modification time of this file, in milliseconds since the - * Java epoch (midnight Jan 1, 1970 UTC). - * @deprecated use {@link #getLastModifiedInstant()} instead - */ - @Deprecated - public long getLastModified() { - return decodeTS(P_MTIME); - } - /** * Get the cached last modification date of this file. *

@@ -682,18 +644,6 @@ public class DirCacheEntry { return decodeTSInstant(P_MTIME); } - /** - * Set the cached last modification date of this file, using milliseconds. - * - * @param when - * new cached modification date of the file, in milliseconds. - * @deprecated use {@link #setLastModified(Instant)} instead - */ - @Deprecated - public void setLastModified(long when) { - encodeTS(P_MTIME, when); - } - /** * Set the cached last modification date of this file. * -- cgit v1.2.3 From 3c8b60b081551a1afb74b3d05e1375eb34443abd Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 30 Aug 2024 17:11:34 +0200 Subject: Remove deprecated PackInvalidException constructors Change-Id: I8db6b1dd8e39811ebde676389833dbc906106452 --- .../eclipse/jgit/errors/PackInvalidException.java | 24 ---------------------- 1 file changed, 24 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/errors/PackInvalidException.java b/org.eclipse.jgit/src/org/eclipse/jgit/errors/PackInvalidException.java index 1fd80867b9..38982fdf23 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/errors/PackInvalidException.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/errors/PackInvalidException.java @@ -22,18 +22,6 @@ import org.eclipse.jgit.internal.JGitText; public class PackInvalidException extends IOException { private static final long serialVersionUID = 1L; - /** - * Construct a pack invalid error. - * - * @param path - * path of the invalid pack file. - * @deprecated Use {@link #PackInvalidException(File, Throwable)}. - */ - @Deprecated - public PackInvalidException(File path) { - this(path, null); - } - /** * Construct a pack invalid error with cause. * @@ -47,18 +35,6 @@ public class PackInvalidException extends IOException { this(path.getAbsolutePath(), cause); } - /** - * Construct a pack invalid error. - * - * @param path - * path of the invalid pack file. - * @deprecated Use {@link #PackInvalidException(String, Throwable)}. - */ - @Deprecated - public PackInvalidException(String path) { - this(path, null); - } - /** * Construct a pack invalid error with cause. * -- cgit v1.2.3 From 65bc6e4f055d1522476b2ca2da1613a39341bf0e Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 30 Aug 2024 17:16:24 +0200 Subject: Remove deprecated lastModified accessors returning long - FileSnapshot#lastModified - LockFile#getCommmitLastModified Change-Id: I6962166ca5decbb332dfb25851c93debfe2ca90c --- .../org/eclipse/jgit/internal/storage/file/FileSnapshot.java | 11 ----------- .../src/org/eclipse/jgit/internal/storage/file/LockFile.java | 11 ----------- 2 files changed, 22 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java index c88ac984ec..8701a1291b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java @@ -278,17 +278,6 @@ public class FileSnapshot { this.fileKey = fileKey; } - /** - * Get time of last snapshot update - * - * @return time of last snapshot update - * @deprecated use {@link #lastModifiedInstant()} instead - */ - @Deprecated - public long lastModified() { - return lastModified.toEpochMilli(); - } - /** * Get time of last snapshot update * diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LockFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LockFile.java index 1983541e4a..9e12ee8a0c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LockFile.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LockFile.java @@ -524,17 +524,6 @@ public class LockFile { } } - /** - * Get the modification time of the output file when it was committed. - * - * @return modification time of the lock file right before we committed it. - * @deprecated use {@link #getCommitLastModifiedInstant()} instead - */ - @Deprecated - public long getCommitLastModified() { - return commitSnapshot.lastModified(); - } - /** * Get the modification time of the output file when it was committed. * -- cgit v1.2.3 From ec97acb808f0685ca3e09389301bf1d3c606f607 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 30 Aug 2024 17:17:29 +0200 Subject: Remove deprecated FileSnapshot.save(long) method Change-Id: I9b77142127cc96ee6658e85a6cf0586d941c5c0c --- .../jgit/internal/storage/file/FileSnapshot.java | 23 ---------------------- 1 file changed, 23 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java index 8701a1291b..c4fa3eeb99 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java @@ -125,29 +125,6 @@ public class FileSnapshot { return fileKey == null ? MISSING_FILEKEY : fileKey; } - /** - * Record a snapshot for a file for which the last modification time is - * already known. - *

- * This method should be invoked before the file is accessed. - *

- * Note that this method cannot rely on measuring file timestamp resolution - * to avoid racy git issues caused by finite file timestamp resolution since - * it's unknown in which filesystem the file is located. Hence the worst - * case fallback for timestamp resolution is used. - * - * @param modified - * the last modification time of the file - * @return the snapshot. - * @deprecated use {@link #save(Instant)} instead. - */ - @Deprecated - public static FileSnapshot save(long modified) { - final Instant read = Instant.now(); - return new FileSnapshot(read, Instant.ofEpochMilli(modified), - UNKNOWN_SIZE, FALLBACK_TIMESTAMP_RESOLUTION, MISSING_FILEKEY); - } - /** * Record a snapshot for a file for which the last modification time is * already known. -- cgit v1.2.3 From 63348b21dbfdc002e48c92a4274576e8d1b85f1c Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 30 Aug 2024 17:21:51 +0200 Subject: Remove deprecated static #equals(AnyObjectId, AnyObjectId) method Change-Id: I72544d2b3c85b8f96c2b8f94b86fb9b362f09475 --- .../src/org/eclipse/jgit/lib/AnyObjectId.java | 17 ----------------- 1 file changed, 17 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/AnyObjectId.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/AnyObjectId.java index c58133adab..f742e993a0 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/AnyObjectId.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/AnyObjectId.java @@ -27,23 +27,6 @@ import org.eclipse.jgit.util.References; */ public abstract class AnyObjectId implements Comparable { - /** - * Compare two object identifier byte sequences for equality. - * - * @param firstObjectId - * the first identifier to compare. Must not be null. - * @param secondObjectId - * the second identifier to compare. Must not be null. - * @return true if the two identifiers are the same. - * @deprecated use {@link #isEqual(AnyObjectId, AnyObjectId)} instead - */ - @Deprecated - @SuppressWarnings("AmbiguousMethodReference") - public static boolean equals(final AnyObjectId firstObjectId, - final AnyObjectId secondObjectId) { - return isEqual(firstObjectId, secondObjectId); - } - /** * Compare two object identifier byte sequences for equality. * -- cgit v1.2.3 From 7650a68ad6419edbdd7018102f47a68342551e1b Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 30 Aug 2024 17:23:02 +0200 Subject: Remove deprecated CommitBuilder#setEncoding(String) method Change-Id: I5bd8d0c292151bfa58325d51ef51928715871cf1 --- .../src/org/eclipse/jgit/lib/CommitBuilder.java | 13 ------------- 1 file changed, 13 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/CommitBuilder.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/CommitBuilder.java index ea33082229..ad3c2c091d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/CommitBuilder.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/CommitBuilder.java @@ -194,19 +194,6 @@ public class CommitBuilder extends ObjectBuilder { } } - /** - * Set the encoding for the commit information. - * - * @param encodingName - * the encoding name. See - * {@link java.nio.charset.Charset#forName(String)}. - * @deprecated use {@link #setEncoding(Charset)} instead. - */ - @Deprecated - public void setEncoding(String encodingName) { - setEncoding(Charset.forName(encodingName)); - } - @Override public byte[] build() throws UnsupportedEncodingException { ByteArrayOutputStream os = new ByteArrayOutputStream(); -- cgit v1.2.3 From 3585035a71ff5eba66aba792dbbffb5bcb4690e0 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 30 Aug 2024 17:24:00 +0200 Subject: Remove deprecated ConfigConstants#CONFIG_KEY_STREAM_FILE_TRESHOLD Change-Id: I3521ba8f8456160bd18ccb22c7d4a131aaac4ff2 --- org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java | 6 ------ 1 file changed, 6 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java index 4be0cceedc..acb54d7405 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java @@ -358,12 +358,6 @@ public final class ConfigConstants { */ public static final String CONFIG_KEY_STREAM_FILE_THRESHOLD = "streamFileThreshold"; - /** - * @deprecated typo, use CONFIG_KEY_STREAM_FILE_THRESHOLD instead - */ - @Deprecated(since = "6.8") - public static final String CONFIG_KEY_STREAM_FILE_TRESHOLD = CONFIG_KEY_STREAM_FILE_THRESHOLD; - /** * The "packedGitMmap" key * @since 5.1.13 -- cgit v1.2.3 From 16f8db059adb87d0544d17e3deb58886ed41adb5 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 30 Aug 2024 17:28:57 +0200 Subject: Constants: remove deprecated #CHARSET and #CHARACTER_ENCODING Change-Id: If2d3c1a96560e0bc5e352bdbcab4c191cbed1a42 --- .../src/org/eclipse/jgit/lib/Constants.java | 24 +--------------------- 1 file changed, 1 insertion(+), 23 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java index 8599bf7dbc..3eb35be79a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java @@ -15,7 +15,6 @@ package org.eclipse.jgit.lib; import static java.nio.charset.StandardCharsets.UTF_8; import java.nio.ByteBuffer; -import java.nio.charset.Charset; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.text.MessageFormat; @@ -203,24 +202,6 @@ public final class Constants { */ public static final byte[] PACK_SIGNATURE = { 'P', 'A', 'C', 'K' }; - /** - * Native character encoding for commit messages, file names... - * - * @deprecated Use {@link java.nio.charset.StandardCharsets#UTF_8} directly - * instead. - */ - @Deprecated - public static final Charset CHARSET; - - /** - * Native character encoding for commit messages, file names... - * - * @deprecated Use {@link java.nio.charset.StandardCharsets#UTF_8} directly - * instead. - */ - @Deprecated - public static final String CHARACTER_ENCODING; - /** Default main branch name */ public static final String MASTER = "master"; @@ -745,13 +726,12 @@ public final class Constants { } /** - * Convert a string to a byte array in the standard character encoding. + * Convert a string to a byte array in the standard character encoding UTF8. * * @param str * the string to convert. May contain any Unicode characters. * @return a byte array representing the requested string, encoded using the * default character encoding (UTF-8). - * @see #CHARACTER_ENCODING */ public static byte[] encode(String str) { final ByteBuffer bb = UTF_8.encode(str); @@ -770,8 +750,6 @@ public final class Constants { static { if (OBJECT_ID_LENGTH != newMessageDigest().getDigestLength()) throw new LinkageError(JGitText.get().incorrectOBJECT_ID_LENGTH); - CHARSET = UTF_8; - CHARACTER_ENCODING = UTF_8.name(); } /** name of the file containing the commit msg for a merge commit */ -- cgit v1.2.3 From 8140ad78d1659355bcf647188ccc7fa47dbf4b33 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 30 Aug 2024 17:32:04 +0200 Subject: CoreConfig: remove deprecated #isLogAllRefUpdates method Change-Id: I4e5f96696b57512488f48e66a82760b2e8671878 --- .../src/org/eclipse/jgit/lib/CoreConfig.java | 19 ------------------- 1 file changed, 19 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/CoreConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/CoreConfig.java index 9fa5d75a3f..49602a75eb 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/CoreConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/CoreConfig.java @@ -163,8 +163,6 @@ public class CoreConfig { private final int packIndexVersion; - private final LogRefUpdates logAllRefUpdates; - private final String excludesfile; private final String attributesfile; @@ -205,9 +203,6 @@ public class CoreConfig { ConfigConstants.CONFIG_KEY_COMPRESSION, DEFAULT_COMPRESSION); packIndexVersion = rc.getInt(ConfigConstants.CONFIG_PACK_SECTION, ConfigConstants.CONFIG_KEY_INDEXVERSION, 2); - logAllRefUpdates = rc.getEnum(ConfigConstants.CONFIG_CORE_SECTION, null, - ConfigConstants.CONFIG_KEY_LOGALLREFUPDATES, - LogRefUpdates.TRUE); excludesfile = rc.getString(ConfigConstants.CONFIG_CORE_SECTION, null, ConfigConstants.CONFIG_KEY_EXCLUDESFILE); attributesfile = rc.getString(ConfigConstants.CONFIG_CORE_SECTION, @@ -235,20 +230,6 @@ public class CoreConfig { return packIndexVersion; } - /** - * Whether to log all refUpdates - * - * @return whether to log all refUpdates - * @deprecated since 5.6; default value depends on whether the repository is - * bare. Use - * {@link Config#getEnum(String, String, String, Enum)} - * directly. - */ - @Deprecated - public boolean isLogAllRefUpdates() { - return !LogRefUpdates.FALSE.equals(logAllRefUpdates); - } - /** * Get path of excludesfile * -- cgit v1.2.3 From db9443d80505bd4b75326ca6a2ef41cc4beb24e4 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 30 Aug 2024 17:33:43 +0200 Subject: Remove deprecated RefDatabase#getRef(String) method Change-Id: I89f42db2b9dabee18d4220457436b9f9b6340f50 --- .../src/org/eclipse/jgit/lib/RefDatabase.java | 17 ----------------- 1 file changed, 17 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefDatabase.java index 9e05a39731..2cf24185c7 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefDatabase.java @@ -237,23 +237,6 @@ public abstract class RefDatabase { return false; } - /** - * Compatibility synonym for {@link #findRef(String)}. - * - * @param name - * the name of the reference. May be a short name which must be - * searched for using the standard {@link #SEARCH_PATH}. - * @return the reference (if it exists); else {@code null}. - * @throws IOException - * the reference space cannot be accessed. - * @deprecated Use {@link #findRef(String)} instead. - */ - @Deprecated - @Nullable - public final Ref getRef(String name) throws IOException { - return findRef(name); - } - /** * Read a single reference. *

-- cgit v1.2.3 From ac1265a513cb6953ef6ddada61eed2235bff186f Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 30 Aug 2024 17:36:05 +0200 Subject: Make deprecated Repository#peel(Ref) private Change-Id: I1c16196bba00a5d0f54c10261cc08185305ba4a3 --- org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java index 9dde99fada..b1ecca4b60 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java @@ -1164,11 +1164,9 @@ public abstract class Repository implements AutoCloseable { * new Ref object representing the same data as Ref, but isPeeled() * will be true and getPeeledObjectId will contain the peeled object * (or null). - * @deprecated use {@code getRefDatabase().peel(ref)} instead. */ - @Deprecated @NonNull - public Ref peel(Ref ref) { + private Ref peel(Ref ref) { try { return getRefDatabase().peel(ref); } catch (IOException e) { -- cgit v1.2.3 From 925f0d14df661046a257c5035351b17106165765 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 30 Aug 2024 17:36:58 +0200 Subject: Remove deprecated Repository#hasObject(AnyObjectId) method Change-Id: I473dff6bdc23cfb126d22e18c168390a0e21301d --- .../src/org/eclipse/jgit/lib/Repository.java | 19 ------------------- 1 file changed, 19 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java index b1ecca4b60..0562840915 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java @@ -306,25 +306,6 @@ public abstract class Repository implements AutoCloseable { return fs; } - /** - * Whether the specified object is stored in this repo or any of the known - * shared repositories. - * - * @param objectId - * a {@link org.eclipse.jgit.lib.AnyObjectId} object. - * @return true if the specified object is stored in this repo or any of the - * known shared repositories. - * @deprecated use {@code getObjectDatabase().has(objectId)} - */ - @Deprecated - public boolean hasObject(AnyObjectId objectId) { - try { - return getObjectDatabase().has(objectId); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - /** * Open an object from this repository. *

-- cgit v1.2.3 From 7162722e9c2d373d64f8db4160be26ec440bbeb6 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 30 Aug 2024 17:37:48 +0200 Subject: Remove deprecated TagBuilder#toByteArray method Change-Id: I14e78bcd4bbdb491bcc44a53ff19609b79c0831b --- .../src/org/eclipse/jgit/lib/TagBuilder.java | 17 ----------------- 1 file changed, 17 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/TagBuilder.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/TagBuilder.java index bbc614448f..ea73d95102 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/TagBuilder.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/TagBuilder.java @@ -206,23 +206,6 @@ public class TagBuilder extends ObjectBuilder { return os.toByteArray(); } - /** - * Format this builder's state as an annotated tag object. - * - * @return this object in the canonical annotated tag format, suitable for - * storage in a repository, or {@code null} if the tag cannot be - * encoded - * @deprecated since 5.11; use {@link #build()} instead - */ - @Deprecated - public byte[] toByteArray() { - try { - return build(); - } catch (UnsupportedEncodingException e) { - return null; - } - } - @SuppressWarnings("nls") @Override public String toString() { -- cgit v1.2.3 From 987a24ba220012c6200bbbb7b80bdc8e56e89d44 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 30 Aug 2024 17:42:16 +0200 Subject: Remove deprecated MergeFormatter#formatMerge using charset name Change-Id: Id2bdcb865203ed192fea35cfcf82c34667710726 --- .../src/org/eclipse/jgit/merge/MergeFormatter.java | 34 ---------------------- 1 file changed, 34 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/merge/MergeFormatter.java b/org.eclipse.jgit/src/org/eclipse/jgit/merge/MergeFormatter.java index a35b30eb01..5006f9e0cd 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/merge/MergeFormatter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/merge/MergeFormatter.java @@ -111,40 +111,6 @@ public class MergeFormatter { new MergeFormatterPass(out, res, seqName, charset, true).formatMerge(); } - /** - * Formats the results of a merge of exactly two - * {@link org.eclipse.jgit.diff.RawText} objects in a Git conformant way. - * This convenience method accepts the names for the three sequences (base - * and the two merged sequences) as explicit parameters and doesn't require - * the caller to specify a List - * - * @param out - * the {@link java.io.OutputStream} where to write the textual - * presentation - * @param res - * the merge result which should be presented - * @param baseName - * the name ranges from the base should get - * @param oursName - * the name ranges from ours should get - * @param theirsName - * the name ranges from theirs should get - * @param charsetName - * the name of the character set used when writing conflict - * metadata - * @throws java.io.IOException - * if an IO error occurred - * @deprecated use - * {@link #formatMerge(OutputStream, MergeResult, String, String, String, Charset)} - * instead. - */ - @Deprecated - public void formatMerge(OutputStream out, MergeResult res, String baseName, - String oursName, String theirsName, String charsetName) throws IOException { - formatMerge(out, res, baseName, oursName, theirsName, - Charset.forName(charsetName)); - } - /** * Formats the results of a merge of exactly two * {@link org.eclipse.jgit.diff.RawText} objects in a Git conformant way. -- cgit v1.2.3 From e5422546003f7ad623d821e2e84abfd5f34a56c2 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 30 Aug 2024 17:43:10 +0200 Subject: Remove deprecated MergeFormatter#formatMerge using charset name Change-Id: I7d0bdb61a8698e94bb40c22fe1c40c70cec65dfc --- .../src/org/eclipse/jgit/merge/MergeFormatter.java | 31 ---------------------- 1 file changed, 31 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/merge/MergeFormatter.java b/org.eclipse.jgit/src/org/eclipse/jgit/merge/MergeFormatter.java index 5006f9e0cd..079db4a07f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/merge/MergeFormatter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/merge/MergeFormatter.java @@ -22,37 +22,6 @@ import org.eclipse.jgit.diff.RawText; * A class to convert merge results into a Git conformant textual presentation */ public class MergeFormatter { - /** - * Formats the results of a merge of {@link org.eclipse.jgit.diff.RawText} - * objects in a Git conformant way. This method also assumes that the - * {@link org.eclipse.jgit.diff.RawText} objects being merged are line - * oriented files which use LF as delimiter. This method will also use LF to - * separate chunks and conflict metadata, therefore it fits only to texts - * that are LF-separated lines. - * - * @param out - * the output stream where to write the textual presentation - * @param res - * the merge result which should be presented - * @param seqName - * When a conflict is reported each conflicting range will get a - * name. This name is following the "<<<<<<< - * " or ">>>>>>> " conflict markers. The - * names for the sequences are given in this list - * @param charsetName - * the name of the character set used when writing conflict - * metadata - * @throws java.io.IOException - * if an IO error occurred - * @deprecated Use - * {@link #formatMerge(OutputStream, MergeResult, List, Charset)} - * instead. - */ - @Deprecated - public void formatMerge(OutputStream out, MergeResult res, - List seqName, String charsetName) throws IOException { - formatMerge(out, res, seqName, Charset.forName(charsetName)); - } /** * Formats the results of a merge of {@link org.eclipse.jgit.diff.RawText} -- cgit v1.2.3 From 42d3182614b82cfe139012fb3c66bfeae444b91a Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 30 Aug 2024 17:44:24 +0200 Subject: Remove deprecated MergeMessageFormatter#formatWithConflicts Change-Id: I3040f655318c47f268433294720a99325ae78863 --- .../org/eclipse/jgit/merge/MergeMessageFormatter.java | 18 ------------------ 1 file changed, 18 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/merge/MergeMessageFormatter.java b/org.eclipse.jgit/src/org/eclipse/jgit/merge/MergeMessageFormatter.java index e0c083f55c..039d7d844c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/merge/MergeMessageFormatter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/merge/MergeMessageFormatter.java @@ -91,24 +91,6 @@ public class MergeMessageFormatter { return sb.toString(); } - /** - * Add section with conflicting paths to merge message. Lines are prefixed - * with a hash. - * - * @param message - * the original merge message - * @param conflictingPaths - * the paths with conflicts - * @return merge message with conflicting paths added - * @deprecated since 6.1; use - * {@link #formatWithConflicts(String, Iterable, char)} instead - */ - @Deprecated - public String formatWithConflicts(String message, - List conflictingPaths) { - return formatWithConflicts(message, conflictingPaths, '#'); - } - /** * Add section with conflicting paths to merge message. * -- cgit v1.2.3 From 0143d6de0891b3ecd5a6212a659bfc9c1b1b3da9 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 30 Aug 2024 17:45:41 +0200 Subject: Remove deprecated PatchApplier#applyPatch(InputStream) Change-Id: I79923b95861af983b676d436938aaeebe8c5b658 --- .../src/org/eclipse/jgit/patch/PatchApplier.java | 27 ---------------------- 1 file changed, 27 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/patch/PatchApplier.java b/org.eclipse.jgit/src/org/eclipse/jgit/patch/PatchApplier.java index cb6cc6efa7..23e09b9479 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/patch/PatchApplier.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/patch/PatchApplier.java @@ -41,7 +41,6 @@ import java.util.zip.InflaterInputStream; import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.api.errors.FilterFailedException; -import org.eclipse.jgit.api.errors.PatchFormatException; import org.eclipse.jgit.attributes.Attribute; import org.eclipse.jgit.attributes.Attributes; import org.eclipse.jgit.attributes.FilterCommand; @@ -287,32 +286,6 @@ public class PatchApplier { } } - /** - * Applies the given patch - * - * @param patchInput - * the patch to apply. - * @return the result of the patch - * @throws PatchFormatException - * if the patch cannot be parsed - * @throws IOException - * if the patch read fails - * @deprecated use {@link #applyPatch(Patch)} instead - */ - @Deprecated - public Result applyPatch(InputStream patchInput) - throws PatchFormatException, IOException { - Patch p = new Patch(); - try (InputStream inStream = patchInput) { - p.parse(inStream); - - if (!p.getErrors().isEmpty()) { - throw new PatchFormatException(p.getErrors()); - } - } - return applyPatch(p); - } - /** * Applies the given patch * -- cgit v1.2.3 From fc76f226c3aa8e55683f95b56e24d47e91ce0e47 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 30 Aug 2024 17:47:16 +0200 Subject: WindowCacheStats: remove deprecated #getOpenFiles, #getOpenBytes Change-Id: If254177fd7914cb1f909f6620032bb4016208855 --- .../eclipse/jgit/storage/file/WindowCacheStats.java | 21 --------------------- 1 file changed, 21 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCacheStats.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCacheStats.java index 7cb8618302..668b92cf53 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCacheStats.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/file/WindowCacheStats.java @@ -22,27 +22,6 @@ import org.eclipse.jgit.internal.storage.file.WindowCache; */ @MXBean public interface WindowCacheStats { - /** - * Get number of open files - * - * @return the number of open files. - * @deprecated use {@link #getOpenFileCount()} instead - */ - @Deprecated - public static int getOpenFiles() { - return (int) WindowCache.getInstance().getStats().getOpenFileCount(); - } - - /** - * Get number of open bytes - * - * @return the number of open bytes. - * @deprecated use {@link #getOpenByteCount()} instead - */ - @Deprecated - public static long getOpenBytes() { - return WindowCache.getInstance().getStats().getOpenByteCount(); - } /** * Get cache statistics -- cgit v1.2.3 From 7b65a847219308beaf5b4542e45ab2927ff0e4f8 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 30 Aug 2024 17:49:12 +0200 Subject: PacketLineIn: make deprecated #END and #DELIM private Change-Id: I1b7ba9f7e3dffe54313fc5d27051ad56a02a05b9 --- .../src/org/eclipse/jgit/transport/PacketLineIn.java | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/PacketLineIn.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/PacketLineIn.java index ed33eaed07..614ad88246 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/PacketLineIn.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/PacketLineIn.java @@ -43,24 +43,13 @@ public class PacketLineIn { /** * Magic return from {@link #readString()} when a flush packet is found. - * - * @deprecated Callers should use {@link #isEnd(String)} to check if a - * string is the end marker, or - * {@link PacketLineIn#readStrings()} to iterate over all - * strings in the input stream until the marker is reached. */ - @Deprecated - public static final String END = new String(); /* must not string pool */ + private static final String END = new String(); /* must not string pool */ /** * Magic return from {@link #readString()} when a delim packet is found. - * - * @since 5.0 - * @deprecated Callers should use {@link #isDelimiter(String)} to check if a - * string is the delimiter. */ - @Deprecated - public static final String DELIM = new String(); /* must not string pool */ + private static final String DELIM = new String(); /* must not string pool */ enum AckNackResult { /** NAK */ -- cgit v1.2.3 From 66f340865b9470fa8fbc9d12e0c08bf09ebf8364 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 30 Aug 2024 17:50:59 +0200 Subject: Remove deprecated class ReceivePack.FirstLine Change-Id: I1143612088dba4fdced8f14c97293d17ff9658bb --- .../org/eclipse/jgit/transport/ReceivePack.java | 46 ---------------------- 1 file changed, 46 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/ReceivePack.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/ReceivePack.java index ddde6038e9..21752a1294 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/ReceivePack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/ReceivePack.java @@ -88,52 +88,6 @@ import org.eclipse.jgit.util.io.TimeoutOutputStream; * Implements the server side of a push connection, receiving objects. */ public class ReceivePack { - /** - * Data in the first line of a request, the line itself plus capabilities. - * - * @deprecated Use {@link FirstCommand} instead. - * @since 5.6 - */ - @Deprecated - public static class FirstLine { - private final FirstCommand command; - - /** - * Parse the first line of a receive-pack request. - * - * @param line - * line from the client. - */ - public FirstLine(String line) { - command = FirstCommand.fromLine(line); - } - - /** - * Get non-capabilities part of the line - * - * @return non-capabilities part of the line. - */ - public String getLine() { - return command.getLine(); - } - - /** - * Get capabilities parsed from the line - * - * @return capabilities parsed from the line. - */ - public Set getCapabilities() { - Set reconstructedCapabilites = new HashSet<>(); - for (Map.Entry e : command.getCapabilities() - .entrySet()) { - String cap = e.getValue() == null ? e.getKey() - : e.getKey() + "=" + e.getValue(); //$NON-NLS-1$ - reconstructedCapabilites.add(cap); - } - - return reconstructedCapabilites; - } - } /** Database we write the stored objects into. */ private final Repository db; -- cgit v1.2.3 From 0d4e81e50cebce9a75f0a33b2106c9922375c77a Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 30 Aug 2024 21:49:54 +0200 Subject: Remove deprecated ReceivePack#setEchoCommandFailures method Change-Id: Ieeeb33b8b3dc8272023af5d47b3330f4a94c5b96 --- .../src/org/eclipse/jgit/transport/ReceivePack.java | 16 ---------------- 1 file changed, 16 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/ReceivePack.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/ReceivePack.java index 21752a1294..6f211e0794 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/ReceivePack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/ReceivePack.java @@ -2102,22 +2102,6 @@ public class ReceivePack { this.unpackErrorHandler = unpackErrorHandler; } - /** - * Set whether this class will report command failures as warning messages - * before sending the command results. - * - * @param echo - * if true this class will report command failures as warning - * messages before sending the command results. This is usually - * not necessary, but may help buggy Git clients that discard the - * errors when all branches fail. - * @deprecated no widely used Git versions need this any more - */ - @Deprecated - public void setEchoCommandFailures(boolean echo) { - // No-op. - } - /** * Get the client session-id * -- cgit v1.2.3 From d67227653a776a39e3da64e5ee134dab831c1b06 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 30 Aug 2024 21:51:27 +0200 Subject: Remove deprecated RefAdvertiser#send(Map) method Change-Id: Ifffcfa6bf9baf9f6879a5a7e0f05d317347983f6 --- .../org/eclipse/jgit/transport/RefAdvertiser.java | 21 --------------------- 1 file changed, 21 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/RefAdvertiser.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/RefAdvertiser.java index f72c421920..3d4bea2e48 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/RefAdvertiser.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/RefAdvertiser.java @@ -178,7 +178,6 @@ public abstract class RefAdvertiser { * * This method must be invoked prior to any of the following: *

    - *
  • {@link #send(Map)}
    • *
  • {@link #send(Collection)}
    • *
* @@ -195,7 +194,6 @@ public abstract class RefAdvertiser { *

* This method must be invoked prior to any of the following: *

    - *
  • {@link #send(Map)}
    • *
  • {@link #send(Collection)}
    • *
  • {@link #advertiseHave(AnyObjectId)}
    • *
@@ -230,7 +228,6 @@ public abstract class RefAdvertiser { *

* This method must be invoked prior to any of the following: *

    - *
  • {@link #send(Map)}
    • *
  • {@link #send(Collection)}
    • *
  • {@link #advertiseHave(AnyObjectId)}
    • *
@@ -249,24 +246,6 @@ public abstract class RefAdvertiser { } } - /** - * Format an advertisement for the supplied refs. - * - * @param refs - * zero or more refs to format for the client. The collection is - * sorted before display if necessary, and therefore may appear - * in any order. - * @return set of ObjectIds that were advertised to the client. - * @throws java.io.IOException - * the underlying output stream failed to write out an - * advertisement record. - * @deprecated use {@link #send(Collection)} instead. - */ - @Deprecated - public Set send(Map refs) throws IOException { - return send(refs.values()); - } - /** * Format an advertisement for the supplied refs. * -- cgit v1.2.3 From d64e09230c8bf1edae9566da15dd71b17d37b110 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 30 Aug 2024 21:52:31 +0200 Subject: Remove deprecated Transport#getFilterBlobLimit methods Change-Id: Ia72cc228b0acaa8df8f405bd224916cc76eadd22 --- .../src/org/eclipse/jgit/transport/Transport.java | 22 ---------------------- 1 file changed, 22 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/Transport.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/Transport.java index b335675da5..ac76e83d34 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/Transport.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/Transport.java @@ -1120,28 +1120,6 @@ public abstract class Transport implements AutoCloseable { removeDeletedRefs = remove; } - /** - * @return the blob limit value set with {@link #setFilterBlobLimit} or - * {@link #setFilterSpec(FilterSpec)}, or -1 if no blob limit value - * was set - * @since 5.0 - * @deprecated Use {@link #getFilterSpec()} instead - */ - @Deprecated - public final long getFilterBlobLimit() { - return filterSpec.getBlobLimit(); - } - - /** - * @param bytes exclude blobs of size greater than this - * @since 5.0 - * @deprecated Use {@link #setFilterSpec(FilterSpec)} instead - */ - @Deprecated - public final void setFilterBlobLimit(long bytes) { - setFilterSpec(FilterSpec.withBlobLimit(bytes)); - } - /** * Get filter spec * -- cgit v1.2.3 From 88053eedcc89d6fded81e5edb39ae41d237ba3dd Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 30 Aug 2024 21:54:50 +0200 Subject: UserAgent: remove deprecated #getAgent, #hasAgent methods Change-Id: Ib53de6dabea7f73ecfde85cb30f49fa05ee48551 --- .../src/org/eclipse/jgit/transport/UserAgent.java | 41 ---------------------- 1 file changed, 41 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UserAgent.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UserAgent.java index 7b052ad4a7..b23ee97dcb 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UserAgent.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UserAgent.java @@ -10,10 +10,6 @@ package org.eclipse.jgit.transport; -import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_AGENT; - -import java.util.Set; - import org.eclipse.jgit.util.StringUtils; /** @@ -91,43 +87,6 @@ public class UserAgent { userAgent = StringUtils.isEmptyOrNull(agent) ? null : clean(agent); } - /** - * - * @param options - * options - * @param transportAgent - * name of transport agent - * @return The transport agent. - * @deprecated Capabilities with <key>=<value> shape are now - * parsed alongside other capabilities in the ReceivePack flow. - */ - @Deprecated - static String getAgent(Set options, String transportAgent) { - if (options == null || options.isEmpty()) { - return transportAgent; - } - for (String o : options) { - if (o.startsWith(OPTION_AGENT) - && o.length() > OPTION_AGENT.length() - && o.charAt(OPTION_AGENT.length()) == '=') { - return o.substring(OPTION_AGENT.length() + 1); - } - } - return transportAgent; - } - - /** - * - * @param options - * options - * @return True if the transport agent is set. False otherwise. - * @deprecated Capabilities with <key>=<value> shape are now - * parsed alongside other capabilities in the ReceivePack flow. - */ - @Deprecated - static boolean hasAgent(Set options) { - return getAgent(options, null) != null; - } private UserAgent() { } -- cgit v1.2.3 From 8baef22711676bf853dfc24ef145122a335f4312 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 30 Aug 2024 21:59:53 +0200 Subject: Remove deprecated lastModified methods using long from - FileUtils#lastModified - FileUtils#setLastModified - FS#getLastModifiedTime - FS#lastModified - FS#setLastModified - FileTreeIterator.Entry#getEntryLastModified - WorkingTreeIterator#getEntryLastModified - WorkingTreeIterator.Entry#getEntryLastModified Change-Id: I5073f05c32f8f626383a91048470c79332983121 --- .../eclipse/jgit/treewalk/FileTreeIterator.java | 6 --- .../eclipse/jgit/treewalk/WorkingTreeIterator.java | 27 ------------ org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java | 50 ---------------------- .../src/org/eclipse/jgit/util/FileUtils.java | 33 -------------- 4 files changed, 116 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/FileTreeIterator.java b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/FileTreeIterator.java index 36fa72028e..0cac374844 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/FileTreeIterator.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/FileTreeIterator.java @@ -371,12 +371,6 @@ public class FileTreeIterator extends WorkingTreeIterator { return attributes.getLength(); } - @Override - @Deprecated - public long getLastModified() { - return attributes.getLastModifiedInstant().toEpochMilli(); - } - /** * @since 5.1.9 */ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java index 95e9964192..f16d800f63 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/WorkingTreeIterator.java @@ -619,18 +619,6 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator { return canonLen; } - /** - * Get the last modified time of this entry. - * - * @return last modified time of this file, in milliseconds since the epoch - * (Jan 1, 1970 UTC). - * @deprecated use {@link #getEntryLastModifiedInstant()} instead - */ - @Deprecated - public long getEntryLastModified() { - return current().getLastModified(); - } - /** * Get the last modified time of this entry. * @@ -1222,21 +1210,6 @@ public abstract class WorkingTreeIterator extends AbstractTreeIterator { */ public abstract long getLength(); - /** - * Get the last modified time of this entry. - *

- * Note: Efficient implementation required. - *

- * The implementation of this method must be efficient. If a subclass - * needs to compute the value they should cache the reference within an - * instance member instead. - * - * @return time since the epoch (in ms) of the last change. - * @deprecated use {@link #getLastModifiedInstant()} instead - */ - @Deprecated - public abstract long getLastModified(); - /** * Get the last modified time of this entry. *

diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java index 6933a6c567..4433dccc62 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java @@ -1080,24 +1080,6 @@ public abstract class FS { */ public abstract boolean setExecute(File f, boolean canExec); - /** - * Get the last modified time of a file system object. If the OS/JRE support - * symbolic links, the modification time of the link is returned, rather - * than that of the link target. - * - * @param f - * a {@link java.io.File} object. - * @return last modified time of f - * @throws java.io.IOException - * if an IO error occurred - * @since 3.0 - * @deprecated use {@link #lastModifiedInstant(Path)} instead - */ - @Deprecated - public long lastModified(File f) throws IOException { - return FileUtils.lastModified(f); - } - /** * Get the last modified time of a file system object. If the OS/JRE support * symbolic links, the modification time of the link is returned, rather @@ -1126,25 +1108,6 @@ public abstract class FS { return FileUtils.lastModifiedInstant(f.toPath()); } - /** - * Set the last modified time of a file system object. - *

- * For symlinks it sets the modified time of the link target. - * - * @param f - * a {@link java.io.File} object. - * @param time - * last modified time - * @throws java.io.IOException - * if an IO error occurred - * @since 3.0 - * @deprecated use {@link #setLastModified(Path, Instant)} instead - */ - @Deprecated - public void setLastModified(File f, long time) throws IOException { - FileUtils.setLastModified(f, time); - } - /** * Set the last modified time of a file system object. *

@@ -2425,19 +2388,6 @@ public abstract class FS { return creationTime; } - /** - * Get the time when the file was last modified in milliseconds since - * the epoch - * - * @return the time (milliseconds since 1970-01-01) when this object was - * last modified - * @deprecated use getLastModifiedInstant instead - */ - @Deprecated - public long getLastModifiedTime() { - return lastModifiedInstant.toEpochMilli(); - } - /** * Get the time when this object was last modified * diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FileUtils.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FileUtils.java index cab0e6a0a9..39c67f1b86 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FileUtils.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FileUtils.java @@ -770,24 +770,6 @@ public class FileUtils { return Files.isSymbolicLink(file.toPath()); } - /** - * Get the lastModified attribute for a given file - * - * @param file - * the file - * @return lastModified attribute for given file, not following symbolic - * links - * @throws IOException - * if an IO error occurred - * @deprecated use {@link #lastModifiedInstant(Path)} instead which returns - * FileTime - */ - @Deprecated - static long lastModified(File file) throws IOException { - return Files.getLastModifiedTime(toPath(file), LinkOption.NOFOLLOW_LINKS) - .toMillis(); - } - /** * Get last modified timestamp of a file * @@ -827,21 +809,6 @@ public class FileUtils { return Files.readAttributes(file.toPath(), BasicFileAttributes.class, LinkOption.NOFOLLOW_LINKS); } - /** - * Set the last modified time of a file system object. - * - * @param file - * the file - * @param time - * last modified timestamp - * @throws IOException - * if an IO error occurred - */ - @Deprecated - static void setLastModified(File file, long time) throws IOException { - Files.setLastModifiedTime(toPath(file), FileTime.fromMillis(time)); - } - /** * Set the last modified time of a file system object. * -- cgit v1.2.3 From 0b531cd6e3f6a71975c208ce67f298a0b672d32d Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 30 Aug 2024 22:07:17 +0200 Subject: FS, FS_Posix: remove deprecated #createNewFile(File) method Change-Id: Id34a0be998eee360e69f74b469c4990afa153c1b --- org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java | 19 ------ .../src/org/eclipse/jgit/util/FS_POSIX.java | 67 ---------------------- 2 files changed, 86 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java index 4433dccc62..8e0b93f082 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java @@ -1762,25 +1762,6 @@ public abstract class FS { FileUtils.createSymLink(path, target); } - /** - * Create a new file. See {@link java.io.File#createNewFile()}. Subclasses - * of this class may take care to provide a safe implementation for this - * even if {@link #supportsAtomicCreateNewFile()} is false - * - * @param path - * the file to be created - * @return true if the file was created, false if - * the file already existed - * @throws java.io.IOException - * if an IO error occurred - * @deprecated use {@link #createNewFileAtomic(File)} instead - * @since 4.5 - */ - @Deprecated - public boolean createNewFile(File path) throws IOException { - return path.createNewFile(); - } - /** * A token representing a file created by * {@link #createNewFileAtomic(File)}. The token must be retained until the diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_POSIX.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_POSIX.java index e73095f5a8..db2b5b4f71 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_POSIX.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_POSIX.java @@ -341,73 +341,6 @@ public class FS_POSIX extends FS { return supportsAtomicFileCreation == AtomicFileCreation.SUPPORTED; } - @Override - @SuppressWarnings("boxing") - /** - * {@inheritDoc} - *

- * An implementation of the File#createNewFile() semantics which works also - * on NFS. If the config option - * {@code core.supportsAtomicCreateNewFile = true} (which is the default) - * then simply File#createNewFile() is called. - * - * But if {@code core.supportsAtomicCreateNewFile = false} then after - * successful creation of the lock file a hard link to that lock file is - * created and the attribute nlink of the lock file is checked to be 2. If - * multiple clients manage to create the same lock file nlink would be - * greater than 2 showing the error. - * - * @see "https://www.time-travellers.org/shane/papers/NFS_considered_harmful.html" - * - * @deprecated use {@link FS_POSIX#createNewFileAtomic(File)} instead - * @since 4.5 - */ - @Deprecated - public boolean createNewFile(File lock) throws IOException { - if (!lock.createNewFile()) { - return false; - } - if (supportsAtomicCreateNewFile()) { - return true; - } - Path lockPath = lock.toPath(); - Path link = null; - FileStore store = null; - try { - store = Files.getFileStore(lockPath); - } catch (SecurityException e) { - return true; - } - try { - Boolean canLink = CAN_HARD_LINK.computeIfAbsent(store, - s -> Boolean.TRUE); - if (Boolean.FALSE.equals(canLink)) { - return true; - } - link = Files.createLink( - Paths.get(lock.getAbsolutePath() + ".lnk"), //$NON-NLS-1$ - lockPath); - Integer nlink = (Integer) Files.getAttribute(lockPath, - "unix:nlink"); //$NON-NLS-1$ - if (nlink > 2) { - LOG.warn(MessageFormat.format( - JGitText.get().failedAtomicFileCreation, lockPath, - nlink)); - return false; - } else if (nlink < 2) { - CAN_HARD_LINK.put(store, Boolean.FALSE); - } - return true; - } catch (UnsupportedOperationException | IllegalArgumentException e) { - CAN_HARD_LINK.put(store, Boolean.FALSE); - return true; - } finally { - if (link != null) { - Files.delete(link); - } - } - } - /** * {@inheritDoc} *

-- cgit v1.2.3 From 2a10b416f9468a6c4cc09eca68678131ec7e7db4 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 30 Aug 2024 22:09:11 +0200 Subject: Remove deprecated FS#setAsyncFileStoreAttributes method Change-Id: Iea9f61fd65772ab247b382c93035e3c9974705aa --- org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java | 15 --------------- 1 file changed, 15 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java index 8e0b93f082..860c1c92fd 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java @@ -897,21 +897,6 @@ public abstract class FS { return detect(null); } - /** - * Whether FileStore attributes should be determined asynchronously - * - * @param asynch - * whether FileStore attributes should be determined - * asynchronously. If false access to cached attributes may block - * for some seconds for the first call per FileStore - * @since 5.1.9 - * @deprecated Use {@link FileStoreAttributes#setBackground} instead - */ - @Deprecated - public static void setAsyncFileStoreAttributes(boolean asynch) { - FileStoreAttributes.setBackground(asynch); - } - /** * Auto-detect the appropriate file system abstraction, taking into account * the presence of a Cygwin installation on the system. Using jgit in -- cgit v1.2.3 From f9a608b57e9e217578d9b37c74c249e3f3837661 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 30 Aug 2024 22:17:23 +0200 Subject: Remove deprecated RawParseUtils#UTF8_CHARSET Change-Id: I4b3f1344f77a575b2d0e00462806eddb4e9a6b80 --- org.eclipse.jgit/src/org/eclipse/jgit/util/RawParseUtils.java | 8 -------- 1 file changed, 8 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/RawParseUtils.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/RawParseUtils.java index 46d0bc85ff..2ce86904c5 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/RawParseUtils.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/RawParseUtils.java @@ -44,14 +44,6 @@ import org.eclipse.jgit.lib.PersonIdent; * Handy utility functions to parse raw object contents. */ public final class RawParseUtils { - /** - * UTF-8 charset constant. - * - * @since 2.2 - * @deprecated use {@link java.nio.charset.StandardCharsets#UTF_8} instead - */ - @Deprecated - public static final Charset UTF8_CHARSET = UTF_8; private static final byte[] digits10; -- cgit v1.2.3 From 0ae586b81fa5deb46633ccaa29607f565587d841 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 30 Aug 2024 22:18:50 +0200 Subject: Remove deprecated AutoLFInputStream constructors Change-Id: I00241e45d947582886658fa528cc20a961fed9e6 --- .../eclipse/jgit/util/io/AutoLFInputStream.java | 38 ---------------------- 1 file changed, 38 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/AutoLFInputStream.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/AutoLFInputStream.java index 2385865674..4b9706a3ce 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/AutoLFInputStream.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/AutoLFInputStream.java @@ -147,44 +147,6 @@ public class AutoLFInputStream extends InputStream { && flags.contains(StreamFlag.FOR_CHECKOUT); } - /** - * Creates a new InputStream, wrapping the specified stream. - * - * @param in - * raw input stream - * @param detectBinary - * whether binaries should be detected - * @since 2.0 - * @deprecated since 5.9, use {@link #create(InputStream, StreamFlag...)} - * instead - */ - @Deprecated - public AutoLFInputStream(InputStream in, boolean detectBinary) { - this(in, detectBinary, false); - } - - /** - * Creates a new InputStream, wrapping the specified stream. - * - * @param in - * raw input stream - * @param detectBinary - * whether binaries should be detected - * @param abortIfBinary - * throw an IOException if the file is binary - * @since 3.3 - * @deprecated since 5.9, use {@link #create(InputStream, StreamFlag...)} - * instead - */ - @Deprecated - public AutoLFInputStream(InputStream in, boolean detectBinary, - boolean abortIfBinary) { - this.in = in; - this.detectBinary = detectBinary; - this.abortIfBinary = abortIfBinary; - this.forCheckout = false; - } - @Override public int read() throws IOException { final int read = read(single, 0, 1); -- cgit v1.2.3 From e5d2898997462e0f2409c09497ab62c6cda2dbaf Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 6 Sep 2024 12:26:44 +0200 Subject: Replace custom encoder `Constants#encode` by JDK implementation Using the implementation provided in the JDK since Java 1.6 by `String#getBytes(Charset)` reduces JGit maintenance effort and improves performance. The method Constants#encode was implemented when JGit still used Java 1.5. See [1]. Kudos to Marcin for proposing to use this improvement in RefWriter [2]. I think it should be used generally. [1] https://repo.or.cz/jgit.git?a=commit;h=bfa3da225f198b19061158499b1135aff07d85b3 [2] https://eclipse.gerrithub.io/c/eclipse-jgit/jgit/+/1195180 Also-By: Marcin Czech Change-Id: I361ed6286b98351a315b8a8ffc3cb845831d35b2 --- org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java index 3eb35be79a..c27047f25d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java @@ -14,7 +14,6 @@ package org.eclipse.jgit.lib; import static java.nio.charset.StandardCharsets.UTF_8; -import java.nio.ByteBuffer; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.text.MessageFormat; @@ -734,17 +733,7 @@ public final class Constants { * default character encoding (UTF-8). */ public static byte[] encode(String str) { - final ByteBuffer bb = UTF_8.encode(str); - final int len = bb.limit(); - if (bb.hasArray() && bb.arrayOffset() == 0) { - final byte[] arr = bb.array(); - if (arr.length == len) - return arr; - } - - final byte[] arr = new byte[len]; - bb.get(arr); - return arr; + return str.getBytes(UTF_8); } static { -- cgit v1.2.3 From 0fd76114e3436ac635641d06371fd8833179312d Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 6 Sep 2024 13:42:27 +0200 Subject: Replace custom encoder Constants#encodeASCII by JDK implementation Ensure that the method still throws an IllegalArgumentException for malformed input or if the String contains unmappable characters. Change-Id: I6a340aa1af60c315272ff13b6bf2041ba30c94ca --- .../src/org/eclipse/jgit/lib/Constants.java | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java index c27047f25d..cb4a5b51fe 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java @@ -12,8 +12,13 @@ package org.eclipse.jgit.lib; +import static java.nio.charset.StandardCharsets.US_ASCII; import static java.nio.charset.StandardCharsets.UTF_8; +import java.nio.CharBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.CharsetEncoder; +import java.nio.charset.CodingErrorAction; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.text.MessageFormat; @@ -714,14 +719,15 @@ public final class Constants { * the 7-bit ASCII character space. */ public static byte[] encodeASCII(String s) { - final byte[] r = new byte[s.length()]; - for (int k = r.length - 1; k >= 0; k--) { - final char c = s.charAt(k); - if (c > 127) - throw new IllegalArgumentException(MessageFormat.format(JGitText.get().notASCIIString, s)); - r[k] = (byte) c; + try { + CharsetEncoder encoder = US_ASCII.newEncoder() + .onUnmappableCharacter(CodingErrorAction.REPORT) + .onMalformedInput(CodingErrorAction.REPORT); + return encoder.encode(CharBuffer.wrap(s)).array(); + } catch (CharacterCodingException e) { + throw new IllegalArgumentException( + MessageFormat.format(JGitText.get().notASCIIString, s), e); } - return r; } /** -- cgit v1.2.3 From 62d0b220c87d6b51de620418de0a86be7af274cb Mon Sep 17 00:00:00 2001 From: "jackdt@google.com" Date: Mon, 9 Sep 2024 14:05:52 -0700 Subject: PackReverseIndex: Fix javadoc in position methods Position in the API refers to reverse-index position (offset order), not primary index position (sha1 order) Change-Id: I7dbe314dac1b3a128dd1c4ed93b9be28fd0802cc --- .../org/eclipse/jgit/internal/storage/file/PackReverseIndex.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackReverseIndex.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackReverseIndex.java index ef9753cd79..720a3bcbff 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackReverseIndex.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackReverseIndex.java @@ -75,20 +75,20 @@ public interface PackReverseIndex { throws CorruptObjectException; /** - * Find the position in the primary index of the object at the given pack + * Find the position in the reverse index of the object at the given pack * offset. * * @param offset * the pack offset of the object - * @return the position in the primary index of the object + * @return the position in the reverse index of the object */ int findPosition(long offset); /** - * Find the object that is in the given position in the primary index. + * Find the object that is in the given position in the reverse index. * * @param nthPosition - * the position of the object in the primary index + * the position of the object in the reverse index * @return the object in that position */ ObjectId findObjectByPosition(int nthPosition); -- cgit v1.2.3 From 77e7479a36e5e9742043d0c54a02119a1097f623 Mon Sep 17 00:00:00 2001 From: "jackdt@google.com" Date: Wed, 4 Sep 2024 16:07:23 -0700 Subject: PackIndex: Simplify Iterator/MutableEntry interaction The iterator keeps the current position in the index and the MutableEntry reads data from there on-demand, but the iterator needs to know about the entry and this creates a complicated interaction. Make MutableEntry a simple data object and let the iterator iterate and populate it before returning it. Code is clearer and implementors only needs to worry about the iterator. This fixes also MutableEntry visibility, that was preventing subclassing from out of the package. Change-Id: I35010d1f80237e421dd51b8d3d61a8ecb03e0d01 --- .../jgit/internal/storage/file/PackIndex.java | 38 ++++++++++++------- .../jgit/internal/storage/file/PackIndexV1.java | 42 +++++++++------------ .../jgit/internal/storage/file/PackIndexV2.java | 43 +++++++++------------- 3 files changed, 60 insertions(+), 63 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java index c0540d5a4c..95050d472e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java @@ -302,8 +302,9 @@ public interface PackIndex * */ class MutableEntry { + /** Buffer of the ObjectId visited by the EntriesIterator. */ final MutableObjectId idBuffer = new MutableObjectId(); - + /** Offset into the packfile of the current object. */ long offset; /** @@ -321,7 +322,6 @@ public interface PackIndex * @return hex string describing the object id of this entry. */ public String name() { - ensureId(); return idBuffer.name(); } @@ -331,7 +331,6 @@ public interface PackIndex * @return a copy of the object id. */ public ObjectId toObjectId() { - ensureId(); return idBuffer.toObjectId(); } @@ -342,32 +341,32 @@ public interface PackIndex */ public MutableEntry cloneEntry() { final MutableEntry r = new MutableEntry(); - ensureId(); r.idBuffer.fromObjectId(idBuffer); r.offset = offset; return r; } - - void ensureId() { - // Override in implementations. - } } /** * Base implementation of the iterator over index entries. */ abstract class EntriesIterator implements Iterator { - protected final MutableEntry entry = initEntry(); - private final long objectCount; + private final MutableEntry entry = new MutableEntry(); + + /** Counts number of entries accessed so far. */ + private long returnedNumber = 0; + + /** + * Default constructor. + * + * @param objectCount the number of objects in the PackFile. + */ protected EntriesIterator(long objectCount) { this.objectCount = objectCount; } - protected long returnedNumber = 0; - - protected abstract MutableEntry initEntry(); @Override public boolean hasNext() { @@ -379,7 +378,18 @@ public interface PackIndex * element. */ @Override - public abstract MutableEntry next(); + public MutableEntry next() { + readNext(entry); + returnedNumber++; + return entry; + } + + /** + * Used by subclasses to load the next entry into the MutableEntry. + * + * @param entry the container of the next Iterator entry. + */ + protected abstract void readNext(MutableEntry entry); @Override public void remove() { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV1.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV1.java index d7c83785d8..99f3315811 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV1.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV1.java @@ -203,7 +203,7 @@ class PackIndexV1 implements PackIndex { @Override public Iterator iterator() { - return new IndexV1Iterator(objectCnt); + return new EntriesIteratorV1(this); } @Override @@ -246,36 +246,30 @@ class PackIndexV1 implements PackIndex { return packChecksum; } - private class IndexV1Iterator extends EntriesIterator { - int levelOne; + private static class EntriesIteratorV1 extends EntriesIterator { + private int levelOne; - int levelTwo; + private int levelTwo; - IndexV1Iterator(long objectCount) { - super(objectCount); - } + private final PackIndexV1 packIndex; - @Override - protected MutableEntry initEntry() { - return new MutableEntry() { - @Override - protected void ensureId() { - idBuffer.fromRaw(idxdata[levelOne], levelTwo - - Constants.OBJECT_ID_LENGTH); - } - }; + private EntriesIteratorV1(PackIndexV1 packIndex) { + super(packIndex.objectCnt); + this.packIndex = packIndex; } @Override - public MutableEntry next() { - for (; levelOne < idxdata.length; levelOne++) { - if (idxdata[levelOne] == null) + protected void readNext(MutableEntry entry) { + for (; levelOne < packIndex.idxdata.length; levelOne++) { + if (packIndex.idxdata[levelOne] == null) continue; - if (levelTwo < idxdata[levelOne].length) { - entry.offset = NB.decodeUInt32(idxdata[levelOne], levelTwo); - levelTwo += Constants.OBJECT_ID_LENGTH + 4; - returnedNumber++; - return entry; + if (levelTwo < packIndex.idxdata[levelOne].length) { + entry.offset = NB.decodeUInt32(packIndex.idxdata[levelOne], + levelTwo); + this.levelTwo += Constants.OBJECT_ID_LENGTH + 4; + entry.idBuffer.fromRaw(packIndex.idxdata[levelOne], + levelTwo - Constants.OBJECT_ID_LENGTH); + return; } levelTwo = 0; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV2.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV2.java index caf8b71180..f23380dd6b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV2.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV2.java @@ -224,7 +224,7 @@ class PackIndexV2 implements PackIndex { @Override public Iterator iterator() { - return new EntriesIteratorV2(objectCnt); + return new EntriesIteratorV2(this); } @Override @@ -289,41 +289,34 @@ class PackIndexV2 implements PackIndex { return packChecksum; } - private class EntriesIteratorV2 extends EntriesIterator { - int levelOne; + private static class EntriesIteratorV2 extends EntriesIterator { + private int levelOne = 0; - int levelTwo; + private int levelTwo = 0; - EntriesIteratorV2(long objectCount){ - super(objectCount); - } + private final PackIndexV2 packIndex; - @Override - protected MutableEntry initEntry() { - return new MutableEntry() { - @Override - protected void ensureId() { - idBuffer.fromRaw(names[levelOne], levelTwo - - Constants.OBJECT_ID_LENGTH / 4); - } - }; + private EntriesIteratorV2(PackIndexV2 packIndex) { + super(packIndex.objectCnt); + this.packIndex = packIndex; } @Override - public MutableEntry next() { - for (; levelOne < names.length; levelOne++) { - if (levelTwo < names[levelOne].length) { + protected void readNext(MutableEntry entry) { + for (; levelOne < packIndex.names.length; levelOne++) { + if (levelTwo < packIndex.names[levelOne].length) { int idx = levelTwo / (Constants.OBJECT_ID_LENGTH / 4) * 4; - long offset = NB.decodeUInt32(offset32[levelOne], idx); + long offset = NB.decodeUInt32(packIndex.offset32[levelOne], + idx); if ((offset & IS_O64) != 0) { idx = (8 * (int) (offset & ~IS_O64)); - offset = NB.decodeUInt64(offset64, idx); + offset = NB.decodeUInt64(packIndex.offset64, idx); } entry.offset = offset; - - levelTwo += Constants.OBJECT_ID_LENGTH / 4; - returnedNumber++; - return entry; + this.levelTwo += Constants.OBJECT_ID_LENGTH / 4; + entry.idBuffer.fromRaw(packIndex.names[levelOne], + levelTwo - Constants.OBJECT_ID_LENGTH / 4); + return; } levelTwo = 0; } -- cgit v1.2.3 From c3c78ddd9243aca1d03575c37672b767daaa503c Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Mon, 16 Sep 2024 12:44:00 -0700 Subject: PackWriter: Remove constructor with only Reader as param At construction time, PackWriter receives the PackConfig as a parameter or reads it from the repository. The only exception is when the constructor receives only a reader (no repo nor conf provided?!). Remove PackWriter(Reader) and let callers be explicit what conf to use. This makes clearer the flow of conf in the PackWriter. Change-Id: If12e122797ffc8c44fc3c435ca1b000ca016645b --- .../src/org/eclipse/jgit/pgm/debug/ShowPackDelta.java | 3 ++- .../eclipse/jgit/internal/storage/dfs/DfsPackFileTest.java | 3 ++- .../org/eclipse/jgit/internal/storage/pack/PackWriter.java | 13 ------------- 3 files changed, 4 insertions(+), 15 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/ShowPackDelta.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/ShowPackDelta.java index c95f1384e8..74e322ff7f 100644 --- a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/ShowPackDelta.java +++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/ShowPackDelta.java @@ -31,6 +31,7 @@ import org.eclipse.jgit.pgm.Command; import org.eclipse.jgit.pgm.TextBuiltin; import org.eclipse.jgit.revwalk.RevObject; import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.storage.pack.PackConfig; import org.eclipse.jgit.util.TemporaryBuffer; import org.kohsuke.args4j.Argument; @@ -68,7 +69,7 @@ class ShowPackDelta extends TextBuiltin { ObjectReuseAsIs asis = (ObjectReuseAsIs) reader; ObjectToPack target = asis.newObjectToPack(obj, obj.getType()); - PackWriter pw = new PackWriter(reader) { + PackWriter pw = new PackWriter(new PackConfig(), reader) { @Override public void select(ObjectToPack otp, StoredObjectRepresentation next) { otp.select(next); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackFileTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackFileTest.java index bc851f8dde..9680019f88 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackFileTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackFileTest.java @@ -41,6 +41,7 @@ import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.NullProgressMonitor; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.storage.pack.PackConfig; import org.eclipse.jgit.transport.ReceiveCommand; import org.junit.Before; import org.junit.Test; @@ -309,7 +310,7 @@ public class DfsPackFileTest { private void assertPackSize() throws IOException { try (DfsReader ctx = db.getObjectDatabase().newReader(); - PackWriter pw = new PackWriter(ctx); + PackWriter pw = new PackWriter(new PackConfig(), ctx); ByteArrayOutputStream os = new ByteArrayOutputStream(); PackOutputStream out = new PackOutputStream( NullProgressMonitor.INSTANCE, os, pw)) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java index 4350f97915..d3e30f3f6c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java @@ -302,19 +302,6 @@ public class PackWriter implements AutoCloseable { this(repo, repo.newObjectReader()); } - /** - * Create a writer to load objects from the specified reader. - *

- * Objects for packing are specified in {@link #preparePack(Iterator)} or - * {@link #preparePack(ProgressMonitor, Set, Set)}. - * - * @param reader - * reader to read from the repository with. - */ - public PackWriter(ObjectReader reader) { - this(new PackConfig(), reader); - } - /** * Create writer for specified repository. *

-- cgit v1.2.3 From 1af7aa891910131a6a6012ef156510609417bd5e Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Tue, 17 Sep 2024 13:50:59 -0700 Subject: DfsReader: Fallback to regular size read if size index throws The reader can get IOException when reading the object size index, but that index is an optimization, the size should still be available in the pack. Use the regular #getObjectSize() as a fallback when we get an IOException from the object size index. Change-Id: Ic5ec2cfc7c698aa94c6cfd5960cbab6c129f595a --- .../jgit/internal/storage/dfs/DfsReader.java | 61 +++++++++++++++++----- 1 file changed, 48 insertions(+), 13 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java index 9cfcbaa5f7..62f6753e5d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java @@ -511,18 +511,15 @@ public class DfsReader extends ObjectReader implements ObjectReuseAsIs { throw new MissingObjectException(objectId.copy(), typeHint); } - if (typeHint != Constants.OBJ_BLOB || !pack.hasObjectSizeIndex(this)) { + if (typeHint != Constants.OBJ_BLOB || !safeHasObjectSizeIndex(pack)) { return pack.getObjectSize(this, objectId); } - long sz = pack.getIndexedObjectSize(this, objectId); + Optional maybeSz = safeGetIndexedObjectSize(pack, objectId); + long sz = maybeSz.orElse(-1L); if (sz >= 0) { - stats.objectSizeIndexHit += 1; return sz; } - - // Object wasn't in the index - stats.objectSizeIndexMiss += 1; return pack.getObjectSize(this, objectId); } @@ -541,23 +538,61 @@ public class DfsReader extends ObjectReader implements ObjectReuseAsIs { } stats.isNotLargerThanCallCount += 1; - if (typeHint != Constants.OBJ_BLOB || !pack.hasObjectSizeIndex(this)) { + if (typeHint != Constants.OBJ_BLOB || !safeHasObjectSizeIndex(pack)) { + return pack.getObjectSize(this, objectId) <= limit; + } + + Optional maybeSz = safeGetIndexedObjectSize(pack, objectId); + if (maybeSz.isEmpty()) { + // Exception in object size index return pack.getObjectSize(this, objectId) <= limit; } - long sz = pack.getIndexedObjectSize(this, objectId); + long sz = maybeSz.get(); + if (sz >= 0) { + return sz <= limit; + } + + if (isLimitInsideIndexThreshold(pack, limit)) { + // With threshold T, not-found means object < T + // If limit L > T, then object < T < L + return true; + } + + return pack.getObjectSize(this, objectId) <= limit; + } + + private boolean safeHasObjectSizeIndex(DfsPackFile pack) { + try { + return pack.hasObjectSizeIndex(this); + } catch (IOException e) { + return false; + } + } + + private Optional safeGetIndexedObjectSize(DfsPackFile pack, + AnyObjectId objectId) { + long sz; + try { + sz = pack.getIndexedObjectSize(this, objectId); + } catch (IOException e) { + // Do not count the exception as an index miss + return Optional.empty(); + } if (sz < 0) { stats.objectSizeIndexMiss += 1; } else { stats.objectSizeIndexHit += 1; } + return Optional.of(sz); + } - // Got size from index or we didn't but we are sure it should be there. - if (sz >= 0 || pack.getObjectSizeIndexThreshold(this) <= limit) { - return sz <= limit; + private boolean isLimitInsideIndexThreshold(DfsPackFile pack, long limit) { + try { + return pack.getObjectSizeIndexThreshold(this) <= limit; + } catch (IOException e) { + return false; } - - return pack.getObjectSize(this, objectId) <= limit; } private DfsPackFile findPackWithObject(AnyObjectId objectId) -- cgit v1.2.3 From e7d569c8c7851a6e7013d6eb4d57ba6d90c101a4 Mon Sep 17 00:00:00 2001 From: "jackdt@google.com" Date: Thu, 19 Sep 2024 13:28:27 -0700 Subject: PackIndex: Add protected setters to populate the MutableEntry Implementations of the iterator out of package receive a MutableEntry but they don't have a way to set data into it. Add setters to the MutableEntry via protected methods in the iterator. This way, only implementors of the Iterator can modify the entry (cannot be modified e.g. by callers). Change-Id: Id50c69d8be230ebdb8108acc47df13abcad0af0a --- .../jgit/internal/storage/file/PackIndex.java | 54 ++++++++++++++++++---- .../jgit/internal/storage/file/PackIndexV1.java | 8 ++-- .../jgit/internal/storage/file/PackIndexV2.java | 6 +-- 3 files changed, 53 insertions(+), 15 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java index 95050d472e..dfea5c1c80 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java @@ -359,10 +359,11 @@ public interface PackIndex private long returnedNumber = 0; /** - * Default constructor. - * - * @param objectCount the number of objects in the PackFile. - */ + * Construct an iterator that can move objectCount times forward. + * + * @param objectCount + * the number of objects in the PackFile. + */ protected EntriesIterator(long objectCount) { this.objectCount = objectCount; } @@ -379,17 +380,54 @@ public interface PackIndex */ @Override public MutableEntry next() { - readNext(entry); + readNext(); returnedNumber++; return entry; } /** * Used by subclasses to load the next entry into the MutableEntry. + *

+ * Subclasses are expected to populate the entry with + * {@link #setIdBuffer} and {@link #setOffset}. + */ + protected abstract void readNext(); + + + /** + * Copies to the entry an {@link ObjectId} from the int buffer and + * position idx + * + * @param raw + * the raw data + * @param idx + * the index into {@code raw} + */ + protected void setIdBuffer(int[] raw, int idx) { + entry.idBuffer.fromRaw(raw, idx); + } + + /** + * Copies to the entry an {@link ObjectId} from the byte array at + * position idx. * - * @param entry the container of the next Iterator entry. - */ - protected abstract void readNext(MutableEntry entry); + * @param raw + * the raw data + * @param idx + * the index into {@code raw} + */ + protected void setIdBuffer(byte[] raw, int idx) { + entry.idBuffer.fromRaw(raw, idx); + } + + /** + * Sets the {@code offset} to the entry + * + * @param offset the offset in the pack file + */ + protected void setOffset(long offset) { + entry.offset = offset; + } @Override public void remove() { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV1.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV1.java index 99f3315811..be48358a0d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV1.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV1.java @@ -259,15 +259,15 @@ class PackIndexV1 implements PackIndex { } @Override - protected void readNext(MutableEntry entry) { + protected void readNext() { for (; levelOne < packIndex.idxdata.length; levelOne++) { if (packIndex.idxdata[levelOne] == null) continue; if (levelTwo < packIndex.idxdata[levelOne].length) { - entry.offset = NB.decodeUInt32(packIndex.idxdata[levelOne], - levelTwo); + super.setOffset(NB.decodeUInt32(packIndex.idxdata[levelOne], + levelTwo)); this.levelTwo += Constants.OBJECT_ID_LENGTH + 4; - entry.idBuffer.fromRaw(packIndex.idxdata[levelOne], + super.setIdBuffer(packIndex.idxdata[levelOne], levelTwo - Constants.OBJECT_ID_LENGTH); return; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV2.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV2.java index f23380dd6b..36e54fcd62 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV2.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexV2.java @@ -302,7 +302,7 @@ class PackIndexV2 implements PackIndex { } @Override - protected void readNext(MutableEntry entry) { + protected void readNext() { for (; levelOne < packIndex.names.length; levelOne++) { if (levelTwo < packIndex.names[levelOne].length) { int idx = levelTwo / (Constants.OBJECT_ID_LENGTH / 4) * 4; @@ -312,9 +312,9 @@ class PackIndexV2 implements PackIndex { idx = (8 * (int) (offset & ~IS_O64)); offset = NB.decodeUInt64(packIndex.offset64, idx); } - entry.offset = offset; + super.setOffset(offset); this.levelTwo += Constants.OBJECT_ID_LENGTH / 4; - entry.idBuffer.fromRaw(packIndex.names[levelOne], + super.setIdBuffer(packIndex.names[levelOne], levelTwo - Constants.OBJECT_ID_LENGTH / 4); return; } -- cgit v1.2.3 From 1332b5156a2d8413d778a630f16cd1e030ac89ef Mon Sep 17 00:00:00 2001 From: "jackdt@google.com" Date: Mon, 23 Sep 2024 13:19:48 -0700 Subject: Record failing paths in recursive merge. This is meant to help diagnose LOCK_FAILURE errors, which otherwise provides very little information in https://eclipse.googlesource.com/jgit/jgit/+/refs/heads/master/org.eclipse.jgit/src/org/eclipse/jgit/lib/BatchRefUpdate.java#731. Change-Id: I3d544c899fe66effbd107ea2f38d73f6f253a7e6 --- .../org/eclipse/jgit/internal/JGitText.properties | 2 +- .../src/org/eclipse/jgit/merge/RecursiveMerger.java | 21 +++++++++++++++++++-- 2 files changed, 20 insertions(+), 3 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties index 11435b8ce4..b1450ef057 100644 --- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties +++ b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties @@ -502,7 +502,7 @@ mergeConflictOnNotes=Merge conflict on note {0}. base = {1}, ours = {2}, theirs mergeStrategyAlreadyExistsAsDefault=Merge strategy "{0}" already exists as a default strategy mergeStrategyDoesNotSupportHeads=merge strategy {0} does not support {1} heads to be merged into HEAD mergeUsingStrategyResultedInDescription=Merge of revisions {0} with base {1} using strategy {2} resulted in: {3}. {4} -mergeRecursiveConflictsWhenMergingCommonAncestors=Multiple common ancestors were found and merging them resulted in a conflict: {0}, {1} +mergeRecursiveConflictsWhenMergingCommonAncestors=Multiple common ancestors were found and merging them resulted in a conflict: {0}, {1}\nFailing paths: {2} mergeRecursiveTooManyMergeBasesFor = "More than {0} merge bases for:\n a {1}\n b {2} found:\n count {3}" mergeToolNotGivenError=No merge tool provided and no defaults configured. mergeToolNullError=Parameter for merge tool cannot be null. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/merge/RecursiveMerger.java b/org.eclipse.jgit/src/org/eclipse/jgit/merge/RecursiveMerger.java index 1162a615f2..24614f7d18 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/merge/RecursiveMerger.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/merge/RecursiveMerger.java @@ -22,6 +22,7 @@ import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.TimeZone; +import java.util.stream.Collectors; import org.eclipse.jgit.dircache.DirCache; import org.eclipse.jgit.errors.IncorrectObjectTypeException; @@ -185,12 +186,15 @@ public class RecursiveMerger extends ResolveMerger { if (mergeTrees(bcTree, currentBase.getTree(), nextBase.getTree(), true)) currentBase = createCommitForTree(resultTree, parents); - else + else { + String failedPaths = failingPathsMessage(); throw new NoMergeBaseException( NoMergeBaseException.MergeBaseFailureReason.CONFLICTS_DURING_MERGE_BASE_CALCULATION, MessageFormat.format( JGitText.get().mergeRecursiveConflictsWhenMergingCommonAncestors, - currentBase.getName(), nextBase.getName())); + currentBase.getName(), nextBase.getName(), + failedPaths)); + } } } finally { inCore = oldIncore; @@ -236,4 +240,17 @@ public class RecursiveMerger extends ResolveMerger { new Date((time + 1) * 1000L), TimeZone.getTimeZone("GMT+0000")); //$NON-NLS-1$ } + + private String failingPathsMessage() { + int max = 25; + String failedPaths = failingPaths.entrySet().stream().limit(max) + .map(entry -> entry.getKey() + ":" + entry.getValue()) + .collect(Collectors.joining("\n")); + + if (failingPaths.size() > max) { + failedPaths = String.format("%s\n... (%s failing paths omitted)", + failedPaths, failingPaths.size() - max); + } + return failedPaths; + } } -- cgit v1.2.3 From c5407a2b5967e19859d9028db2ad72c54a6f047c Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Tue, 24 Sep 2024 15:23:04 -0700 Subject: DfsInserter: Create PackConfig from repo instead of repo.conf PackConfig can be constructed from the repo or from a config. While browing the code, it is easier to follow the provenance of the configuration when using the repository constructor. Use the PackConfig(Repository) constructor in the DfsInserter. Internally it calls PackConfig(repo.getConfig()) so this should be a noop. Change-Id: Ifce5bc87404ca8ec9a821d28253d489056faad9a --- .../src/org/eclipse/jgit/internal/storage/dfs/DfsInserter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsInserter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsInserter.java index a07d8416c4..86fbd445be 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsInserter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsInserter.java @@ -93,7 +93,7 @@ public class DfsInserter extends ObjectInserter { */ protected DfsInserter(DfsObjDatabase db) { this.db = db; - PackConfig pc = new PackConfig(db.getRepository().getConfig()); + PackConfig pc = new PackConfig(db.getRepository()); this.minBytesForObjectSizeIndex = pc.getMinBytesForObjSizeIndex(); } -- cgit v1.2.3 From 56a5db65b3e6c0c838b29071323332439dbbc08f Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Tue, 24 Sep 2024 10:51:22 +0200 Subject: AdvertisedRequestValidator: fix WantNotValidException caused by race Fetch with protocol V2 failed under the following conditions - fetch uses bidirectional protocol (git, ssh) which uses a shortcut to determine invalid wants - not all wants are advertised - race condition: wanted ref is updated during fetch by another thread after the thread serving upload-pack determined wants and before it checks not advertised wants Fix this by calling `new ReachableCommitRequestValidator().checkWants(up, wants)` instead of throwing WantNotValidException in [1] if this race happened in the same way like it's done for unidirectional protocols (http) [2]. [1] https://github.com/eclipse-jgit/jgit/blob/stable-6.10/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java#L2002 [2] https://github.com/eclipse-jgit/jgit/blob/stable-6.10/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java#L2000 Bug: jgit-48 Change-Id: I32f28502923815dc49781aab5d810c9afbe7e7e6 --- org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java index 3264f556fa..8b97ee10ca 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java @@ -1968,10 +1968,9 @@ public class UploadPack implements Closeable { @Override public void checkWants(UploadPack up, List wants) throws PackProtocolException, IOException { - if (!up.isBiDirectionalPipe()) + if (!up.isBiDirectionalPipe() || !wants.isEmpty()) { new ReachableCommitRequestValidator().checkWants(up, wants); - else if (!wants.isEmpty()) - throw new WantNotValidException(wants.iterator().next()); + } } } -- cgit v1.2.3 From 4ca0d13c272366cbaa503273ab29fa28b11bcdaa Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Tue, 24 Sep 2024 16:32:31 -0700 Subject: FileRepository: Remove unnecessary setConfig call The constructor of GC sets exactly the same config. Remove this set to make clearer from where the config is coming. Change-Id: Idb71c7827f180923092ef5392545df81960ee93a --- .../src/org/eclipse/jgit/internal/storage/file/FileRepository.java | 2 -- 1 file changed, 2 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java index b5d29a3fc8..230dc6f45a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java @@ -60,7 +60,6 @@ import org.eclipse.jgit.lib.StoredConfig; import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.storage.file.FileBasedConfig; import org.eclipse.jgit.storage.file.FileRepositoryBuilder; -import org.eclipse.jgit.storage.pack.PackConfig; import org.eclipse.jgit.transport.ReceiveCommand; import org.eclipse.jgit.util.FileUtils; import org.eclipse.jgit.util.IO; @@ -595,7 +594,6 @@ public class FileRepository extends Repository { @Override public void autoGC(ProgressMonitor monitor) { GC gc = new GC(this); - gc.setPackConfig(new PackConfig(this)); gc.setProgressMonitor(monitor); gc.setAuto(true); gc.setBackground(shouldAutoDetach()); -- cgit v1.2.3 From 17882b1cd8e66a174468633b1b8eb695c86011cf Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Mon, 16 Sep 2024 08:30:39 +0200 Subject: NoteMapMerger: remove unnecessary cast Change-Id: I3be4963f506529bbadc5b6dfc0b625ee85effc1f --- org.eclipse.jgit/src/org/eclipse/jgit/notes/NoteMapMerger.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/notes/NoteMapMerger.java b/org.eclipse.jgit/src/org/eclipse/jgit/notes/NoteMapMerger.java index 79ceb1316a..30512c17b8 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/notes/NoteMapMerger.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/notes/NoteMapMerger.java @@ -199,7 +199,7 @@ public class NoteMapMerger { if (child == null) return; if (child instanceof InMemoryNoteBucket) - b.setBucket(cell, ((InMemoryNoteBucket) child).writeTree(inserter)); + b.setBucket(cell, child.writeTree(inserter)); else b.setBucket(cell, child.getTreeId()); } -- cgit v1.2.3 From 7e85e7e21e29769c596694ea8492ffbc6dc3d9af Mon Sep 17 00:00:00 2001 From: Laura Hamelin Date: Mon, 10 Jun 2024 13:42:03 -0700 Subject: DfsBlockCache: use PackExtBlockCacheTable when configured Adds the usage of PackExtBlockCacheTable to the DfsBlockCache, replacing the current DfsBlockCacheTable when PackExtCacheConfigurations exists. When no PackExtCacheConfigurations exists the current DfsBlockCacheTable implementation will be used. Change-Id: I42222a0cb43785baba907a49077dd9874d19d891 --- .../internal/storage/dfs/DfsBlockCacheTest.java | 53 ++++++++++++++++++++-- .../jgit/internal/storage/dfs/DfsBlockCache.java | 10 ++-- 2 files changed, 56 insertions(+), 7 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheTest.java index fef0563f48..3c7cc075d2 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheTest.java @@ -13,20 +13,24 @@ package org.eclipse.jgit.internal.storage.dfs; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static org.eclipse.jgit.lib.Constants.OBJ_BLOB; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import java.time.Duration; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.EnumSet; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.LongStream; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.LongStream; +import org.eclipse.jgit.internal.storage.dfs.DfsBlockCacheConfig.DfsBlockCachePackExtConfig; import org.eclipse.jgit.internal.storage.dfs.DfsBlockCacheConfig.IndexEventConsumer; import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.junit.TestRepository; @@ -39,14 +43,35 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TestName; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameter; +import org.junit.runners.Parameterized.Parameters; +@RunWith(Parameterized.class) public class DfsBlockCacheTest { @Rule public TestName testName = new TestName(); + private TestRng rng; + private DfsBlockCache cache; + private ExecutorService pool; + private enum CacheType { + SINGLE_TABLE_CLOCK_BLOCK_CACHE, EXT_SPLIT_TABLE_CLOCK_BLOCK_CACHE + } + + @Parameters(name = "cache type: {0}") + public static Iterable data() { + return Arrays.asList(CacheType.SINGLE_TABLE_CLOCK_BLOCK_CACHE, + CacheType.EXT_SPLIT_TABLE_CLOCK_BLOCK_CACHE); + } + + @Parameter + public CacheType cacheType; + @Before public void setUp() { rng = new TestRng(testName.getMethodName()); @@ -448,8 +473,28 @@ public class DfsBlockCacheTest { } private void resetCache(int concurrencyLevel) { - DfsBlockCache.reconfigure(new DfsBlockCacheConfig().setBlockSize(512) - .setConcurrencyLevel(concurrencyLevel).setBlockLimit(1 << 20)); + DfsBlockCacheConfig cacheConfig = new DfsBlockCacheConfig() + .setBlockSize(512).setConcurrencyLevel(concurrencyLevel) + .setBlockLimit(1 << 20); + switch (cacheType) { + case SINGLE_TABLE_CLOCK_BLOCK_CACHE: + // SINGLE_TABLE_CLOCK_BLOCK_CACHE doesn't modify the config. + break; + case EXT_SPLIT_TABLE_CLOCK_BLOCK_CACHE: + List packExtCacheConfigs = new ArrayList<>(); + for (PackExt packExt : PackExt.values()) { + DfsBlockCacheConfig extCacheConfig = new DfsBlockCacheConfig() + .setBlockSize(512).setConcurrencyLevel(concurrencyLevel) + .setBlockLimit(1 << 20) + .setPackExtCacheConfigurations(packExtCacheConfigs); + packExtCacheConfigs.add(new DfsBlockCachePackExtConfig( + EnumSet.of(packExt), extCacheConfig)); + } + cacheConfig.setPackExtCacheConfigurations(packExtCacheConfigs); + break; + } + assertNotNull(cacheConfig); + DfsBlockCache.reconfigure(cacheConfig); cache = DfsBlockCache.getInstance(); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCache.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCache.java index 3e1300c867..0334450fbe 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCache.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCache.java @@ -97,7 +97,12 @@ public final class DfsBlockCache { double streamRatio = cfg.getStreamRatio(); maxStreamThroughCache = (long) (maxBytes * streamRatio); - dfsBlockCacheTable = new ClockBlockCacheTable(cfg); + if (!cfg.getPackExtCacheConfigurations().isEmpty()) { + dfsBlockCacheTable = PackExtBlockCacheTable + .fromBlockCacheConfigs(cfg); + } else { + dfsBlockCacheTable = new ClockBlockCacheTable(cfg); + } for (int i = 0; i < PackExt.values().length; ++i) { Integer limit = cfg.getCacheHotMap().get(PackExt.values()[i]); @@ -158,8 +163,7 @@ public final class DfsBlockCache { * @return total number of requests (hit + miss), per pack file extension. */ public long[] getTotalRequestCount() { - return dfsBlockCacheTable.getBlockCacheStats() - .getTotalRequestCount(); + return dfsBlockCacheTable.getBlockCacheStats().getTotalRequestCount(); } /** -- cgit v1.2.3 From dd8c3dab8af85a4c367949c06c648d1e6c9a9f4a Mon Sep 17 00:00:00 2001 From: Laura Hamelin Date: Wed, 24 Jul 2024 13:56:28 -0700 Subject: dfs: add configurable name to block cache table stats The addition of a name will help show statistics broken down per inner cache table when more than one cache table is used. The name configuration is obtained from the config subsection name prefixed by `dfs`, or `dfs` for the base case. Change-Id: Ia16c794f094d756441b779e3b1f1a3c992443509 --- .../storage/dfs/ClockBlockCacheTableTest.java | 49 ++++++++++++++++++++++ .../storage/dfs/DfsBlockCacheConfigTest.java | 34 +++++++++++++++ .../storage/dfs/PackExtBlockCacheTableTest.java | 32 ++++++++++++++ .../internal/storage/dfs/ClockBlockCacheTable.java | 13 +++++- .../internal/storage/dfs/DfsBlockCacheConfig.java | 33 +++++++++++++++ .../internal/storage/dfs/DfsBlockCacheTable.java | 21 ++++++++++ .../storage/dfs/PackExtBlockCacheTable.java | 37 +++++++++++++--- 7 files changed, 212 insertions(+), 7 deletions(-) create mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/ClockBlockCacheTableTest.java (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/ClockBlockCacheTableTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/ClockBlockCacheTableTest.java new file mode 100644 index 0000000000..cb68bbc515 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/ClockBlockCacheTableTest.java @@ -0,0 +1,49 @@ +package org.eclipse.jgit.internal.storage.dfs; + +import static org.eclipse.jgit.internal.storage.dfs.DfsBlockCacheConfig.DEFAULT_NAME; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; + +import org.junit.Test; + +public class ClockBlockCacheTableTest { + private static final String NAME = "name"; + + @Test + public void getName_nameNotConfigured_returnsDefaultName() { + ClockBlockCacheTable cacheTable = new ClockBlockCacheTable( + createBlockCacheConfig()); + + assertThat(cacheTable.getName(), equalTo(DEFAULT_NAME)); + } + + @Test + public void getName_nameConfigured_returnsConfiguredName() { + ClockBlockCacheTable cacheTable = new ClockBlockCacheTable( + createBlockCacheConfig().setName(NAME)); + + assertThat(cacheTable.getName(), equalTo(NAME)); + } + + @Test + public void getBlockCacheStats_nameNotConfigured_returnsBlockCacheStatsWithDefaultName() { + ClockBlockCacheTable cacheTable = new ClockBlockCacheTable( + createBlockCacheConfig()); + + assertThat(cacheTable.getBlockCacheStats().getName(), + equalTo(DEFAULT_NAME)); + } + + @Test + public void getBlockCacheStats_nameConfigured_returnsBlockCacheStatsWithConfiguredName() { + ClockBlockCacheTable cacheTable = new ClockBlockCacheTable( + createBlockCacheConfig().setName(NAME)); + + assertThat(cacheTable.getBlockCacheStats().getName(), equalTo(NAME)); + } + + private static DfsBlockCacheConfig createBlockCacheConfig() { + return new DfsBlockCacheConfig().setBlockSize(512) + .setConcurrencyLevel(4).setBlockLimit(1024); + } +} \ No newline at end of file diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfigTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfigTest.java index c93f48d79c..4b7aae05f0 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfigTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfigTest.java @@ -38,6 +38,7 @@ package org.eclipse.jgit.internal.storage.dfs; +import static org.eclipse.jgit.internal.storage.dfs.DfsBlockCacheConfig.DEFAULT_NAME; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_CORE_SECTION; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_DFS_CACHE_PREFIX; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_DFS_SECTION; @@ -48,6 +49,7 @@ import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_PACK_EXTENSIONS; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_STREAM_RATIO; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertThrows; @@ -172,6 +174,38 @@ public class DfsBlockCacheConfigTest { is(indexConfig)); } + @Test + public void fromConfigs_baseConfigOnly_nameSetFromConfigDfsSubSection() { + Config config = new Config(); + + DfsBlockCacheConfig blockCacheConfig = new DfsBlockCacheConfig() + .fromConfig(config); + assertThat(blockCacheConfig.getName(), equalTo(DEFAULT_NAME)); + } + + @Test + public void fromConfigs_namesSetFromConfigDfsCachePrefixSubSections() { + Config config = new Config(); + config.setString(CONFIG_CORE_SECTION, CONFIG_DFS_SECTION, + CONFIG_KEY_STREAM_RATIO, "0.5"); + config.setString(CONFIG_CORE_SECTION, CONFIG_DFS_CACHE_PREFIX + "name1", + CONFIG_KEY_PACK_EXTENSIONS, PackExt.PACK.name()); + config.setString(CONFIG_CORE_SECTION, CONFIG_DFS_CACHE_PREFIX + "name2", + CONFIG_KEY_PACK_EXTENSIONS, PackExt.BITMAP_INDEX.name()); + + DfsBlockCacheConfig blockCacheConfig = new DfsBlockCacheConfig() + .fromConfig(config); + assertThat(blockCacheConfig.getName(), equalTo("dfs")); + assertThat( + blockCacheConfig.getPackExtCacheConfigurations().get(0) + .getPackExtCacheConfiguration().getName(), + equalTo("dfs.name1")); + assertThat( + blockCacheConfig.getPackExtCacheConfigurations().get(1) + .getPackExtCacheConfiguration().getName(), + equalTo("dfs.name2")); + } + @Test public void fromConfigs_dfsBlockCachePackExtConfigWithDuplicateExtensions_throws() { Config config = new Config(); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/PackExtBlockCacheTableTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/PackExtBlockCacheTableTest.java index 8c003e0cb3..f2a5abcacd 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/PackExtBlockCacheTableTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/PackExtBlockCacheTableTest.java @@ -11,6 +11,7 @@ package org.eclipse.jgit.internal.storage.dfs; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.sameInstance; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertNotNull; @@ -37,6 +38,8 @@ import org.mockito.Mockito; @SuppressWarnings({ "boxing", "unchecked" }) public class PackExtBlockCacheTableTest { + private static final String CACHE_NAME = "CacheName"; + @Test public void fromBlockCacheConfigs_createsDfsPackExtBlockCacheTables() { DfsBlockCacheConfig cacheConfig = new DfsBlockCacheConfig(); @@ -384,6 +387,29 @@ public class PackExtBlockCacheTableTest { assertThat(tables.get(dfsStreamKey, 0), sameInstance(ref)); } + @Test + public void getName() { + DfsBlockCacheStats packStats = new DfsBlockCacheStats(); + PackExtBlockCacheTable tables = PackExtBlockCacheTable.fromCacheTables( + cacheTableWithStats(/* name= */ "defaultName", packStats), + Map.of(PackExt.PACK, cacheTableWithStats(/* name= */ "packName", + packStats))); + + assertThat(tables.getName(), equalTo("defaultName,packName")); + } + + @Test + public void getBlockCacheStats_getName_returnsPackExtCacheTableName() { + DfsBlockCacheStats packStats = new DfsBlockCacheStats(); + PackExtBlockCacheTable tables = PackExtBlockCacheTable.fromCacheTables( + cacheTableWithStats(/* name= */ "defaultName", packStats), + Map.of(PackExt.PACK, cacheTableWithStats(/* name= */ "packName", + packStats))); + + assertThat(tables.getBlockCacheStats().getName(), + equalTo("defaultName,packName")); + } + @Test public void getBlockCacheStats_getCurrentSize_consolidatesAllTableCurrentSizes() { long[] currentSizes = createEmptyStatsArray(); @@ -573,7 +599,13 @@ public class PackExtBlockCacheTableTest { private static DfsBlockCacheTable cacheTableWithStats( DfsBlockCacheStats dfsBlockCacheStats) { + return cacheTableWithStats(CACHE_NAME, dfsBlockCacheStats); + } + + private static DfsBlockCacheTable cacheTableWithStats(String name, + DfsBlockCacheStats dfsBlockCacheStats) { DfsBlockCacheTable cacheTable = mock(DfsBlockCacheTable.class); + when(cacheTable.getName()).thenReturn(name); when(cacheTable.getBlockCacheStats()).thenReturn(dfsBlockCacheStats); return cacheTable; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/ClockBlockCacheTable.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/ClockBlockCacheTable.java index ce71a71d67..bbee23b061 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/ClockBlockCacheTable.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/ClockBlockCacheTable.java @@ -47,6 +47,11 @@ import org.eclipse.jgit.internal.storage.pack.PackExt; * invocations is also fixed in size. */ final class ClockBlockCacheTable implements DfsBlockCacheTable { + /** + * Table name. + */ + private final String name; + /** Number of entries in {@link #table}. */ private final int tableSize; @@ -129,7 +134,8 @@ final class ClockBlockCacheTable implements DfsBlockCacheTable { -1, 0, null); clockHand.next = clockHand; - this.dfsBlockCacheStats = new DfsBlockCacheStats(); + this.name = cfg.getName(); + this.dfsBlockCacheStats = new DfsBlockCacheStats(this.name); this.refLockWaitTime = cfg.getRefLockWaitTimeConsumer(); this.indexEventConsumer = cfg.getIndexEventConsumer(); } @@ -139,6 +145,11 @@ final class ClockBlockCacheTable implements DfsBlockCacheTable { return dfsBlockCacheStats; } + @Override + public String getName() { + return name; + } + @Override public boolean hasBlock0(DfsStreamKey key) { HashEntry e1 = table.get(slot(key, 0)); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java index fa68b979fb..52eb9e0785 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java @@ -49,6 +49,10 @@ public class DfsBlockCacheConfig { /** Default number of max cache hits. */ public static final int DEFAULT_CACHE_HOT_MAX = 1; + static final String DEFAULT_NAME = ""; + + private String name; + private long blockLimit; private int blockSize; @@ -69,6 +73,7 @@ public class DfsBlockCacheConfig { * Create a default configuration. */ public DfsBlockCacheConfig() { + name = DEFAULT_NAME; setBlockLimit(32 * MB); setBlockSize(64 * KB); setStreamRatio(0.30); @@ -77,6 +82,29 @@ public class DfsBlockCacheConfig { packExtCacheConfigurations = Collections.emptyList(); } + /** + * Get the name for the block cache configured by this cache config. + * + * @return the name for the block cache configured by this cache config. + */ + public String getName() { + return name; + } + + /** + * Set the name for the block cache configured by this cache config. + *

+ * Made visible for testing. + * + * @param name + * the name for the block cache configured by this cache config. + * @return {@code this} + */ + DfsBlockCacheConfig setName(String name) { + this.name = name; + return this; + } + /** * Get maximum number bytes of heap memory to dedicate to caching pack file * data. @@ -308,6 +336,11 @@ public class DfsBlockCacheConfig { Long.valueOf(cfgBlockLimit), Long.valueOf(cfgBlockSize))); } + // Set name only if `core dfs` is configured, otherwise fall back to the + // default. + if (rc.getSubsections(section).contains(subSection)) { + this.name = subSection; + } setBlockLimit(cfgBlockLimit); setBlockSize(cfgBlockSize); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheTable.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheTable.java index 309f2d1595..de5dc23963 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheTable.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheTable.java @@ -137,10 +137,19 @@ public interface DfsBlockCacheTable { */ BlockCacheStats getBlockCacheStats(); + /** + * Get the name of the table. + * + * @return this table's name. + */ + String getName(); + /** * Provides methods used with Block Cache statistics. */ interface BlockCacheStats { + String getName(); + /** * Get total number of bytes in the cache, per pack file extension. * @@ -195,6 +204,8 @@ public interface DfsBlockCacheTable { * Keeps track of stats for a Block Cache table. */ class DfsBlockCacheStats implements BlockCacheStats { + private final String name; + /** * Number of times a block was found in the cache, per pack file * extension. @@ -220,12 +231,22 @@ public interface DfsBlockCacheTable { private final AtomicReference liveBytes; DfsBlockCacheStats() { + this(""); + } + + DfsBlockCacheStats(String name) { + this.name = name; statHit = new AtomicReference<>(newCounters()); statMiss = new AtomicReference<>(newCounters()); statEvict = new AtomicReference<>(newCounters()); liveBytes = new AtomicReference<>(newCounters()); } + @Override + public String getName() { + return name; + } + /** * Increment the {@code statHit} count. * diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/PackExtBlockCacheTable.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/PackExtBlockCacheTable.java index 858f731b70..7f21b3f10a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/PackExtBlockCacheTable.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/PackExtBlockCacheTable.java @@ -37,6 +37,11 @@ import org.eclipse.jgit.internal.storage.pack.PackExt; * type. */ class PackExtBlockCacheTable implements DfsBlockCacheTable { + /** + * Table name. + */ + private final String name; + private final DfsBlockCacheTable defaultBlockCacheTable; // Holds the unique tables backing the extBlockCacheTables values. @@ -120,13 +125,19 @@ class PackExtBlockCacheTable implements DfsBlockCacheTable { Set blockCacheTables = new HashSet<>(); blockCacheTables.add(defaultBlockCacheTable); blockCacheTables.addAll(packExtBlockCacheTables.values()); - return new PackExtBlockCacheTable(defaultBlockCacheTable, + String name = defaultBlockCacheTable.getName() + "," + + packExtBlockCacheTables.values().stream() + .map(DfsBlockCacheTable::getName).sorted() + .collect(Collectors.joining(",")); + return new PackExtBlockCacheTable(name, defaultBlockCacheTable, List.copyOf(blockCacheTables), packExtBlockCacheTables); } - private PackExtBlockCacheTable(DfsBlockCacheTable defaultBlockCacheTable, + private PackExtBlockCacheTable(String name, + DfsBlockCacheTable defaultBlockCacheTable, List blockCacheTableList, Map extBlockCacheTables) { + this.name = name; this.defaultBlockCacheTable = defaultBlockCacheTable; this.blockCacheTableList = blockCacheTableList; this.extBlockCacheTables = extBlockCacheTables; @@ -178,9 +189,15 @@ class PackExtBlockCacheTable implements DfsBlockCacheTable { @Override public BlockCacheStats getBlockCacheStats() { - return new CacheStats(blockCacheTableList.stream() - .map(DfsBlockCacheTable::getBlockCacheStats) - .collect(Collectors.toList())); + return new CacheStats(name, + blockCacheTableList.stream() + .map(DfsBlockCacheTable::getBlockCacheStats) + .collect(Collectors.toList())); + } + + @Override + public String getName() { + return name; } private DfsBlockCacheTable getTable(PackExt packExt) { @@ -198,12 +215,20 @@ class PackExtBlockCacheTable implements DfsBlockCacheTable { } private static class CacheStats implements BlockCacheStats { + private final String name; + private final List blockCacheStats; - private CacheStats(List blockCacheStats) { + private CacheStats(String name, List blockCacheStats) { + this.name = name; this.blockCacheStats = blockCacheStats; } + @Override + public String getName() { + return name; + } + @Override public long[] getCurrentSize() { long[] sums = emptyPackStats(); -- cgit v1.2.3 From 0155f8bf6e569b487251767be985332a438ccc39 Mon Sep 17 00:00:00 2001 From: Kamil Musin Date: Tue, 8 Oct 2024 13:36:00 +0200 Subject: RevolveMerger: honor ignoreConflicts also for binary files Currently difference in binary files during merge will cause them to be added to unmergedPaths regardless of whether ignoreConflicts is true. This creates an issue during merging with strategy "RECURSIVE", as it makes it impossible to create a virtual commit if there is a difference in a binary file. Resulting in the CONFLICTS_DURING_MERGE_BASE_CALCULATION error being thrown. This is especially problematic, since JGit has a rather simplistic rules for considering file binary, which easily leads to false positives. What we should do instead is keep OURS. This will not lead to silently ignoring difference in the final result. It will allow creation of virtual merge-base commit, and then the difference would be presented again in the final merge results. In essense it only affects what's shown as BASE in 3-way merge. Additionally, this is correct because - It's consistent with treatment of other unmergeable entities, for example Gitlinks - It's consistent with behaviour of CGit: - https://git-scm.com/docs/gitattributes#Documentation/gitattributes.txt-binary states on diffs in binary OURS is picked by default. - In code: https://git.kernel.org/pub/scm/git/git.git/tree/merge-ll.c#n81 - ignoreConflicts in CGit afterwards ignores all issues with content merging https://git.kernel.org/pub/scm/git/git.git/tree/merge-ort.c#n5201 We also adjust the behaviour when .gitattributes tell us to treat the file as binary for the purpose of the merge. We only change the behaviour when ignoreConlicts = true, as otherwise the current behaviour works as intended. Change-Id: I2b69f80a13d250aad3fe12dd438b2763f3022270 --- .../tst/org/eclipse/jgit/merge/MergerTest.java | 70 ++++++++++++++++++++++ .../src/org/eclipse/jgit/merge/ResolveMerger.java | 17 +++++- 2 files changed, 85 insertions(+), 2 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/merge/MergerTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/merge/MergerTest.java index 3a036acaca..c6a6321cf8 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/merge/MergerTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/merge/MergerTest.java @@ -1792,7 +1792,77 @@ public class MergerTest extends RepositoryTestCase { // children mergeResult = git.merge().include(commitC3S).call(); assertEquals(mergeResult.getMergeStatus(), MergeStatus.MERGED); + } + + /** + * Merging two commits when binary files have equal content, but conflicting content in the + * virtual ancestor. + * + *

+ * This test has the same set up as + * {@code checkFileDirMergeConflictInVirtualAncestor_NoConflictInChildren}, only + * with the content conflict in A1 and A2. + */ + @Theory + public void checkBinaryMergeConflictInVirtualAncestor(MergeStrategy strategy) throws Exception { + if (!strategy.equals(MergeStrategy.RECURSIVE)) { + return; + } + + Git git = Git.wrap(db); + + // master + writeTrashFile("c", "initial file"); + git.add().addFilepattern("c").call(); + RevCommit commitI = git.commit().setMessage("Initial commit").call(); + + writeTrashFile("a", "\0\1\1\1\1\0"); // content in Ancestor 1 + git.add().addFilepattern("a").call(); + RevCommit commitA1 = git.commit().setMessage("Ancestor 1").call(); + + writeTrashFile("a", "\0\1\2\3\4\5\0"); // content in Child 1 (commited on master) + git.add().addFilepattern("a").call(); + // commit C1M + git.commit().setMessage("Child 1 on master").call(); + + git.checkout().setCreateBranch(true).setStartPoint(commitI).setName("branch-to-merge").call(); + writeTrashFile("a", "\0\2\2\2\2\0"); // content in Ancestor 1 + git.add().addFilepattern("a").call(); + RevCommit commitA2 = git.commit().setMessage("Ancestor 2").call(); + + // second branch + git.checkout().setCreateBranch(true).setStartPoint(commitA1).setName("second-branch").call(); + writeTrashFile("a", "\0\5\4\3\2\1\0"); // content in Child 2 (commited on second-branch) + git.add().addFilepattern("a").call(); + // commit C2S + git.commit().setMessage("Child 2 on second-branch").call(); + // Merge branch-to-merge into second-branch + MergeResult mergeResult = git.merge().include(commitA2).setStrategy(strategy).call(); + assertEquals(mergeResult.getNewHead(), null); + assertEquals(mergeResult.getMergeStatus(), MergeStatus.CONFLICTING); + // Resolve the conflict manually + writeTrashFile("a", "\0\3\3\3\3\0"); // merge conflict resolution + git.add().addFilepattern("a").call(); + RevCommit commitC3S = git.commit().setMessage("Child 3 on second bug - resolve merge conflict").call(); + + // Merge branch-to-merge into master + git.checkout().setName("master").call(); + mergeResult = git.merge().include(commitA2).setStrategy(strategy).call(); + assertEquals(mergeResult.getNewHead(), null); + assertEquals(mergeResult.getMergeStatus(), MergeStatus.CONFLICTING); + + // Resolve the conflict manually - set the same value as in resolution above + writeTrashFile("a", "\0\3\3\3\3\0"); // merge conflict resolution + git.add().addFilepattern("a").call(); + // commit C4M + git.commit().setMessage("Child 4 on master - resolve merge conflict").call(); + + // Merge C4M (second-branch) into master (C3S) + // Conflict in virtual base should be here, but there are no conflicts in + // children + mergeResult = git.merge().include(commitC3S).call(); + assertEquals(mergeResult.getMergeStatus(), MergeStatus.MERGED); } /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java b/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java index 1ad41be423..50c2c1570c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java @@ -1273,6 +1273,13 @@ public class ResolveMerger extends ThreeWayMerger { default: break; } + if (ignoreConflicts) { + // If the path is selected to be treated as binary via attributes, we do not perform + // content merge. When ignoreConflicts = true, we simply keep OURS to allow virtual commit + // to be built. + keep(ourDce); + return true; + } // add the conflicting path to merge result String currentPath = tw.getPathString(); MergeResult result = new MergeResult<>( @@ -1312,8 +1319,12 @@ public class ResolveMerger extends ThreeWayMerger { addToCheckout(currentPath, null, attributes); return true; } catch (BinaryBlobException e) { - // if the file is binary in either OURS, THEIRS or BASE - // here, we don't have an option to ignore conflicts + // The file is binary in either OURS, THEIRS or BASE + if (ignoreConflicts) { + // When ignoreConflicts = true, we simply keep OURS to allow virtual commit to be built. + keep(ourDce); + return true; + } } } switch (getContentMergeStrategy()) { @@ -1354,6 +1365,8 @@ public class ResolveMerger extends ThreeWayMerger { } } } else { + // This is reachable if contentMerge() call above threw BinaryBlobException, so we don't + // need to check ignoreConflicts here, since it's already handled above. result.setContainsConflicts(true); addConflict(base, ours, theirs); unmergedPaths.add(currentPath); -- cgit v1.2.3 From 7a4c059322f13dace12b8a3be5e8e4ee0cb088bc Mon Sep 17 00:00:00 2001 From: Nasser Grainawi Date: Tue, 8 Oct 2024 14:46:22 -0700 Subject: WindowCursor: Fix parameter name in javadoc This doc was copied from another where the parameter had a shorter name. Change-Id: I6ae1afa525f02e26ab7224cced56db946f679bb8 Signed-off-by: Nasser Grainawi --- .../src/org/eclipse/jgit/internal/storage/file/WindowCursor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCursor.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCursor.java index 01f514b939..11c45471e4 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCursor.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCursor.java @@ -205,7 +205,7 @@ final class WindowCursor extends ObjectReader implements ObjectReuseAsIs { * @param cnt * number of bytes to copy. This value may exceed the number of * bytes remaining in the window starting at offset - * pos. + * position. * @return number of bytes actually copied; this may be less than * cnt if cnt exceeded the number of bytes * available. -- cgit v1.2.3 From 9e1cd8aece4fbba301edb1eddbd0e7d36d3bb40c Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Mon, 16 Sep 2024 08:30:39 +0200 Subject: NoteMapMerger: remove unnecessary cast Change-Id: I3be4963f506529bbadc5b6dfc0b625ee85effc1f --- org.eclipse.jgit/src/org/eclipse/jgit/notes/NoteMapMerger.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/notes/NoteMapMerger.java b/org.eclipse.jgit/src/org/eclipse/jgit/notes/NoteMapMerger.java index 79ceb1316a..30512c17b8 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/notes/NoteMapMerger.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/notes/NoteMapMerger.java @@ -199,7 +199,7 @@ public class NoteMapMerger { if (child == null) return; if (child instanceof InMemoryNoteBucket) - b.setBucket(cell, ((InMemoryNoteBucket) child).writeTree(inserter)); + b.setBucket(cell, child.writeTree(inserter)); else b.setBucket(cell, child.getTreeId()); } -- cgit v1.2.3 From 2f15177e56d6fff79a84f178bfb0b33c926ddece Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Wed, 9 Oct 2024 01:41:30 +0200 Subject: UploadPack: suppress resource warning for DepthWalk.RevWalk which borrows the ObjectReader from `walk` which is closed by UploadPack#close. Change-Id: Idb91f025c2872421702034381bb55d292d0e74ed --- org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java index 3772c00819..8945f6e67f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java @@ -2369,7 +2369,8 @@ public class UploadPack implements Closeable { : req.getDepth() - 1; pw.setShallowPack(req.getDepth(), unshallowCommits); - // Ownership is transferred below + // dw borrows the reader from walk which is closed by #close + @SuppressWarnings("resource") DepthWalk.RevWalk dw = new DepthWalk.RevWalk( walk.getObjectReader(), walkDepth); dw.setDeepenSince(req.getDeepenSince()); -- cgit v1.2.3 From c9586d19ab97e46239f1f9a4a522b914a5016107 Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Fri, 11 Oct 2024 13:25:57 -0700 Subject: DfsReaderOptions: read loadRevIndexInParallel from config The options have the field but it isn't loaded from the config. This forces a workaround downstream. Read the option from the config, as the others. Change-Id: I7720812e0577d8f45f6b7f5b8495a8b64729125e --- Documentation/config-options.md | 1 + .../org/eclipse/jgit/internal/storage/dfs/DfsReaderOptions.java | 4 ++++ org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java | 7 +++++++ 3 files changed, 12 insertions(+) (limited to 'org.eclipse.jgit/src') diff --git a/Documentation/config-options.md b/Documentation/config-options.md index 78930e6267..eeb78ff550 100644 --- a/Documentation/config-options.md +++ b/Documentation/config-options.md @@ -31,6 +31,7 @@ For details on native git options see also the official [git config documentatio | `core.dfs.blockSize` | `64 kiB` | ⃞ | Size in bytes of a single window read in from the pack file into the DFS block cache. | | `core.dfs.concurrencyLevel` | `32` | ⃞ | The estimated number of threads concurrently accessing the DFS block cache. | | `core.dfs.deltaBaseCacheLimit` | `10 MiB` | ⃞ | Maximum number of bytes to hold in per-reader DFS delta base cache. | +| `core.dfs.loadRevIndexInParallel` | false; | ⃞ | Try to load the reverse index in parallel with the bitmap index. | | `core.dfs.streamFileThreshold` | `50 MiB` | ⃞ | The size threshold beyond which objects must be streamed. | | `core.dfs.streamBuffer` | Block size of the pack | ⃞ | Number of bytes to use for buffering when streaming a pack file during copying. If 0 the block size of the pack is used| | `core.dfs.streamRatio` | `0.30` | ⃞ | Ratio of DFS block cache to occupy with a copied pack. Values between `0` and `1.0`. | diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReaderOptions.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReaderOptions.java index 5f5e81977b..c3974691a1 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReaderOptions.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReaderOptions.java @@ -13,6 +13,7 @@ package org.eclipse.jgit.internal.storage.dfs; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_CORE_SECTION; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_DFS_SECTION; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_DELTA_BASE_CACHE_LIMIT; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_LOAD_REV_INDEX_IN_PARALLEL; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_STREAM_BUFFER; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_STREAM_FILE_THRESHOLD; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_USE_OBJECT_SIZE_INDEX; @@ -197,6 +198,9 @@ public class DfsReaderOptions { setUseObjectSizeIndex(rc.getBoolean(CONFIG_CORE_SECTION, CONFIG_DFS_SECTION, CONFIG_KEY_USE_OBJECT_SIZE_INDEX, false)); + setLoadRevIndexInParallel( + rc.getBoolean(CONFIG_CORE_SECTION, CONFIG_DFS_SECTION, + CONFIG_KEY_LOAD_REV_INDEX_IN_PARALLEL, false)); return this; } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java index acb54d7405..c5f27688be 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java @@ -1027,4 +1027,11 @@ public final class ConfigConstants { * @since 7.0 */ public static final String CONFIG_KEY_USE_OBJECT_SIZE_INDEX = "useObjectSizeIndex"; + + /** + * The "loadRevIndexInParallel" key + * + * @since 7.1 + */ + public static final String CONFIG_KEY_LOAD_REV_INDEX_IN_PARALLEL = "loadRevIndexInParallel"; } -- cgit v1.2.3 From 1519c147948eb1108bdf45f2aeed84746dacff9c Mon Sep 17 00:00:00 2001 From: Luca Milanesio Date: Mon, 7 Oct 2024 23:16:58 +0100 Subject: Align request policies with CGit CGit defines the SHA request policies using a bitmask that represents which policy is implied by another policy. For example, in CGit the ALLOW_TIP_SHA1 is 0x01 and ALLOW_REACHABLE_SHA1 is 0x02, which are associated to two different bit in a 3-bit value. The ALLOW_ANY_SHA1 value is 0x07 which denotes a different policy that implies the previous two ones, because is represented with a 3-bit bitmask having all ones. Associate the JGit RequestPolicy enum to the same CGit bitmask values and use the same logic for the purpose of advertising the server capabilities. The JGit code becomes easier to read and associate with its counterpart in CGit, especially during the capabilities advertising phase. Also add a new utility method RequestPolicy.implies() which is more readable than a direct bitmask and operator. Bug: jgit-68 Change-Id: I6b2649b06623a3b8226ee8413e4f1f58ad8ea28b --- .../src/org/eclipse/jgit/transport/UploadPack.java | 38 ++++++++++++++-------- 1 file changed, 25 insertions(+), 13 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java index 8945f6e67f..216cde1e37 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java @@ -117,13 +117,13 @@ public class UploadPack implements Closeable { /** Policy the server uses to validate client requests */ public enum RequestPolicy { /** Client may only ask for objects the server advertised a reference for. */ - ADVERTISED, + ADVERTISED(0x08), /** * Client may ask for any commit reachable from a reference advertised by * the server. */ - REACHABLE_COMMIT, + REACHABLE_COMMIT(0x02), /** * Client may ask for objects that are the tip of any reference, even if not @@ -133,18 +133,34 @@ public class UploadPack implements Closeable { * * @since 3.1 */ - TIP, + TIP(0x01), /** * Client may ask for any commit reachable from any reference, even if that - * reference wasn't advertised. + * reference wasn't advertised, implies REACHABLE_COMMIT and TIP. * * @since 3.1 */ - REACHABLE_COMMIT_TIP, + REACHABLE_COMMIT_TIP(0x03), - /** Client may ask for any SHA-1 in the repository. */ - ANY; + /** Client may ask for any SHA-1 in the repository, implies REACHABLE_COMMIT_TIP. */ + ANY(0x07); + + private final int bitmask; + + RequestPolicy(int bitmask) { + this.bitmask = bitmask; + } + + /** + * Check if the current policy implies another, based on its bitmask. + * + * @param implied the implied policy based on its bitmask. + * @return true if the policy is implied. + */ + public boolean implies(RequestPolicy implied) { + return (bitmask & implied.bitmask) != 0; + } } /** @@ -1582,13 +1598,9 @@ public class UploadPack implements Closeable { if (!biDirectionalPipe) adv.advertiseCapability(OPTION_NO_DONE); RequestPolicy policy = getRequestPolicy(); - if (policy == RequestPolicy.TIP - || policy == RequestPolicy.REACHABLE_COMMIT_TIP - || policy == null) + if (policy == null || policy.implies(RequestPolicy.TIP)) adv.advertiseCapability(OPTION_ALLOW_TIP_SHA1_IN_WANT); - if (policy == RequestPolicy.REACHABLE_COMMIT - || policy == RequestPolicy.REACHABLE_COMMIT_TIP - || policy == null) + if (policy == null || policy.implies(RequestPolicy.REACHABLE_COMMIT)) adv.advertiseCapability(OPTION_ALLOW_REACHABLE_SHA1_IN_WANT); adv.advertiseCapability(OPTION_AGENT, UserAgent.get()); if (transferConfig.isAllowFilter()) { -- cgit v1.2.3 From 6a169f10835c780e65b4e920dce3bed5006dc68d Mon Sep 17 00:00:00 2001 From: Laura Hamelin Date: Mon, 29 Jul 2024 16:12:15 -0700 Subject: DfsBlockCacheConfigs: add debug configuration print This will write out configuration values on a line by line basis to a given PrintWriter. Primary usage is as a semi-formatted debug print of the configuration values used by dfs block cache. Change-Id: I96724262245e4aa3423734a8b10de83322c4f89f --- .../storage/dfs/DfsBlockCacheConfigTest.java | 44 ++++++++++++++++++ .../internal/storage/dfs/DfsBlockCacheConfig.java | 54 +++++++++++++++++++++- 2 files changed, 97 insertions(+), 1 deletion(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfigTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfigTest.java index 4b7aae05f0..65774e6d6c 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfigTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfigTest.java @@ -54,7 +54,12 @@ import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; import static org.junit.Assert.assertThrows; +import java.io.ByteArrayOutputStream; +import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import org.eclipse.jgit.internal.JGitText; @@ -250,6 +255,45 @@ public class DfsBlockCacheConfigTest { () -> new DfsBlockCacheConfig().fromConfig(config)); } + @Test + public void writeConfigurationDebug_writesConfigsToWriter() + throws Exception { + Config config = new Config(); + config.setLong(CONFIG_CORE_SECTION, CONFIG_DFS_SECTION, + CONFIG_KEY_BLOCK_LIMIT, 50 * 1024); + config.setInt(CONFIG_CORE_SECTION, CONFIG_DFS_SECTION, + CONFIG_KEY_BLOCK_SIZE, 1024); + config.setInt(CONFIG_CORE_SECTION, CONFIG_DFS_SECTION, + CONFIG_KEY_CONCURRENCY_LEVEL, 3); + config.setString(CONFIG_CORE_SECTION, CONFIG_DFS_SECTION, + CONFIG_KEY_STREAM_RATIO, "0.5"); + addPackExtConfigEntry(config, "pack", List.of(PackExt.PACK), + /* blockLimit= */ 20 * 512, /* blockSize= */ 512); + + DfsBlockCacheConfig cacheConfig = new DfsBlockCacheConfig() + .fromConfig(config); + Map hotmap = Map.of(PackExt.PACK, 10); + cacheConfig.setCacheHotMap(hotmap); + + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + cacheConfig.print(new PrintWriter(byteArrayOutputStream, true, + StandardCharsets.UTF_8)); + + String writenConfig = byteArrayOutputStream + .toString(StandardCharsets.UTF_8); + + List writenLines = Arrays.asList(writenConfig.split("\n")); + assertThat(writenLines, + equalTo(List.of("Name: dfs", " BlockLimit: " + (50 * 1024), + " BlockSize: 1024", " StreamRatio: 0.5", + " ConcurrencyLevel: 3", + " CacheHotMapEntry: " + PackExt.PACK + " : " + 10, + " Name: dfs.pack", " BlockLimit: " + 20 * 512, + " BlockSize: 512", " StreamRatio: 0.3", + " ConcurrencyLevel: 32", + " PackExts: " + List.of(PackExt.PACK)))); + } + private static void addPackExtConfigEntry(Config config, String configName, List packExts, long blockLimit, int blockSize) { String packExtConfigName = CONFIG_DFS_CACHE_PREFIX + configName; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java index 52eb9e0785..0d56689a67 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java @@ -19,6 +19,7 @@ import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_CONCURRENCY_LEVEL; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_PACK_EXTENSIONS; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_STREAM_RATIO; +import java.io.PrintWriter; import java.text.MessageFormat; import java.time.Duration; import java.util.ArrayList; @@ -82,6 +83,51 @@ public class DfsBlockCacheConfig { packExtCacheConfigurations = Collections.emptyList(); } + /** + * Print the current cache configuration to the given {@link PrintWriter}. + * + * @param writer + * {@link PrintWriter} to write the cache's configuration to. + */ + public void print(PrintWriter writer) { + print(/* linePrefix= */ "", /* pad= */ " ", writer); + } + + /** + * Print the current cache configuration to the given {@link PrintWriter}. + * + * @param linePrefix + * prefix to prepend all writen lines with. Ex a string of 0 or + * more " " entries. + * @param pad + * filler used to extend linePrefix. Ex a multiple of " ". + * @param writer + * {@link PrintWriter} to write the cache's configuration to. + */ + private void print(String linePrefix, String pad, PrintWriter writer) { + String currentPrefixLevel = linePrefix; + if (!name.isEmpty() || !packExtCacheConfigurations.isEmpty()) { + String name = this.name; + if (name.isEmpty()) { + name = ""; + } + writer.println(linePrefix + "Name: " + name); + currentPrefixLevel += pad; + } + writer.println(currentPrefixLevel + "BlockLimit: " + blockLimit); + writer.println(currentPrefixLevel + "BlockSize: " + blockSize); + writer.println(currentPrefixLevel + "StreamRatio: " + streamRatio); + writer.println( + currentPrefixLevel + "ConcurrencyLevel: " + concurrencyLevel); + for (Map.Entry entry : cacheHotMap.entrySet()) { + writer.println(currentPrefixLevel + "CacheHotMapEntry: " + + entry.getKey() + " : " + entry.getValue()); + } + for (DfsBlockCachePackExtConfig extConfig : packExtCacheConfigurations) { + extConfig.print(currentPrefixLevel, pad, writer); + } + } + /** * Get the name for the block cache configured by this cache config. * @@ -531,5 +577,11 @@ public class DfsBlockCacheConfig { return new DfsBlockCachePackExtConfig(EnumSet.copyOf(packExts), dfsBlockCacheConfig); } + + void print(String linePrefix, String pad, PrintWriter writer) { + packExtCacheConfiguration.print(linePrefix, pad, writer); + writer.println(linePrefix + pad + "PackExts: " + + packExts.stream().sorted().toList()); + } } -} \ No newline at end of file +} -- cgit v1.2.3 From fe8919d107cbfbc52b84f9b8e9fe55668dfbebb1 Mon Sep 17 00:00:00 2001 From: Laura Hamelin Date: Wed, 16 Oct 2024 15:16:00 -0700 Subject: DfsBlockCache: refactor stats implementations. The stats interface has an implementation in the interface itself and another inside the PackExtBlockCache class. This asymmetry gets on the way to implement stats-per-table later. Make DfsBlockCacheStats (the stats of a single table) a top-level class and create an aggregator class to combine multiple stats. This makes the stats classes mirror the table classes structure (singles tables + composite). This change is part of a refactor to support providing detailed stats breakdowns for cache implementations using multiple table instances while keeping the existing "aggregated" view of cache stats. Change-Id: I79c11e4ea24afe4b449efdbb47bc81eed363ffd3 --- .../storage/dfs/AggregatedBlockCacheStatsTest.java | 215 +++++++++++++++++++++ .../storage/dfs/PackExtBlockCacheTableTest.java | 1 - .../storage/dfs/AggregatedBlockCacheStats.java | 127 ++++++++++++ .../internal/storage/dfs/DfsBlockCacheStats.java | 197 +++++++++++++++++++ .../internal/storage/dfs/DfsBlockCacheTable.java | 194 +------------------ .../storage/dfs/PackExtBlockCacheTable.java | 100 +--------- 6 files changed, 547 insertions(+), 287 deletions(-) create mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/AggregatedBlockCacheStatsTest.java create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/AggregatedBlockCacheStats.java create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheStats.java (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/AggregatedBlockCacheStatsTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/AggregatedBlockCacheStatsTest.java new file mode 100644 index 0000000000..ac769498e2 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/AggregatedBlockCacheStatsTest.java @@ -0,0 +1,215 @@ +/* + * Copyright (c) 2024, Google LLC and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.storage.dfs; + +import static org.eclipse.jgit.internal.storage.dfs.DfsBlockCacheTable.BlockCacheStats; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.junit.Assert.assertArrayEquals; + +import java.util.List; + +import org.eclipse.jgit.internal.storage.pack.PackExt; +import org.junit.Test; + +public class AggregatedBlockCacheStatsTest { + @Test + public void getName() { + BlockCacheStats aggregatedBlockCacheStats = AggregatedBlockCacheStats + .fromStatsList("name", List.of()); + + assertThat(aggregatedBlockCacheStats.getName(), equalTo("name")); + } + + @Test + public void getCurrentSize_aggregatesCurrentSizes() { + long[] currentSizes = createEmptyStatsArray(); + + DfsBlockCacheStats packStats = new DfsBlockCacheStats(); + packStats.addToLiveBytes(new TestKey(PackExt.PACK), 5); + currentSizes[PackExt.PACK.getPosition()] = 5; + + DfsBlockCacheStats bitmapStats = new DfsBlockCacheStats(); + bitmapStats.addToLiveBytes(new TestKey(PackExt.BITMAP_INDEX), 6); + currentSizes[PackExt.BITMAP_INDEX.getPosition()] = 6; + + DfsBlockCacheStats indexStats = new DfsBlockCacheStats(); + indexStats.addToLiveBytes(new TestKey(PackExt.INDEX), 7); + currentSizes[PackExt.INDEX.getPosition()] = 7; + + BlockCacheStats aggregatedBlockCacheStats = AggregatedBlockCacheStats + .fromStatsList("name", + List.of(packStats, bitmapStats, indexStats)); + + assertArrayEquals(aggregatedBlockCacheStats.getCurrentSize(), + currentSizes); + } + + @Test + public void getHitCount_aggregatesHitCounts() { + long[] hitCounts = createEmptyStatsArray(); + + DfsBlockCacheStats packStats = new DfsBlockCacheStats(); + incrementCounter(5, + () -> packStats.incrementHit(new TestKey(PackExt.PACK))); + hitCounts[PackExt.PACK.getPosition()] = 5; + + DfsBlockCacheStats bitmapStats = new DfsBlockCacheStats(); + incrementCounter(6, () -> bitmapStats + .incrementHit(new TestKey(PackExt.BITMAP_INDEX))); + hitCounts[PackExt.BITMAP_INDEX.getPosition()] = 6; + + DfsBlockCacheStats indexStats = new DfsBlockCacheStats(); + incrementCounter(7, + () -> indexStats.incrementHit(new TestKey(PackExt.INDEX))); + hitCounts[PackExt.INDEX.getPosition()] = 7; + + BlockCacheStats aggregatedBlockCacheStats = AggregatedBlockCacheStats + .fromStatsList("name", + List.of(packStats, bitmapStats, indexStats)); + + assertArrayEquals(aggregatedBlockCacheStats.getHitCount(), hitCounts); + } + + @Test + public void getMissCount_aggregatesMissCounts() { + long[] missCounts = createEmptyStatsArray(); + + DfsBlockCacheStats packStats = new DfsBlockCacheStats(); + incrementCounter(5, + () -> packStats.incrementMiss(new TestKey(PackExt.PACK))); + missCounts[PackExt.PACK.getPosition()] = 5; + + DfsBlockCacheStats bitmapStats = new DfsBlockCacheStats(); + incrementCounter(6, () -> bitmapStats + .incrementMiss(new TestKey(PackExt.BITMAP_INDEX))); + missCounts[PackExt.BITMAP_INDEX.getPosition()] = 6; + + DfsBlockCacheStats indexStats = new DfsBlockCacheStats(); + incrementCounter(7, + () -> indexStats.incrementMiss(new TestKey(PackExt.INDEX))); + missCounts[PackExt.INDEX.getPosition()] = 7; + + BlockCacheStats aggregatedBlockCacheStats = AggregatedBlockCacheStats + .fromStatsList("name", + List.of(packStats, bitmapStats, indexStats)); + + assertArrayEquals(aggregatedBlockCacheStats.getMissCount(), missCounts); + } + + @Test + public void getTotalRequestCount_aggregatesRequestCounts() { + long[] totalRequestCounts = createEmptyStatsArray(); + + DfsBlockCacheStats packStats = new DfsBlockCacheStats(); + incrementCounter(5, () -> { + packStats.incrementHit(new TestKey(PackExt.PACK)); + packStats.incrementMiss(new TestKey(PackExt.PACK)); + }); + totalRequestCounts[PackExt.PACK.getPosition()] = 10; + + DfsBlockCacheStats bitmapStats = new DfsBlockCacheStats(); + incrementCounter(6, () -> { + bitmapStats.incrementHit(new TestKey(PackExt.BITMAP_INDEX)); + bitmapStats.incrementMiss(new TestKey(PackExt.BITMAP_INDEX)); + }); + totalRequestCounts[PackExt.BITMAP_INDEX.getPosition()] = 12; + + DfsBlockCacheStats indexStats = new DfsBlockCacheStats(); + incrementCounter(7, () -> { + indexStats.incrementHit(new TestKey(PackExt.INDEX)); + indexStats.incrementMiss(new TestKey(PackExt.INDEX)); + }); + totalRequestCounts[PackExt.INDEX.getPosition()] = 14; + + BlockCacheStats aggregatedBlockCacheStats = AggregatedBlockCacheStats + .fromStatsList("name", + List.of(packStats, bitmapStats, indexStats)); + + assertArrayEquals(aggregatedBlockCacheStats.getTotalRequestCount(), + totalRequestCounts); + } + + @Test + public void getHitRatio_aggregatesHitRatios() { + long[] hitRatios = createEmptyStatsArray(); + + DfsBlockCacheStats packStats = new DfsBlockCacheStats(); + incrementCounter(5, + () -> packStats.incrementHit(new TestKey(PackExt.PACK))); + hitRatios[PackExt.PACK.getPosition()] = 100; + + DfsBlockCacheStats bitmapStats = new DfsBlockCacheStats(); + incrementCounter(6, () -> { + bitmapStats.incrementHit(new TestKey(PackExt.BITMAP_INDEX)); + bitmapStats.incrementMiss(new TestKey(PackExt.BITMAP_INDEX)); + }); + hitRatios[PackExt.BITMAP_INDEX.getPosition()] = 50; + + DfsBlockCacheStats indexStats = new DfsBlockCacheStats(); + incrementCounter(7, + () -> indexStats.incrementMiss(new TestKey(PackExt.INDEX))); + hitRatios[PackExt.INDEX.getPosition()] = 0; + + BlockCacheStats aggregatedBlockCacheStats = AggregatedBlockCacheStats + .fromStatsList("Name", + List.of(packStats, bitmapStats, indexStats)); + + assertArrayEquals(aggregatedBlockCacheStats.getHitRatio(), hitRatios); + } + + @Test + public void getEvictions_aggregatesEvictions() { + long[] evictions = createEmptyStatsArray(); + + DfsBlockCacheStats packStats = new DfsBlockCacheStats(); + incrementCounter(5, + () -> packStats.incrementEvict(new TestKey(PackExt.PACK))); + evictions[PackExt.PACK.getPosition()] = 5; + + DfsBlockCacheStats bitmapStats = new DfsBlockCacheStats(); + incrementCounter(6, () -> bitmapStats + .incrementEvict(new TestKey(PackExt.BITMAP_INDEX))); + evictions[PackExt.BITMAP_INDEX.getPosition()] = 6; + + DfsBlockCacheStats indexStats = new DfsBlockCacheStats(); + incrementCounter(7, + () -> indexStats.incrementEvict(new TestKey(PackExt.INDEX))); + evictions[PackExt.INDEX.getPosition()] = 7; + + BlockCacheStats aggregatedBlockCacheStats = AggregatedBlockCacheStats + .fromStatsList("Name", + List.of(packStats, bitmapStats, indexStats)); + + assertArrayEquals(aggregatedBlockCacheStats.getEvictions(), evictions); + } + + private static void incrementCounter(int amount, Runnable fn) { + for (int i = 0; i < amount; i++) { + fn.run(); + } + } + + private static long[] createEmptyStatsArray() { + return new long[PackExt.values().length]; + } + + private static class TestKey extends DfsStreamKey { + TestKey(PackExt packExt) { + super(0, packExt); + } + + @Override + public boolean equals(Object o) { + return false; + } + } +} \ No newline at end of file diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/PackExtBlockCacheTableTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/PackExtBlockCacheTableTest.java index f2a5abcacd..c5c964bcab 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/PackExtBlockCacheTableTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/PackExtBlockCacheTableTest.java @@ -31,7 +31,6 @@ import java.util.Map; import org.eclipse.jgit.internal.storage.dfs.DfsBlockCache.Ref; import org.eclipse.jgit.internal.storage.dfs.DfsBlockCache.RefLoader; import org.eclipse.jgit.internal.storage.dfs.DfsBlockCacheConfig.DfsBlockCachePackExtConfig; -import org.eclipse.jgit.internal.storage.dfs.DfsBlockCacheTable.DfsBlockCacheStats; import org.eclipse.jgit.internal.storage.pack.PackExt; import org.junit.Test; import org.mockito.Mockito; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/AggregatedBlockCacheStats.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/AggregatedBlockCacheStats.java new file mode 100644 index 0000000000..743f4606c4 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/AggregatedBlockCacheStats.java @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2024, Google LLC and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.storage.dfs; + +import static org.eclipse.jgit.internal.storage.dfs.DfsBlockCacheTable.BlockCacheStats; + +import java.util.List; + +import org.eclipse.jgit.internal.storage.pack.PackExt; + +/** + * Aggregates values for all given {@link BlockCacheStats}. + */ +class AggregatedBlockCacheStats implements BlockCacheStats { + private final String name; + + private final List blockCacheStats; + + static BlockCacheStats fromStatsList(String name, + List blockCacheStats) { + if (blockCacheStats.size() == 1) { + return blockCacheStats.get(0); + } + return new AggregatedBlockCacheStats(name, blockCacheStats); + } + + private AggregatedBlockCacheStats(String name, + List blockCacheStats) { + this.name = name; + this.blockCacheStats = blockCacheStats; + } + + @Override + public String getName() { + return name; + } + + @Override + public long[] getCurrentSize() { + long[] sums = emptyPackStats(); + for (BlockCacheStats blockCacheStatsEntry : blockCacheStats) { + sums = add(sums, blockCacheStatsEntry.getCurrentSize()); + } + return sums; + } + + @Override + public long[] getHitCount() { + long[] sums = emptyPackStats(); + for (BlockCacheStats blockCacheStatsEntry : blockCacheStats) { + sums = add(sums, blockCacheStatsEntry.getHitCount()); + } + return sums; + } + + @Override + public long[] getMissCount() { + long[] sums = emptyPackStats(); + for (BlockCacheStats blockCacheStatsEntry : blockCacheStats) { + sums = add(sums, blockCacheStatsEntry.getMissCount()); + } + return sums; + } + + @Override + public long[] getTotalRequestCount() { + long[] sums = emptyPackStats(); + for (BlockCacheStats blockCacheStatsEntry : blockCacheStats) { + sums = add(sums, blockCacheStatsEntry.getTotalRequestCount()); + } + return sums; + } + + @Override + public long[] getHitRatio() { + long[] hit = getHitCount(); + long[] miss = getMissCount(); + long[] ratio = new long[Math.max(hit.length, miss.length)]; + for (int i = 0; i < ratio.length; i++) { + if (i >= hit.length) { + ratio[i] = 0; + } else if (i >= miss.length) { + ratio[i] = 100; + } else { + long total = hit[i] + miss[i]; + ratio[i] = total == 0 ? 0 : hit[i] * 100 / total; + } + } + return ratio; + } + + @Override + public long[] getEvictions() { + long[] sums = emptyPackStats(); + for (BlockCacheStats blockCacheStatsEntry : blockCacheStats) { + sums = add(sums, blockCacheStatsEntry.getEvictions()); + } + return sums; + } + + private static long[] emptyPackStats() { + return new long[PackExt.values().length]; + } + + private static long[] add(long[] first, long[] second) { + long[] sums = new long[Integer.max(first.length, second.length)]; + int i; + for (i = 0; i < Integer.min(first.length, second.length); i++) { + sums[i] = first[i] + second[i]; + } + for (int j = i; j < first.length; j++) { + sums[j] = first[i]; + } + for (int j = i; j < second.length; j++) { + sums[j] = second[i]; + } + return sums; + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheStats.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheStats.java new file mode 100644 index 0000000000..518db8c82d --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheStats.java @@ -0,0 +1,197 @@ +/* + * Copyright (c) 2024, Google LLC and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * http://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.storage.dfs; + +import static org.eclipse.jgit.internal.storage.dfs.DfsBlockCacheTable.BlockCacheStats; + +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; + +import org.eclipse.jgit.internal.storage.pack.PackExt; + +/** + * Keeps track of stats for a Block Cache table. + */ +class DfsBlockCacheStats implements BlockCacheStats { + private final String name; + + /** + * Number of times a block was found in the cache, per pack file extension. + */ + private final AtomicReference statHit; + + /** + * Number of times a block was not found, and had to be loaded, per pack + * file extension. + */ + private final AtomicReference statMiss; + + /** + * Number of blocks evicted due to cache being full, per pack file + * extension. + */ + private final AtomicReference statEvict; + + /** + * Number of bytes currently loaded in the cache, per pack file extension. + */ + private final AtomicReference liveBytes; + + DfsBlockCacheStats() { + this(""); + } + + DfsBlockCacheStats(String name) { + this.name = name; + statHit = new AtomicReference<>(newCounters()); + statMiss = new AtomicReference<>(newCounters()); + statEvict = new AtomicReference<>(newCounters()); + liveBytes = new AtomicReference<>(newCounters()); + } + + @Override + public String getName() { + return name; + } + + /** + * Increment the {@code statHit} count. + * + * @param key + * key identifying which liveBytes entry to update. + */ + void incrementHit(DfsStreamKey key) { + getStat(statHit, key).incrementAndGet(); + } + + /** + * Increment the {@code statMiss} count. + * + * @param key + * key identifying which liveBytes entry to update. + */ + void incrementMiss(DfsStreamKey key) { + getStat(statMiss, key).incrementAndGet(); + } + + /** + * Increment the {@code statEvict} count. + * + * @param key + * key identifying which liveBytes entry to update. + */ + void incrementEvict(DfsStreamKey key) { + getStat(statEvict, key).incrementAndGet(); + } + + /** + * Add {@code size} to the {@code liveBytes} count. + * + * @param key + * key identifying which liveBytes entry to update. + * @param size + * amount to increment the count by. + */ + void addToLiveBytes(DfsStreamKey key, long size) { + getStat(liveBytes, key).addAndGet(size); + } + + @Override + public long[] getCurrentSize() { + return getStatVals(liveBytes); + } + + @Override + public long[] getHitCount() { + return getStatVals(statHit); + } + + @Override + public long[] getMissCount() { + return getStatVals(statMiss); + } + + @Override + public long[] getTotalRequestCount() { + AtomicLong[] hit = statHit.get(); + AtomicLong[] miss = statMiss.get(); + long[] cnt = new long[Math.max(hit.length, miss.length)]; + for (int i = 0; i < hit.length; i++) { + cnt[i] += hit[i].get(); + } + for (int i = 0; i < miss.length; i++) { + cnt[i] += miss[i].get(); + } + return cnt; + } + + @Override + public long[] getHitRatio() { + AtomicLong[] hit = statHit.get(); + AtomicLong[] miss = statMiss.get(); + long[] ratio = new long[Math.max(hit.length, miss.length)]; + for (int i = 0; i < ratio.length; i++) { + if (i >= hit.length) { + ratio[i] = 0; + } else if (i >= miss.length) { + ratio[i] = 100; + } else { + long hitVal = hit[i].get(); + long missVal = miss[i].get(); + long total = hitVal + missVal; + ratio[i] = total == 0 ? 0 : hitVal * 100 / total; + } + } + return ratio; + } + + @Override + public long[] getEvictions() { + return getStatVals(statEvict); + } + + private static AtomicLong[] newCounters() { + AtomicLong[] ret = new AtomicLong[PackExt.values().length]; + for (int i = 0; i < ret.length; i++) { + ret[i] = new AtomicLong(); + } + return ret; + } + + private static long[] getStatVals(AtomicReference stat) { + AtomicLong[] stats = stat.get(); + long[] cnt = new long[stats.length]; + for (int i = 0; i < stats.length; i++) { + cnt[i] = stats[i].get(); + } + return cnt; + } + + private static AtomicLong getStat(AtomicReference stats, + DfsStreamKey key) { + int pos = key.packExtPos; + while (true) { + AtomicLong[] vals = stats.get(); + if (pos < vals.length) { + return vals[pos]; + } + AtomicLong[] expect = vals; + vals = new AtomicLong[Math.max(pos + 1, PackExt.values().length)]; + System.arraycopy(expect, 0, vals, 0, expect.length); + for (int i = expect.length; i < vals.length; i++) { + vals[i] = new AtomicLong(); + } + if (stats.compareAndSet(expect, vals)) { + return vals[pos]; + } + } + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheTable.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheTable.java index de5dc23963..43ba8c36ab 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheTable.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheTable.java @@ -11,10 +11,6 @@ package org.eclipse.jgit.internal.storage.dfs; import java.io.IOException; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; - -import org.eclipse.jgit.internal.storage.pack.PackExt; /** * Block cache table. @@ -148,6 +144,12 @@ public interface DfsBlockCacheTable { * Provides methods used with Block Cache statistics. */ interface BlockCacheStats { + + /** + * Get the name of the block cache generating this instance. + * + * @return this cache's name. + */ String getName(); /** @@ -199,186 +201,4 @@ public interface DfsBlockCacheTable { */ long[] getEvictions(); } - - /** - * Keeps track of stats for a Block Cache table. - */ - class DfsBlockCacheStats implements BlockCacheStats { - private final String name; - - /** - * Number of times a block was found in the cache, per pack file - * extension. - */ - private final AtomicReference statHit; - - /** - * Number of times a block was not found, and had to be loaded, per pack - * file extension. - */ - private final AtomicReference statMiss; - - /** - * Number of blocks evicted due to cache being full, per pack file - * extension. - */ - private final AtomicReference statEvict; - - /** - * Number of bytes currently loaded in the cache, per pack file - * extension. - */ - private final AtomicReference liveBytes; - - DfsBlockCacheStats() { - this(""); - } - - DfsBlockCacheStats(String name) { - this.name = name; - statHit = new AtomicReference<>(newCounters()); - statMiss = new AtomicReference<>(newCounters()); - statEvict = new AtomicReference<>(newCounters()); - liveBytes = new AtomicReference<>(newCounters()); - } - - @Override - public String getName() { - return name; - } - - /** - * Increment the {@code statHit} count. - * - * @param key - * key identifying which liveBytes entry to update. - */ - void incrementHit(DfsStreamKey key) { - getStat(statHit, key).incrementAndGet(); - } - - /** - * Increment the {@code statMiss} count. - * - * @param key - * key identifying which liveBytes entry to update. - */ - void incrementMiss(DfsStreamKey key) { - getStat(statMiss, key).incrementAndGet(); - } - - /** - * Increment the {@code statEvict} count. - * - * @param key - * key identifying which liveBytes entry to update. - */ - void incrementEvict(DfsStreamKey key) { - getStat(statEvict, key).incrementAndGet(); - } - - /** - * Add {@code size} to the {@code liveBytes} count. - * - * @param key - * key identifying which liveBytes entry to update. - * @param size - * amount to increment the count by. - */ - void addToLiveBytes(DfsStreamKey key, long size) { - getStat(liveBytes, key).addAndGet(size); - } - - @Override - public long[] getCurrentSize() { - return getStatVals(liveBytes); - } - - @Override - public long[] getHitCount() { - return getStatVals(statHit); - } - - @Override - public long[] getMissCount() { - return getStatVals(statMiss); - } - - @Override - public long[] getTotalRequestCount() { - AtomicLong[] hit = statHit.get(); - AtomicLong[] miss = statMiss.get(); - long[] cnt = new long[Math.max(hit.length, miss.length)]; - for (int i = 0; i < hit.length; i++) { - cnt[i] += hit[i].get(); - } - for (int i = 0; i < miss.length; i++) { - cnt[i] += miss[i].get(); - } - return cnt; - } - - @Override - public long[] getHitRatio() { - AtomicLong[] hit = statHit.get(); - AtomicLong[] miss = statMiss.get(); - long[] ratio = new long[Math.max(hit.length, miss.length)]; - for (int i = 0; i < ratio.length; i++) { - if (i >= hit.length) { - ratio[i] = 0; - } else if (i >= miss.length) { - ratio[i] = 100; - } else { - long hitVal = hit[i].get(); - long missVal = miss[i].get(); - long total = hitVal + missVal; - ratio[i] = total == 0 ? 0 : hitVal * 100 / total; - } - } - return ratio; - } - - @Override - public long[] getEvictions() { - return getStatVals(statEvict); - } - - private static AtomicLong[] newCounters() { - AtomicLong[] ret = new AtomicLong[PackExt.values().length]; - for (int i = 0; i < ret.length; i++) { - ret[i] = new AtomicLong(); - } - return ret; - } - - private static long[] getStatVals(AtomicReference stat) { - AtomicLong[] stats = stat.get(); - long[] cnt = new long[stats.length]; - for (int i = 0; i < stats.length; i++) { - cnt[i] = stats[i].get(); - } - return cnt; - } - - private static AtomicLong getStat(AtomicReference stats, - DfsStreamKey key) { - int pos = key.packExtPos; - while (true) { - AtomicLong[] vals = stats.get(); - if (pos < vals.length) { - return vals[pos]; - } - AtomicLong[] expect = vals; - vals = new AtomicLong[Math.max(pos + 1, - PackExt.values().length)]; - System.arraycopy(expect, 0, vals, 0, expect.length); - for (int i = expect.length; i < vals.length; i++) { - vals[i] = new AtomicLong(); - } - if (stats.compareAndSet(expect, vals)) { - return vals[pos]; - } - } - } - } -} \ No newline at end of file +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/PackExtBlockCacheTable.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/PackExtBlockCacheTable.java index 7f21b3f10a..e45643be84 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/PackExtBlockCacheTable.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/PackExtBlockCacheTable.java @@ -189,7 +189,7 @@ class PackExtBlockCacheTable implements DfsBlockCacheTable { @Override public BlockCacheStats getBlockCacheStats() { - return new CacheStats(name, + return AggregatedBlockCacheStats.fromStatsList(name, blockCacheTableList.stream() .map(DfsBlockCacheTable::getBlockCacheStats) .collect(Collectors.toList())); @@ -213,102 +213,4 @@ class PackExtBlockCacheTable implements DfsBlockCacheTable { private static PackExt getPackExt(DfsStreamKey key) { return PackExt.values()[key.packExtPos]; } - - private static class CacheStats implements BlockCacheStats { - private final String name; - - private final List blockCacheStats; - - private CacheStats(String name, List blockCacheStats) { - this.name = name; - this.blockCacheStats = blockCacheStats; - } - - @Override - public String getName() { - return name; - } - - @Override - public long[] getCurrentSize() { - long[] sums = emptyPackStats(); - for (BlockCacheStats blockCacheStatsEntry : blockCacheStats) { - sums = add(sums, blockCacheStatsEntry.getCurrentSize()); - } - return sums; - } - - @Override - public long[] getHitCount() { - long[] sums = emptyPackStats(); - for (BlockCacheStats blockCacheStatsEntry : blockCacheStats) { - sums = add(sums, blockCacheStatsEntry.getHitCount()); - } - return sums; - } - - @Override - public long[] getMissCount() { - long[] sums = emptyPackStats(); - for (BlockCacheStats blockCacheStatsEntry : blockCacheStats) { - sums = add(sums, blockCacheStatsEntry.getMissCount()); - } - return sums; - } - - @Override - public long[] getTotalRequestCount() { - long[] sums = emptyPackStats(); - for (BlockCacheStats blockCacheStatsEntry : blockCacheStats) { - sums = add(sums, blockCacheStatsEntry.getTotalRequestCount()); - } - return sums; - } - - @Override - public long[] getHitRatio() { - long[] hit = getHitCount(); - long[] miss = getMissCount(); - long[] ratio = new long[Math.max(hit.length, miss.length)]; - for (int i = 0; i < ratio.length; i++) { - if (i >= hit.length) { - ratio[i] = 0; - } else if (i >= miss.length) { - ratio[i] = 100; - } else { - long total = hit[i] + miss[i]; - ratio[i] = total == 0 ? 0 : hit[i] * 100 / total; - } - } - return ratio; - } - - @Override - public long[] getEvictions() { - long[] sums = emptyPackStats(); - for (BlockCacheStats blockCacheStatsEntry : blockCacheStats) { - sums = add(sums, blockCacheStatsEntry.getEvictions()); - } - return sums; - } - - private static long[] emptyPackStats() { - return new long[PackExt.values().length]; - } - - private static long[] add(long[] first, long[] second) { - long[] sums = new long[Integer.max(first.length, second.length)]; - int i; - for (i = 0; i < Integer.min(first.length, second.length); i++) { - sums[i] = first[i] + second[i]; - } - for (int j = i; j < first.length; j++) { - sums[j] = first[i]; - } - for (int j = i; j < second.length; j++) { - sums[j] = second[i]; - } - return sums; - } - } } -- cgit v1.2.3 From 6406793274876b973da2a7e1f1052e1edca1d4ef Mon Sep 17 00:00:00 2001 From: Laura Hamelin Date: Mon, 21 Oct 2024 15:59:43 -0700 Subject: DfsBlockCacheConfig: update stream.toList to collect(toList) This change updates usage of stream's list collector to support older jdk versions. Change-Id: Ia066d36aef8ab166efd837d5f6f01e5d1fdd3cb7 --- .../src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java index 0d56689a67..acd7add5a9 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java @@ -581,7 +581,7 @@ public class DfsBlockCacheConfig { void print(String linePrefix, String pad, PrintWriter writer) { packExtCacheConfiguration.print(linePrefix, pad, writer); writer.println(linePrefix + pad + "PackExts: " - + packExts.stream().sorted().toList()); + + packExts.stream().sorted().collect(Collectors.toList())); } } } -- cgit v1.2.3 From d4ef88dae905a8d8bca76671674e0423aa37f0bf Mon Sep 17 00:00:00 2001 From: Thomas Wolf Date: Sat, 28 Sep 2024 15:50:45 +0200 Subject: SSH signing: prepare config Include the SSH specifics in the GpgConfig so that we will have access to these configs later on. Change-Id: Iad3d6f2bdb5ba879e1672368c82d367b8ccd246c Signed-off-by: Thomas Wolf --- .../src/org/eclipse/jgit/lib/ConfigConstants.java | 29 +++++++++++++ .../src/org/eclipse/jgit/lib/GpgConfig.java | 50 ++++++++++++++++++++++ 2 files changed, 79 insertions(+) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java index c5f27688be..a57f1b714a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java @@ -205,8 +205,37 @@ public final class ConfigConstants { */ public static final String CONFIG_KEY_SIGNINGKEY = "signingKey"; + /** + * The "ssh" subsection key. + * + * @since 7.1 + */ + public static final String CONFIG_SSH_SUBSECTION = "ssh"; + + /** + * The "defaultKeyCommand" key. + * + * @since 7.1 + */ + public static final String CONFIG_KEY_SSH_DEFAULT_KEY_COMMAND = "defaultKeyCommand"; + + /** + * The "allowedSignersFile" key. + * + * @since 7.1 + */ + public static final String CONFIG_KEY_SSH_ALLOWED_SIGNERS_FILE = "allowedSignersFile"; + + /** + * The "revocationFile" key, + * + * @since 7.1 + */ + public static final String CONFIG_KEY_SSH_REVOCATION_FILE = "revocationFile"; + /** * The "commit" section + * * @since 5.2 */ public static final String CONFIG_COMMIT_SECTION = "commit"; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgConfig.java index fb5c904215..76ed36a6e5 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgConfig.java @@ -61,6 +61,12 @@ public class GpgConfig { private final boolean forceAnnotated; + private final String sshDefaultKeyCommand; + + private final String sshAllowedSignersFile; + + private final String sshRevocationFile; + /** * Create a new GPG config that reads the configuration from config. * @@ -88,6 +94,17 @@ public class GpgConfig { ConfigConstants.CONFIG_KEY_GPGSIGN, false); forceAnnotated = config.getBoolean(ConfigConstants.CONFIG_TAG_SECTION, ConfigConstants.CONFIG_KEY_FORCE_SIGN_ANNOTATED, false); + sshDefaultKeyCommand = config.getString( + ConfigConstants.CONFIG_GPG_SECTION, + ConfigConstants.CONFIG_SSH_SUBSECTION, + ConfigConstants.CONFIG_KEY_SSH_DEFAULT_KEY_COMMAND); + sshAllowedSignersFile = config.getString( + ConfigConstants.CONFIG_GPG_SECTION, + ConfigConstants.CONFIG_SSH_SUBSECTION, + ConfigConstants.CONFIG_KEY_SSH_ALLOWED_SIGNERS_FILE); + sshRevocationFile = config.getString(ConfigConstants.CONFIG_GPG_SECTION, + ConfigConstants.CONFIG_SSH_SUBSECTION, + ConfigConstants.CONFIG_KEY_SSH_REVOCATION_FILE); } /** @@ -151,4 +168,37 @@ public class GpgConfig { public boolean isSignAnnotated() { return forceAnnotated; } + + /** + * Retrieves the value of git config {@code gpg.ssh.defaultKeyCommand}. + * + * @return the value of {@code gpg.ssh.defaultKeyCommand} + * + * @since 7.1 + */ + public String getSshDefaultKeyCommand() { + return sshDefaultKeyCommand; + } + + /** + * Retrieves the value of git config {@code gpg.ssh.allowedSignersFile}. + * + * @return the value of {@code gpg.ssh.allowedSignersFile} + * + * @since 7.1 + */ + public String getSshAllowedSignersFile() { + return sshAllowedSignersFile; + } + + /** + * Retrieves the value of git config {@code gpg.ssh.revocationFile}. + * + * @return the value of {@code gpg.ssh.revocationFile} + * + * @since 7.1 + */ + public String getSshRevocationFile() { + return sshRevocationFile; + } } -- cgit v1.2.3 From 06485d5275f22bda5bcd5a95ecd0ce33719db3aa Mon Sep 17 00:00:00 2001 From: Thomas Wolf Date: Sat, 28 Sep 2024 15:52:31 +0200 Subject: SSH signing: make OpenSSH pattern matching public SSH signing needs the same pattern matching algorithm as is used for host matching in host entries in ~/.ssh/config. So make that pattern matching available via a static method. Change-Id: Ia26f23666f323f44ce66f769fbcd6c85965eb219 Signed-off-by: Thomas Wolf --- .../internal/transport/ssh/OpenSshConfigFile.java | 43 +++++++++++++++------- 1 file changed, 30 insertions(+), 13 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/transport/ssh/OpenSshConfigFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/transport/ssh/OpenSshConfigFile.java index 3e75a9dde3..542d6e94f3 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/transport/ssh/OpenSshConfigFile.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/transport/ssh/OpenSshConfigFile.java @@ -427,7 +427,35 @@ public class OpenSshConfigFile implements SshConfigStore { return value; } - private static boolean patternMatchesHost(String pattern, String name) { + /** + * Tells whether a given {@code name} matches the given list of patterns, + * accounting for negative matches. + * + * @param patterns + * to test {@code name} against; any pattern starting with an + * exclamation mark is a negative pattern + * @param name + * to test + * @return {@code true} if the {@code name} matches at least one of the + * non-negative patterns and none of the negative patterns, + * {@code false} otherwise + * @since 7.1 + */ + public static boolean patternMatch(Iterable patterns, String name) { + boolean doesMatch = false; + for (String pattern : patterns) { + if (pattern.startsWith("!")) { //$NON-NLS-1$ + if (patternMatches(pattern.substring(1), name)) { + return false; + } + } else if (!doesMatch && patternMatches(pattern, name)) { + doesMatch = true; + } + } + return doesMatch; + } + + private static boolean patternMatches(String pattern, String name) { if (pattern.indexOf('*') >= 0 || pattern.indexOf('?') >= 0) { final FileNameMatcher fn; try { @@ -680,18 +708,7 @@ public class OpenSshConfigFile implements SshConfigStore { } boolean matches(String hostName) { - boolean doesMatch = false; - for (String pattern : patterns) { - if (pattern.startsWith("!")) { //$NON-NLS-1$ - if (patternMatchesHost(pattern.substring(1), hostName)) { - return false; - } - } else if (!doesMatch - && patternMatchesHost(pattern, hostName)) { - doesMatch = true; - } - } - return doesMatch; + return patternMatch(patterns, hostName); } private static String toKey(String key) { -- cgit v1.2.3 From 4902b2baf8dde2f6249eb0aeb7f1164921ce5a86 Mon Sep 17 00:00:00 2001 From: Jacek Centkowski Date: Fri, 20 Sep 2024 08:47:13 +0200 Subject: Add `numberOfPackFilesAfterBitmap` to RepoStatistics Introduce a `numberOfPackFilesAfterBitmap` that contains the number of packfiles created since the latest bitmap generation. Notes: * the `repo.getObjectDatabase().getPacks()` that obtains the list of packs (in the existing `getStatistics` function) uses `PackDirectory.scanPacks` that boils down to call to `PackDirectory.scanPacksImpl` which is sorting packs prior returning them therefore the `numberOfPackFilesAfterBitmap` is just all packs before the one that has bitmap attached * the improved version of `packAndPrune` function (one that skips non-existent packfiles) was introduced for testing Change-Id: I608011462f104fc002ac527aa405f492a8a4b0c2 --- ...NumberOfPackFilesAfterBitmapStatisticsTest.java | 180 +++++++++++++++++++++ .../org/eclipse/jgit/internal/storage/file/GC.java | 10 ++ 2 files changed, 190 insertions(+) create mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcNumberOfPackFilesAfterBitmapStatisticsTest.java (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcNumberOfPackFilesAfterBitmapStatisticsTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcNumberOfPackFilesAfterBitmapStatisticsTest.java new file mode 100644 index 0000000000..820f0c768d --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcNumberOfPackFilesAfterBitmapStatisticsTest.java @@ -0,0 +1,180 @@ +/* + * Copyright (c) 2024 Jacek Centkowski and others. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.storage.file; + +import static org.junit.Assert.assertEquals; + +import java.io.BufferedOutputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.util.HashSet; +import java.util.Set; +import java.util.stream.StreamSupport; + +import org.eclipse.jgit.internal.storage.file.PackIndex.MutableEntry; +import org.eclipse.jgit.internal.storage.pack.PackExt; +import org.eclipse.jgit.internal.storage.pack.PackWriter; +import org.eclipse.jgit.junit.TestRepository; +import org.eclipse.jgit.lib.NullProgressMonitor; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.PersonIdent; +import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.util.FileUtils; +import org.junit.Test; + +public class GcNumberOfPackFilesAfterBitmapStatisticsTest extends GcTestCase { + @Test + public void testShouldReportZeroObjectsForInitializedRepo() + throws IOException { + assertEquals(0L, gc.getStatistics().numberOfPackFilesAfterBitmap); + } + + @Test + public void testShouldReportAllPackFilesWhenNoGcWasPerformed() + throws Exception { + packAndPrune(); + long result = gc.getStatistics().numberOfPackFilesAfterBitmap; + + assertEquals(repo.getObjectDatabase().getPacks().size(), result); + } + + @Test + public void testShouldReportNoObjectsDirectlyAfterGc() throws Exception { + // given + addCommit(null); + gc.gc().get(); + assertEquals(1L, repositoryBitmapFiles()); + assertEquals(0L, gc.getStatistics().numberOfPackFilesAfterBitmap); + } + + @Test + public void testShouldReportNewObjectsAfterGcWhenRepositoryProgresses() + throws Exception { + // commit & gc + RevCommit parent = addCommit(null); + gc.gc().get(); + assertEquals(1L, repositoryBitmapFiles()); + + // progress & pack + addCommit(parent); + packAndPrune(); + + assertEquals(1L, gc.getStatistics().numberOfPackFilesAfterBitmap); + } + + @Test + public void testShouldReportNewObjectsFromTheLatestBitmapWhenRepositoryProgresses() + throws Exception { + // commit & gc + RevCommit parent = addCommit(null); + gc.gc().get(); + assertEquals(1L, repositoryBitmapFiles()); + + // progress & gc + parent = addCommit(parent); + gc.gc().get(); + assertEquals(2L, repositoryBitmapFiles()); + + // progress & pack + addCommit(parent); + packAndPrune(); + + assertEquals(1L, gc.getStatistics().numberOfPackFilesAfterBitmap); + } + + private void packAndPrune() throws Exception { + try (SkipNonExistingFilesTestRepository testRepo = new SkipNonExistingFilesTestRepository( + repo)) { + testRepo.packAndPrune(); + } + } + + private RevCommit addCommit(RevCommit parent) throws Exception { + PersonIdent ident = new PersonIdent("repo-metrics", "repo@metrics.com"); + TestRepository.CommitBuilder builder = tr.commit() + .author(ident); + if (parent != null) { + builder.parent(parent); + } + RevCommit commit = builder.create(); + tr.update("master", commit); + parent = commit; + return parent; + } + + private long repositoryBitmapFiles() throws IOException { + return StreamSupport + .stream(Files + .newDirectoryStream(repo.getObjectDatabase() + .getPackDirectory().toPath(), "pack-*.bitmap") + .spliterator(), false) + .count(); + } + + /** + * The TestRepository has a {@link TestRepository#packAndPrune()} function + * but it fails in the last step after GC was performed as it doesn't + * SKIP_MISSING files. In order to circumvent it was copied and improved + * here. + */ + private static class SkipNonExistingFilesTestRepository + extends TestRepository { + private final FileRepository repo; + + private SkipNonExistingFilesTestRepository(FileRepository db) throws IOException { + super(db); + repo = db; + } + + @Override + public void packAndPrune() throws Exception { + ObjectDirectory odb = repo.getObjectDatabase(); + NullProgressMonitor m = NullProgressMonitor.INSTANCE; + + final PackFile pack, idx; + try (PackWriter pw = new PackWriter(repo)) { + Set all = new HashSet<>(); + for (Ref r : repo.getRefDatabase().getRefs()) + all.add(r.getObjectId()); + pw.preparePack(m, all, PackWriter.NONE); + + pack = new PackFile(odb.getPackDirectory(), pw.computeName(), + PackExt.PACK); + try (OutputStream out = new BufferedOutputStream( + new FileOutputStream(pack))) { + pw.writePack(m, m, out); + } + pack.setReadOnly(); + + idx = pack.create(PackExt.INDEX); + try (OutputStream out = new BufferedOutputStream( + new FileOutputStream(idx))) { + pw.writeIndex(out); + } + idx.setReadOnly(); + } + + odb.openPack(pack); + updateServerInfo(); + + // alternative packAndPrune implementation that skips missing files + // after GC. + for (Pack p : odb.getPacks()) { + for (MutableEntry e : p) + FileUtils.delete(odb.fileFor(e.toObjectId()), + FileUtils.SKIP_MISSING); + } + } + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java index 4fafc5a088..34b5ec01e1 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java @@ -1508,6 +1508,12 @@ public class GC { */ public long numberOfPackFiles; + /** + * The number of pack files that were created after the last bitmap + * generation. + */ + public long numberOfPackFilesAfterBitmap; + /** * The number of objects stored as loose objects. */ @@ -1543,6 +1549,8 @@ public class GC { final StringBuilder b = new StringBuilder(); b.append("numberOfPackedObjects=").append(numberOfPackedObjects); //$NON-NLS-1$ b.append(", numberOfPackFiles=").append(numberOfPackFiles); //$NON-NLS-1$ + b.append(", numberOfPackFilesAfterBitmap=") //$NON-NLS-1$ + .append(numberOfPackFilesAfterBitmap); b.append(", numberOfLooseObjects=").append(numberOfLooseObjects); //$NON-NLS-1$ b.append(", numberOfLooseRefs=").append(numberOfLooseRefs); //$NON-NLS-1$ b.append(", numberOfPackedRefs=").append(numberOfPackedRefs); //$NON-NLS-1$ @@ -1569,6 +1577,8 @@ public class GC { ret.sizeOfPackedObjects += p.getPackFile().length(); if (p.getBitmapIndex() != null) ret.numberOfBitmaps += p.getBitmapIndex().getBitmapCount(); + else + ret.numberOfPackFilesAfterBitmap++; } File objDir = repo.getObjectsDirectory(); String[] fanout = objDir.list(); -- cgit v1.2.3 From 52405358a588b6453c7568e7430456826f2ce3b4 Mon Sep 17 00:00:00 2001 From: Sam Delmerico Date: Mon, 7 Oct 2024 15:20:26 -0700 Subject: PackIndexWriter: create interface to write indexes PackWriter assumes that the primary index goes to a file in a well-known format. This cannot accomodate implementations in other storages or formats (e.g. in a database). Create an interface to write the index (PackIndexWriter). This interface will be implemented by the existing pack index writer classes (PackIndexWriterV1 etc.). As the "PackIndexWriter" name was used by the previous superclass of the file writers, we rename that class to "BasePackIndexWriter". Change-Id: Ia7348395315e458fc7adc75a8db5dcb903e2a4a1 --- .../internal/storage/file/AbbreviationTest.java | 1 + .../internal/storage/file/BasePackWriterTest.java | 1079 ++++++++++++++++++++ .../jgit/internal/storage/file/PackWriterTest.java | 1079 -------------------- .../jgit/internal/storage/dfs/DfsInserter.java | 4 +- .../internal/storage/file/BasePackIndexWriter.java | 269 +++++ .../storage/file/ObjectDirectoryPackParser.java | 7 +- .../jgit/internal/storage/file/PackIndex.java | 2 +- .../internal/storage/file/PackIndexWriter.java | 267 ----- .../internal/storage/file/PackIndexWriterV1.java | 4 +- .../internal/storage/file/PackIndexWriterV2.java | 4 +- .../jgit/internal/storage/file/PackInserter.java | 4 +- .../internal/storage/pack/PackIndexWriter.java | 40 + .../jgit/internal/storage/pack/PackWriter.java | 8 +- .../org/eclipse/jgit/storage/pack/PackConfig.java | 6 +- 14 files changed, 1410 insertions(+), 1364 deletions(-) create mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/BasePackWriterTest.java delete mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackWriterTest.java create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/BasePackIndexWriter.java delete mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriter.java create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackIndexWriter.java (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/AbbreviationTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/AbbreviationTest.java index bd36337f35..41a33df0e4 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/AbbreviationTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/AbbreviationTest.java @@ -29,6 +29,7 @@ import java.util.List; import org.eclipse.jgit.errors.AmbiguousObjectException; import org.eclipse.jgit.internal.storage.pack.PackExt; +import org.eclipse.jgit.internal.storage.pack.PackIndexWriter; import org.eclipse.jgit.junit.LocalDiskRepositoryTestCase; import org.eclipse.jgit.junit.TestRepository; import org.eclipse.jgit.lib.AbbreviatedObjectId; diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/BasePackWriterTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/BasePackWriterTest.java new file mode 100644 index 0000000000..92d7465376 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/BasePackWriterTest.java @@ -0,0 +1,1079 @@ +/* + * Copyright (C) 2008, Marek Zawirski and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.storage.file; + +import static org.eclipse.jgit.internal.storage.pack.PackWriter.NONE; +import static org.eclipse.jgit.lib.Constants.INFO_ALTERNATES; +import static org.eclipse.jgit.lib.Constants.OBJ_BLOB; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.errors.MissingObjectException; +import org.eclipse.jgit.internal.storage.file.PackIndex.MutableEntry; +import org.eclipse.jgit.internal.storage.pack.PackExt; +import org.eclipse.jgit.internal.storage.pack.PackWriter; +import org.eclipse.jgit.junit.JGitTestUtil; +import org.eclipse.jgit.junit.TestRepository; +import org.eclipse.jgit.junit.TestRepository.BranchBuilder; +import org.eclipse.jgit.lib.NullProgressMonitor; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectIdSet; +import org.eclipse.jgit.lib.ObjectInserter; +import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.lib.Sets; +import org.eclipse.jgit.revwalk.DepthWalk; +import org.eclipse.jgit.revwalk.ObjectWalk; +import org.eclipse.jgit.revwalk.RevBlob; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevObject; +import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.storage.pack.PackConfig; +import org.eclipse.jgit.storage.pack.PackStatistics; +import org.eclipse.jgit.test.resources.SampleDataRepositoryTestCase; +import org.eclipse.jgit.transport.PackParser; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + +public class BasePackWriterTest extends SampleDataRepositoryTestCase { + + private static final List EMPTY_LIST_REVS = Collections + . emptyList(); + + private static final Set EMPTY_ID_SET = Collections + . emptySet(); + + private PackConfig config; + + private PackWriter writer; + + private ByteArrayOutputStream os; + + private Pack pack; + + private ObjectInserter inserter; + + private FileRepository dst; + + private RevBlob contentA; + + private RevBlob contentB; + + private RevBlob contentC; + + private RevBlob contentD; + + private RevBlob contentE; + + private RevCommit c1; + + private RevCommit c2; + + private RevCommit c3; + + private RevCommit c4; + + private RevCommit c5; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + os = new ByteArrayOutputStream(); + config = new PackConfig(db); + + dst = createBareRepository(); + File alt = new File(dst.getObjectDatabase().getDirectory(), INFO_ALTERNATES); + alt.getParentFile().mkdirs(); + write(alt, db.getObjectDatabase().getDirectory().getAbsolutePath() + "\n"); + } + + @Override + @After + public void tearDown() throws Exception { + if (writer != null) { + writer.close(); + writer = null; + } + if (inserter != null) { + inserter.close(); + inserter = null; + } + super.tearDown(); + } + + /** + * Test constructor for exceptions, default settings, initialization. + * + * @throws IOException + */ + @Test + public void testContructor() throws IOException { + writer = new PackWriter(config, db.newObjectReader()); + assertFalse(writer.isDeltaBaseAsOffset()); + assertTrue(config.isReuseDeltas()); + assertTrue(config.isReuseObjects()); + assertEquals(0, writer.getObjectCount()); + } + + /** + * Change default settings and verify them. + */ + @Test + public void testModifySettings() { + config.setReuseDeltas(false); + config.setReuseObjects(false); + config.setDeltaBaseAsOffset(false); + assertFalse(config.isReuseDeltas()); + assertFalse(config.isReuseObjects()); + assertFalse(config.isDeltaBaseAsOffset()); + + writer = new PackWriter(config, db.newObjectReader()); + writer.setDeltaBaseAsOffset(true); + assertTrue(writer.isDeltaBaseAsOffset()); + assertFalse(config.isDeltaBaseAsOffset()); + } + + /** + * Write empty pack by providing empty sets of interesting/uninteresting + * objects and check for correct format. + * + * @throws IOException + */ + @Test + public void testWriteEmptyPack1() throws IOException { + createVerifyOpenPack(NONE, NONE, false, false); + + assertEquals(0, writer.getObjectCount()); + assertEquals(0, pack.getObjectCount()); + assertEquals("da39a3ee5e6b4b0d3255bfef95601890afd80709", writer + .computeName().name()); + } + + /** + * Write empty pack by providing empty iterator of objects to write and + * check for correct format. + * + * @throws IOException + */ + @Test + public void testWriteEmptyPack2() throws IOException { + createVerifyOpenPack(EMPTY_LIST_REVS); + + assertEquals(0, writer.getObjectCount()); + assertEquals(0, pack.getObjectCount()); + } + + /** + * Try to pass non-existing object as uninteresting, with non-ignoring + * setting. + * + * @throws IOException + */ + @Test + public void testNotIgnoreNonExistingObjects() throws IOException { + final ObjectId nonExisting = ObjectId + .fromString("0000000000000000000000000000000000000001"); + try { + createVerifyOpenPack(NONE, haves(nonExisting), false, false); + fail("Should have thrown MissingObjectException"); + } catch (MissingObjectException x) { + // expected + } + } + + /** + * Try to pass non-existing object as uninteresting, with ignoring setting. + * + * @throws IOException + */ + @Test + public void testIgnoreNonExistingObjects() throws IOException { + final ObjectId nonExisting = ObjectId + .fromString("0000000000000000000000000000000000000001"); + createVerifyOpenPack(NONE, haves(nonExisting), false, true); + // shouldn't throw anything + } + + /** + * Try to pass non-existing object as uninteresting, with ignoring setting. + * Use a repo with bitmap indexes because then PackWriter will use + * PackWriterBitmapWalker which had problems with this situation. + * + * @throws Exception + */ + @Test + public void testIgnoreNonExistingObjectsWithBitmaps() throws Exception { + final ObjectId nonExisting = ObjectId + .fromString("0000000000000000000000000000000000000001"); + new GC(db).gc().get(); + createVerifyOpenPack(NONE, haves(nonExisting), false, true, true); + // shouldn't throw anything + } + + /** + * Create pack basing on only interesting objects, then precisely verify + * content. No delta reuse here. + * + * @throws IOException + */ + @Test + public void testWritePack1() throws IOException { + config.setReuseDeltas(false); + writeVerifyPack1(); + } + + /** + * Test writing pack without object reuse. Pack content/preparation as in + * {@link #testWritePack1()}. + * + * @throws IOException + */ + @Test + public void testWritePack1NoObjectReuse() throws IOException { + config.setReuseDeltas(false); + config.setReuseObjects(false); + writeVerifyPack1(); + } + + /** + * Create pack basing on both interesting and uninteresting objects, then + * precisely verify content. No delta reuse here. + * + * @throws IOException + */ + @Test + public void testWritePack2() throws IOException { + writeVerifyPack2(false); + } + + /** + * Test pack writing with deltas reuse, delta-base first rule. Pack + * content/preparation as in {@link #testWritePack2()}. + * + * @throws IOException + */ + @Test + public void testWritePack2DeltasReuseRefs() throws IOException { + writeVerifyPack2(true); + } + + /** + * Test pack writing with delta reuse. Delta bases referred as offsets. Pack + * configuration as in {@link #testWritePack2DeltasReuseRefs()}. + * + * @throws IOException + */ + @Test + public void testWritePack2DeltasReuseOffsets() throws IOException { + config.setDeltaBaseAsOffset(true); + writeVerifyPack2(true); + } + + /** + * Test pack writing with delta reuse. Raw-data copy (reuse) is made on a + * pack with CRC32 index. Pack configuration as in + * {@link #testWritePack2DeltasReuseRefs()}. + * + * @throws IOException + */ + @Test + public void testWritePack2DeltasCRC32Copy() throws IOException { + final File packDir = db.getObjectDatabase().getPackDirectory(); + final PackFile crc32Pack = new PackFile(packDir, + "pack-34be9032ac282b11fa9babdc2b2a93ca996c9c2f.pack"); + final PackFile crc32Idx = new PackFile(packDir, + "pack-34be9032ac282b11fa9babdc2b2a93ca996c9c2f.idx"); + copyFile(JGitTestUtil.getTestResourceFile( + "pack-34be9032ac282b11fa9babdc2b2a93ca996c9c2f.idxV2"), + crc32Idx); + db.openPack(crc32Pack); + + writeVerifyPack2(true); + } + + /** + * Create pack basing on fixed objects list, then precisely verify content. + * No delta reuse here. + * + * @throws IOException + * @throws MissingObjectException + * + */ + @Test + public void testWritePack3() throws MissingObjectException, IOException { + config.setReuseDeltas(false); + final ObjectId forcedOrder[] = new ObjectId[] { + ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"), + ObjectId.fromString("c59759f143fb1fe21c197981df75a7ee00290799"), + ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035"), + ObjectId.fromString("902d5476fa249b7abc9d84c611577a81381f0327"), + ObjectId.fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3") , + ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259") }; + try (RevWalk parser = new RevWalk(db)) { + final RevObject forcedOrderRevs[] = new RevObject[forcedOrder.length]; + for (int i = 0; i < forcedOrder.length; i++) + forcedOrderRevs[i] = parser.parseAny(forcedOrder[i]); + + createVerifyOpenPack(Arrays.asList(forcedOrderRevs)); + } + + assertEquals(forcedOrder.length, writer.getObjectCount()); + verifyObjectsOrder(forcedOrder); + assertEquals("ed3f96b8327c7c66b0f8f70056129f0769323d86", writer + .computeName().name()); + } + + /** + * Another pack creation: basing on both interesting and uninteresting + * objects. No delta reuse possible here, as this is a specific case when we + * write only 1 commit, associated with 1 tree, 1 blob. + * + * @throws IOException + */ + @Test + public void testWritePack4() throws IOException { + writeVerifyPack4(false); + } + + /** + * Test thin pack writing: 1 blob delta base is on objects edge. Pack + * configuration as in {@link #testWritePack4()}. + * + * @throws IOException + */ + @Test + public void testWritePack4ThinPack() throws IOException { + writeVerifyPack4(true); + } + + /** + * Compare sizes of packs created using {@link #testWritePack2()} and + * {@link #testWritePack2DeltasReuseRefs()}. The pack using deltas should + * be smaller. + * + * @throws Exception + */ + @Test + public void testWritePack2SizeDeltasVsNoDeltas() throws Exception { + config.setReuseDeltas(false); + config.setDeltaCompress(false); + testWritePack2(); + final long sizePack2NoDeltas = os.size(); + tearDown(); + setUp(); + testWritePack2DeltasReuseRefs(); + final long sizePack2DeltasRefs = os.size(); + + assertTrue(sizePack2NoDeltas > sizePack2DeltasRefs); + } + + /** + * Compare sizes of packs created using + * {@link #testWritePack2DeltasReuseRefs()} and + * {@link #testWritePack2DeltasReuseOffsets()}. The pack with delta bases + * written as offsets should be smaller. + * + * @throws Exception + */ + @Test + public void testWritePack2SizeOffsetsVsRefs() throws Exception { + testWritePack2DeltasReuseRefs(); + final long sizePack2DeltasRefs = os.size(); + tearDown(); + setUp(); + testWritePack2DeltasReuseOffsets(); + final long sizePack2DeltasOffsets = os.size(); + + assertTrue(sizePack2DeltasRefs > sizePack2DeltasOffsets); + } + + /** + * Compare sizes of packs created using {@link #testWritePack4()} and + * {@link #testWritePack4ThinPack()}. Obviously, the thin pack should be + * smaller. + * + * @throws Exception + */ + @Test + public void testWritePack4SizeThinVsNoThin() throws Exception { + testWritePack4(); + final long sizePack4 = os.size(); + tearDown(); + setUp(); + testWritePack4ThinPack(); + final long sizePack4Thin = os.size(); + + assertTrue(sizePack4 > sizePack4Thin); + } + + @Test + public void testDeltaStatistics() throws Exception { + config.setDeltaCompress(true); + // TestRepository will close repo + FileRepository repo = createBareRepository(); + ArrayList blobs = new ArrayList<>(); + try (TestRepository testRepo = new TestRepository<>( + repo)) { + blobs.add(testRepo.blob(genDeltableData(1000))); + blobs.add(testRepo.blob(genDeltableData(1005))); + try (PackWriter pw = new PackWriter(repo)) { + NullProgressMonitor m = NullProgressMonitor.INSTANCE; + pw.preparePack(blobs.iterator()); + pw.writePack(m, m, os); + PackStatistics stats = pw.getStatistics(); + assertEquals(1, stats.getTotalDeltas()); + assertTrue("Delta bytes not set.", + stats.byObjectType(OBJ_BLOB).getDeltaBytes() > 0); + } + } + } + + // Generate consistent junk data for building files that delta well + private String genDeltableData(int length) { + assertTrue("Generated data must have a length > 0", length > 0); + char[] data = {'a', 'b', 'c', '\n'}; + StringBuilder builder = new StringBuilder(length); + for (int i = 0; i < length; i++) { + builder.append(data[i % 4]); + } + return builder.toString(); + } + + + @Test + public void testWriteIndex() throws Exception { + config.setIndexVersion(2); + writeVerifyPack4(false); + + PackFile packFile = pack.getPackFile(); + PackFile indexFile = packFile.create(PackExt.INDEX); + + // Validate that IndexPack came up with the right CRC32 value. + final PackIndex idx1 = PackIndex.open(indexFile); + assertTrue(idx1 instanceof PackIndexV2); + assertEquals(0x4743F1E4L, idx1.findCRC32(ObjectId + .fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"))); + + // Validate that an index written by PackWriter is the same. + final File idx2File = new File(indexFile.getAbsolutePath() + ".2"); + try (FileOutputStream is = new FileOutputStream(idx2File)) { + writer.writeIndex(is); + } + final PackIndex idx2 = PackIndex.open(idx2File); + assertTrue(idx2 instanceof PackIndexV2); + assertEquals(idx1.getObjectCount(), idx2.getObjectCount()); + assertEquals(idx1.getOffset64Count(), idx2.getOffset64Count()); + + for (int i = 0; i < idx1.getObjectCount(); i++) { + final ObjectId id = idx1.getObjectId(i); + assertEquals(id, idx2.getObjectId(i)); + assertEquals(idx1.findOffset(id), idx2.findOffset(id)); + assertEquals(idx1.findCRC32(id), idx2.findCRC32(id)); + } + } + + @Test + public void testWriteObjectSizeIndex_noDeltas() throws Exception { + config.setMinBytesForObjSizeIndex(0); + HashSet interesting = new HashSet<>(); + interesting.add(ObjectId + .fromString("82c6b885ff600be425b4ea96dee75dca255b69e7")); + + NullProgressMonitor m1 = NullProgressMonitor.INSTANCE; + writer = new PackWriter(config, db.newObjectReader()); + writer.setUseBitmaps(false); + writer.setThin(false); + writer.setIgnoreMissingUninteresting(false); + writer.preparePack(m1, interesting, NONE); + writer.writePack(m1, m1, os); + + PackIndex idx; + try (ByteArrayOutputStream is = new ByteArrayOutputStream()) { + writer.writeIndex(is); + idx = PackIndex.read(new ByteArrayInputStream(is.toByteArray())); + } + + PackObjectSizeIndex objSizeIdx; + try (ByteArrayOutputStream objSizeStream = new ByteArrayOutputStream()) { + writer.writeObjectSizeIndex(objSizeStream); + objSizeIdx = PackObjectSizeIndexLoader.load( + new ByteArrayInputStream(objSizeStream.toByteArray())); + } + writer.close(); + + ObjectId knownBlob1 = ObjectId + .fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259"); + ObjectId knownBlob2 = ObjectId + .fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3"); + assertEquals(18009, objSizeIdx.getSize(idx.findPosition(knownBlob1))); + assertEquals(18787, objSizeIdx.getSize(idx.findPosition(knownBlob2))); + } + + @Test + public void testWriteReverseIndexConfig() { + assertFalse(config.isWriteReverseIndex()); + config.setWriteReverseIndex(true); + assertTrue(config.isWriteReverseIndex()); + } + + @Test + public void testWriteReverseIndexOff() throws Exception { + config.setWriteReverseIndex(false); + writer = new PackWriter(config, db.newObjectReader()); + ByteArrayOutputStream reverseIndexOutput = new ByteArrayOutputStream(); + + writer.writeReverseIndex(reverseIndexOutput); + + assertEquals(0, reverseIndexOutput.size()); + } + + @Test + public void testWriteReverseIndexOn() throws Exception { + config.setWriteReverseIndex(true); + writeVerifyPack4(false); + ByteArrayOutputStream reverseIndexOutput = new ByteArrayOutputStream(); + int headerBytes = 12; + int bodyBytes = 12; + int footerBytes = 40; + + writer.writeReverseIndex(reverseIndexOutput); + + assertTrue(reverseIndexOutput.size() == headerBytes + bodyBytes + + footerBytes); + } + + @Test + public void testExclude() throws Exception { + // TestRepository closes repo + FileRepository repo = createBareRepository(); + + try (TestRepository testRepo = new TestRepository<>( + repo)) { + BranchBuilder bb = testRepo.branch("refs/heads/master"); + contentA = testRepo.blob("A"); + c1 = bb.commit().add("f", contentA).create(); + testRepo.getRevWalk().parseHeaders(c1); + PackIndex pf1 = writePack(repo, wants(c1), EMPTY_ID_SET); + assertContent(pf1, Arrays.asList(c1.getId(), c1.getTree().getId(), + contentA.getId())); + contentB = testRepo.blob("B"); + c2 = bb.commit().add("f", contentB).create(); + testRepo.getRevWalk().parseHeaders(c2); + PackIndex pf2 = writePack(repo, wants(c2), + Sets.of((ObjectIdSet) pf1)); + assertContent(pf2, Arrays.asList(c2.getId(), c2.getTree().getId(), + contentB.getId())); + } + } + + private static void assertContent(PackIndex pi, List expected) { + assertEquals("Pack index has wrong size.", expected.size(), + pi.getObjectCount()); + for (int i = 0; i < pi.getObjectCount(); i++) + assertTrue( + "Pack index didn't contain the expected id " + + pi.getObjectId(i), + expected.contains(pi.getObjectId(i))); + } + + @Test + public void testShallowIsMinimalDepth1() throws Exception { + try (FileRepository repo = setupRepoForShallowFetch()) { + PackIndex idx = writeShallowPack(repo, 1, wants(c2), NONE, NONE); + assertContent(idx, Arrays.asList(c2.getId(), c2.getTree().getId(), + contentA.getId(), contentB.getId())); + + // Client already has blobs A and B, verify those are not packed. + idx = writeShallowPack(repo, 1, wants(c5), haves(c2), shallows(c2)); + assertContent(idx, Arrays.asList(c5.getId(), c5.getTree().getId(), + contentC.getId(), contentD.getId(), contentE.getId())); + } + } + + @Test + public void testShallowIsMinimalDepth2() throws Exception { + try (FileRepository repo = setupRepoForShallowFetch()) { + PackIndex idx = writeShallowPack(repo, 2, wants(c2), NONE, NONE); + assertContent(idx, + Arrays.asList(c1.getId(), c2.getId(), c1.getTree().getId(), + c2.getTree().getId(), contentA.getId(), + contentB.getId())); + + // Client already has blobs A and B, verify those are not packed. + idx = writeShallowPack(repo, 2, wants(c5), haves(c1, c2), + shallows(c1)); + assertContent(idx, + Arrays.asList(c4.getId(), c5.getId(), c4.getTree().getId(), + c5.getTree().getId(), contentC.getId(), + contentD.getId(), contentE.getId())); + } + } + + @Test + public void testShallowFetchShallowParentDepth1() throws Exception { + try (FileRepository repo = setupRepoForShallowFetch()) { + PackIndex idx = writeShallowPack(repo, 1, wants(c5), NONE, NONE); + assertContent(idx, Arrays.asList(c5.getId(), c5.getTree().getId(), + contentA.getId(), contentB.getId(), contentC.getId(), + contentD.getId(), contentE.getId())); + + idx = writeShallowPack(repo, 1, wants(c4), haves(c5), shallows(c5)); + assertContent(idx, Arrays.asList(c4.getId(), c4.getTree().getId())); + } + } + + @Test + public void testShallowFetchShallowParentDepth2() throws Exception { + try (FileRepository repo = setupRepoForShallowFetch()) { + PackIndex idx = writeShallowPack(repo, 2, wants(c5), NONE, NONE); + assertContent(idx, + Arrays.asList(c4.getId(), c5.getId(), c4.getTree().getId(), + c5.getTree().getId(), contentA.getId(), + contentB.getId(), contentC.getId(), + contentD.getId(), contentE.getId())); + + idx = writeShallowPack(repo, 2, wants(c3), haves(c4, c5), + shallows(c4)); + assertContent(idx, Arrays.asList(c2.getId(), c3.getId(), + c2.getTree().getId(), c3.getTree().getId())); + } + } + + @Test + public void testShallowFetchShallowAncestorDepth1() throws Exception { + try (FileRepository repo = setupRepoForShallowFetch()) { + PackIndex idx = writeShallowPack(repo, 1, wants(c5), NONE, NONE); + assertContent(idx, Arrays.asList(c5.getId(), c5.getTree().getId(), + contentA.getId(), contentB.getId(), contentC.getId(), + contentD.getId(), contentE.getId())); + + idx = writeShallowPack(repo, 1, wants(c3), haves(c5), shallows(c5)); + assertContent(idx, Arrays.asList(c3.getId(), c3.getTree().getId())); + } + } + + @Test + public void testShallowFetchShallowAncestorDepth2() throws Exception { + try (FileRepository repo = setupRepoForShallowFetch()) { + PackIndex idx = writeShallowPack(repo, 2, wants(c5), NONE, NONE); + assertContent(idx, + Arrays.asList(c4.getId(), c5.getId(), c4.getTree().getId(), + c5.getTree().getId(), contentA.getId(), + contentB.getId(), contentC.getId(), + contentD.getId(), contentE.getId())); + + idx = writeShallowPack(repo, 2, wants(c2), haves(c4, c5), + shallows(c4)); + assertContent(idx, Arrays.asList(c1.getId(), c2.getId(), + c1.getTree().getId(), c2.getTree().getId())); + } + } + + @Test + public void testTotalPackFilesScanWhenSearchForReuseTimeoutNotSet() + throws Exception { + FileRepository fileRepository = setUpRepoWithMultiplePackfiles(); + PackWriter mockedPackWriter = Mockito + .spy(new PackWriter(config, fileRepository.newObjectReader())); + + doNothing().when(mockedPackWriter).select(any(), any()); + + try (FileOutputStream packOS = new FileOutputStream( + getPackFileToWrite(fileRepository, mockedPackWriter))) { + mockedPackWriter.writePack(NullProgressMonitor.INSTANCE, + NullProgressMonitor.INSTANCE, packOS); + } + + long numberOfPackFiles = new GC(fileRepository) + .getStatistics().numberOfPackFiles; + int expectedSelectCalls = + // Objects contained in multiple packfiles * number of packfiles + 2 * (int) numberOfPackFiles + + // Objects in single packfile + 1; + verify(mockedPackWriter, times(expectedSelectCalls)).select(any(), + any()); + } + + @Test + public void testTotalPackFilesScanWhenSkippingSearchForReuseTimeoutCheck() + throws Exception { + FileRepository fileRepository = setUpRepoWithMultiplePackfiles(); + PackConfig packConfig = new PackConfig(); + packConfig.setSearchForReuseTimeout(Duration.ofSeconds(-1)); + PackWriter mockedPackWriter = Mockito.spy( + new PackWriter(packConfig, fileRepository.newObjectReader())); + + doNothing().when(mockedPackWriter).select(any(), any()); + + try (FileOutputStream packOS = new FileOutputStream( + getPackFileToWrite(fileRepository, mockedPackWriter))) { + mockedPackWriter.writePack(NullProgressMonitor.INSTANCE, + NullProgressMonitor.INSTANCE, packOS); + } + + long numberOfPackFiles = new GC(fileRepository) + .getStatistics().numberOfPackFiles; + int expectedSelectCalls = + // Objects contained in multiple packfiles * number of packfiles + 2 * (int) numberOfPackFiles + + // Objects contained in single packfile + 1; + verify(mockedPackWriter, times(expectedSelectCalls)).select(any(), + any()); + } + + @Test + public void testPartialPackFilesScanWhenDoingSearchForReuseTimeoutCheck() + throws Exception { + FileRepository fileRepository = setUpRepoWithMultiplePackfiles(); + PackConfig packConfig = new PackConfig(); + packConfig.setSearchForReuseTimeout(Duration.ofSeconds(-1)); + PackWriter mockedPackWriter = Mockito.spy( + new PackWriter(packConfig, fileRepository.newObjectReader())); + mockedPackWriter.enableSearchForReuseTimeout(); + + doNothing().when(mockedPackWriter).select(any(), any()); + + try (FileOutputStream packOS = new FileOutputStream( + getPackFileToWrite(fileRepository, mockedPackWriter))) { + mockedPackWriter.writePack(NullProgressMonitor.INSTANCE, + NullProgressMonitor.INSTANCE, packOS); + } + + int expectedSelectCalls = 3; // Objects in packfiles + verify(mockedPackWriter, times(expectedSelectCalls)).select(any(), + any()); + } + + /** + * Creates objects and packfiles in the following order: + *

    + *
  • Creates 2 objects (C1 = commit, T1 = tree) + *
  • Creates packfile P1 (containing C1, T1) + *
  • Creates 1 object (C2 commit) + *
  • Creates packfile P2 (containing C1, T1, C2) + *
  • Create 1 object (C3 commit) + *
+ * + * @throws Exception + */ + private FileRepository setUpRepoWithMultiplePackfiles() throws Exception { + FileRepository fileRepository = createWorkRepository(); + addRepoToClose(fileRepository); + try (Git git = new Git(fileRepository)) { + // Creates 2 objects (C1 = commit, T1 = tree) + git.commit().setMessage("First commit").call(); + GC gc = new GC(fileRepository); + gc.setPackExpireAgeMillis(Long.MAX_VALUE); + gc.setExpireAgeMillis(Long.MAX_VALUE); + // Creates packfile P1 (containing C1, T1) + gc.gc().get(); + // Creates 1 object (C2 commit) + git.commit().setMessage("Second commit").call(); + // Creates packfile P2 (containing C1, T1, C2) + gc.gc().get(); + // Create 1 object (C3 commit) + git.commit().setMessage("Third commit").call(); + } + return fileRepository; + } + + private PackFile getPackFileToWrite(FileRepository fileRepository, + PackWriter mockedPackWriter) throws IOException { + File packdir = fileRepository.getObjectDatabase().getPackDirectory(); + PackFile packFile = new PackFile(packdir, + mockedPackWriter.computeName(), PackExt.PACK); + + Set all = new HashSet<>(); + for (Ref r : fileRepository.getRefDatabase().getRefs()) { + all.add(r.getObjectId()); + } + + mockedPackWriter.preparePack(NullProgressMonitor.INSTANCE, all, + PackWriter.NONE); + return packFile; + } + + private FileRepository setupRepoForShallowFetch() throws Exception { + FileRepository repo = createBareRepository(); + // TestRepository will close the repo, but we need to return an open + // one! + repo.incrementOpen(); + try (TestRepository r = new TestRepository<>(repo)) { + BranchBuilder bb = r.branch("refs/heads/master"); + contentA = r.blob("A"); + contentB = r.blob("B"); + contentC = r.blob("C"); + contentD = r.blob("D"); + contentE = r.blob("E"); + c1 = bb.commit().add("a", contentA).create(); + c2 = bb.commit().add("b", contentB).create(); + c3 = bb.commit().add("c", contentC).create(); + c4 = bb.commit().add("d", contentD).create(); + c5 = bb.commit().add("e", contentE).create(); + r.getRevWalk().parseHeaders(c5); // fully initialize the tip RevCommit + return repo; + } + } + + private static PackIndex writePack(FileRepository repo, + Set want, Set excludeObjects) + throws IOException { + try (RevWalk walk = new RevWalk(repo)) { + return writePack(repo, walk, 0, want, NONE, excludeObjects); + } + } + + private static PackIndex writeShallowPack(FileRepository repo, int depth, + Set want, Set have, + Set shallow) throws IOException { + // During negotiation, UploadPack would have set up a DepthWalk and + // marked the client's "shallow" commits. Emulate that here. + try (DepthWalk.RevWalk walk = new DepthWalk.RevWalk(repo, depth - 1)) { + walk.assumeShallow(shallow); + return writePack(repo, walk, depth, want, have, EMPTY_ID_SET); + } + } + + private static PackIndex writePack(FileRepository repo, RevWalk walk, + int depth, Set want, + Set have, Set excludeObjects) + throws IOException { + try (PackWriter pw = new PackWriter(repo)) { + pw.setDeltaBaseAsOffset(true); + pw.setReuseDeltaCommits(false); + for (ObjectIdSet idx : excludeObjects) { + pw.excludeObjects(idx); + } + if (depth > 0) { + pw.setShallowPack(depth, null); + } + // ow doesn't need to be closed; caller closes walk. + ObjectWalk ow = walk.toObjectWalkWithSameObjects(); + + pw.preparePack(NullProgressMonitor.INSTANCE, ow, want, have, NONE); + File packdir = repo.getObjectDatabase().getPackDirectory(); + PackFile packFile = new PackFile(packdir, pw.computeName(), + PackExt.PACK); + try (FileOutputStream packOS = new FileOutputStream(packFile)) { + pw.writePack(NullProgressMonitor.INSTANCE, + NullProgressMonitor.INSTANCE, packOS); + } + PackFile idxFile = packFile.create(PackExt.INDEX); + try (FileOutputStream idxOS = new FileOutputStream(idxFile)) { + pw.writeIndex(idxOS); + } + return PackIndex.open(idxFile); + } + } + + // TODO: testWritePackDeltasCycle() + // TODO: testWritePackDeltasDepth() + + private void writeVerifyPack1() throws IOException { + final HashSet interestings = new HashSet<>(); + interestings.add(ObjectId + .fromString("82c6b885ff600be425b4ea96dee75dca255b69e7")); + createVerifyOpenPack(interestings, NONE, false, false); + + final ObjectId expectedOrder[] = new ObjectId[] { + ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"), + ObjectId.fromString("c59759f143fb1fe21c197981df75a7ee00290799"), + ObjectId.fromString("540a36d136cf413e4b064c2b0e0a4db60f77feab"), + ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035"), + ObjectId.fromString("902d5476fa249b7abc9d84c611577a81381f0327"), + ObjectId.fromString("4b825dc642cb6eb9a060e54bf8d69288fbee4904"), + ObjectId.fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3"), + ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259") }; + + assertEquals(expectedOrder.length, writer.getObjectCount()); + verifyObjectsOrder(expectedOrder); + assertEquals("34be9032ac282b11fa9babdc2b2a93ca996c9c2f", writer + .computeName().name()); + } + + private void writeVerifyPack2(boolean deltaReuse) throws IOException { + config.setReuseDeltas(deltaReuse); + final HashSet interestings = new HashSet<>(); + interestings.add(ObjectId + .fromString("82c6b885ff600be425b4ea96dee75dca255b69e7")); + final HashSet uninterestings = new HashSet<>(); + uninterestings.add(ObjectId + .fromString("540a36d136cf413e4b064c2b0e0a4db60f77feab")); + createVerifyOpenPack(interestings, uninterestings, false, false); + + final ObjectId expectedOrder[] = new ObjectId[] { + ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"), + ObjectId.fromString("c59759f143fb1fe21c197981df75a7ee00290799"), + ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035"), + ObjectId.fromString("902d5476fa249b7abc9d84c611577a81381f0327"), + ObjectId.fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3") , + ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259") }; + if (!config.isReuseDeltas() && !config.isDeltaCompress()) { + // If no deltas are in the file the final two entries swap places. + swap(expectedOrder, 4, 5); + } + assertEquals(expectedOrder.length, writer.getObjectCount()); + verifyObjectsOrder(expectedOrder); + assertEquals("ed3f96b8327c7c66b0f8f70056129f0769323d86", writer + .computeName().name()); + } + + private static void swap(ObjectId[] arr, int a, int b) { + ObjectId tmp = arr[a]; + arr[a] = arr[b]; + arr[b] = tmp; + } + + private void writeVerifyPack4(final boolean thin) throws IOException { + final HashSet interestings = new HashSet<>(); + interestings.add(ObjectId + .fromString("82c6b885ff600be425b4ea96dee75dca255b69e7")); + final HashSet uninterestings = new HashSet<>(); + uninterestings.add(ObjectId + .fromString("c59759f143fb1fe21c197981df75a7ee00290799")); + createVerifyOpenPack(interestings, uninterestings, thin, false); + + final ObjectId writtenObjects[] = new ObjectId[] { + ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"), + ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035"), + ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259") }; + assertEquals(writtenObjects.length, writer.getObjectCount()); + ObjectId expectedObjects[]; + if (thin) { + expectedObjects = new ObjectId[4]; + System.arraycopy(writtenObjects, 0, expectedObjects, 0, + writtenObjects.length); + expectedObjects[3] = ObjectId + .fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3"); + + } else { + expectedObjects = writtenObjects; + } + verifyObjectsOrder(expectedObjects); + assertEquals("cded4b74176b4456afa456768b2b5aafb41c44fc", writer + .computeName().name()); + } + + private void createVerifyOpenPack(final Set interestings, + final Set uninterestings, final boolean thin, + final boolean ignoreMissingUninteresting) + throws MissingObjectException, IOException { + createVerifyOpenPack(interestings, uninterestings, thin, + ignoreMissingUninteresting, false); + } + + private void createVerifyOpenPack(final Set interestings, + final Set uninterestings, final boolean thin, + final boolean ignoreMissingUninteresting, boolean useBitmaps) + throws MissingObjectException, IOException { + NullProgressMonitor m = NullProgressMonitor.INSTANCE; + writer = new PackWriter(config, db.newObjectReader()); + writer.setUseBitmaps(useBitmaps); + writer.setThin(thin); + writer.setIgnoreMissingUninteresting(ignoreMissingUninteresting); + writer.preparePack(m, interestings, uninterestings); + writer.writePack(m, m, os); + writer.close(); + verifyOpenPack(thin); + } + + private void createVerifyOpenPack(List objectSource) + throws MissingObjectException, IOException { + NullProgressMonitor m = NullProgressMonitor.INSTANCE; + writer = new PackWriter(config, db.newObjectReader()); + writer.preparePack(objectSource.iterator()); + assertEquals(objectSource.size(), writer.getObjectCount()); + writer.writePack(m, m, os); + writer.close(); + verifyOpenPack(false); + } + + private void verifyOpenPack(boolean thin) throws IOException { + final byte[] packData = os.toByteArray(); + + if (thin) { + PackParser p = index(packData); + try { + p.parse(NullProgressMonitor.INSTANCE); + fail("indexer should grumble about missing object"); + } catch (IOException x) { + // expected + } + } + + ObjectDirectoryPackParser p = (ObjectDirectoryPackParser) index(packData); + p.setKeepEmpty(true); + p.setAllowThin(thin); + p.setIndexVersion(2); + p.parse(NullProgressMonitor.INSTANCE); + pack = p.getPack(); + assertNotNull("have PackFile after parsing", pack); + } + + private PackParser index(byte[] packData) throws IOException { + if (inserter == null) + inserter = dst.newObjectInserter(); + return inserter.newPackParser(new ByteArrayInputStream(packData)); + } + + private void verifyObjectsOrder(ObjectId objectsOrder[]) { + final List entries = new ArrayList<>(); + + for (MutableEntry me : pack) { + entries.add(me.cloneEntry()); + } + Collections.sort(entries, (MutableEntry o1, MutableEntry o2) -> Long + .signum(o1.getOffset() - o2.getOffset())); + + int i = 0; + for (MutableEntry me : entries) { + assertEquals(objectsOrder[i++].toObjectId(), me.toObjectId()); + } + } + + private static Set haves(ObjectId... objects) { + return Sets.of(objects); + } + + private static Set wants(ObjectId... objects) { + return Sets.of(objects); + } + + private static Set shallows(ObjectId... objects) { + return Sets.of(objects); + } +} diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackWriterTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackWriterTest.java deleted file mode 100644 index 24a81b6715..0000000000 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackWriterTest.java +++ /dev/null @@ -1,1079 +0,0 @@ -/* - * Copyright (C) 2008, Marek Zawirski and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.storage.file; - -import static org.eclipse.jgit.internal.storage.pack.PackWriter.NONE; -import static org.eclipse.jgit.lib.Constants.INFO_ALTERNATES; -import static org.eclipse.jgit.lib.Constants.OBJ_BLOB; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.time.Duration; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import org.eclipse.jgit.api.Git; -import org.eclipse.jgit.errors.MissingObjectException; -import org.eclipse.jgit.internal.storage.file.PackIndex.MutableEntry; -import org.eclipse.jgit.internal.storage.pack.PackExt; -import org.eclipse.jgit.internal.storage.pack.PackWriter; -import org.eclipse.jgit.junit.JGitTestUtil; -import org.eclipse.jgit.junit.TestRepository; -import org.eclipse.jgit.junit.TestRepository.BranchBuilder; -import org.eclipse.jgit.lib.NullProgressMonitor; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.ObjectIdSet; -import org.eclipse.jgit.lib.ObjectInserter; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.lib.Sets; -import org.eclipse.jgit.revwalk.DepthWalk; -import org.eclipse.jgit.revwalk.ObjectWalk; -import org.eclipse.jgit.revwalk.RevBlob; -import org.eclipse.jgit.revwalk.RevCommit; -import org.eclipse.jgit.revwalk.RevObject; -import org.eclipse.jgit.revwalk.RevWalk; -import org.eclipse.jgit.storage.pack.PackConfig; -import org.eclipse.jgit.storage.pack.PackStatistics; -import org.eclipse.jgit.test.resources.SampleDataRepositoryTestCase; -import org.eclipse.jgit.transport.PackParser; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.mockito.Mockito; - -public class PackWriterTest extends SampleDataRepositoryTestCase { - - private static final List EMPTY_LIST_REVS = Collections - . emptyList(); - - private static final Set EMPTY_ID_SET = Collections - . emptySet(); - - private PackConfig config; - - private PackWriter writer; - - private ByteArrayOutputStream os; - - private Pack pack; - - private ObjectInserter inserter; - - private FileRepository dst; - - private RevBlob contentA; - - private RevBlob contentB; - - private RevBlob contentC; - - private RevBlob contentD; - - private RevBlob contentE; - - private RevCommit c1; - - private RevCommit c2; - - private RevCommit c3; - - private RevCommit c4; - - private RevCommit c5; - - @Override - @Before - public void setUp() throws Exception { - super.setUp(); - os = new ByteArrayOutputStream(); - config = new PackConfig(db); - - dst = createBareRepository(); - File alt = new File(dst.getObjectDatabase().getDirectory(), INFO_ALTERNATES); - alt.getParentFile().mkdirs(); - write(alt, db.getObjectDatabase().getDirectory().getAbsolutePath() + "\n"); - } - - @Override - @After - public void tearDown() throws Exception { - if (writer != null) { - writer.close(); - writer = null; - } - if (inserter != null) { - inserter.close(); - inserter = null; - } - super.tearDown(); - } - - /** - * Test constructor for exceptions, default settings, initialization. - * - * @throws IOException - */ - @Test - public void testContructor() throws IOException { - writer = new PackWriter(config, db.newObjectReader()); - assertFalse(writer.isDeltaBaseAsOffset()); - assertTrue(config.isReuseDeltas()); - assertTrue(config.isReuseObjects()); - assertEquals(0, writer.getObjectCount()); - } - - /** - * Change default settings and verify them. - */ - @Test - public void testModifySettings() { - config.setReuseDeltas(false); - config.setReuseObjects(false); - config.setDeltaBaseAsOffset(false); - assertFalse(config.isReuseDeltas()); - assertFalse(config.isReuseObjects()); - assertFalse(config.isDeltaBaseAsOffset()); - - writer = new PackWriter(config, db.newObjectReader()); - writer.setDeltaBaseAsOffset(true); - assertTrue(writer.isDeltaBaseAsOffset()); - assertFalse(config.isDeltaBaseAsOffset()); - } - - /** - * Write empty pack by providing empty sets of interesting/uninteresting - * objects and check for correct format. - * - * @throws IOException - */ - @Test - public void testWriteEmptyPack1() throws IOException { - createVerifyOpenPack(NONE, NONE, false, false); - - assertEquals(0, writer.getObjectCount()); - assertEquals(0, pack.getObjectCount()); - assertEquals("da39a3ee5e6b4b0d3255bfef95601890afd80709", writer - .computeName().name()); - } - - /** - * Write empty pack by providing empty iterator of objects to write and - * check for correct format. - * - * @throws IOException - */ - @Test - public void testWriteEmptyPack2() throws IOException { - createVerifyOpenPack(EMPTY_LIST_REVS); - - assertEquals(0, writer.getObjectCount()); - assertEquals(0, pack.getObjectCount()); - } - - /** - * Try to pass non-existing object as uninteresting, with non-ignoring - * setting. - * - * @throws IOException - */ - @Test - public void testNotIgnoreNonExistingObjects() throws IOException { - final ObjectId nonExisting = ObjectId - .fromString("0000000000000000000000000000000000000001"); - try { - createVerifyOpenPack(NONE, haves(nonExisting), false, false); - fail("Should have thrown MissingObjectException"); - } catch (MissingObjectException x) { - // expected - } - } - - /** - * Try to pass non-existing object as uninteresting, with ignoring setting. - * - * @throws IOException - */ - @Test - public void testIgnoreNonExistingObjects() throws IOException { - final ObjectId nonExisting = ObjectId - .fromString("0000000000000000000000000000000000000001"); - createVerifyOpenPack(NONE, haves(nonExisting), false, true); - // shouldn't throw anything - } - - /** - * Try to pass non-existing object as uninteresting, with ignoring setting. - * Use a repo with bitmap indexes because then PackWriter will use - * PackWriterBitmapWalker which had problems with this situation. - * - * @throws Exception - */ - @Test - public void testIgnoreNonExistingObjectsWithBitmaps() throws Exception { - final ObjectId nonExisting = ObjectId - .fromString("0000000000000000000000000000000000000001"); - new GC(db).gc().get(); - createVerifyOpenPack(NONE, haves(nonExisting), false, true, true); - // shouldn't throw anything - } - - /** - * Create pack basing on only interesting objects, then precisely verify - * content. No delta reuse here. - * - * @throws IOException - */ - @Test - public void testWritePack1() throws IOException { - config.setReuseDeltas(false); - writeVerifyPack1(); - } - - /** - * Test writing pack without object reuse. Pack content/preparation as in - * {@link #testWritePack1()}. - * - * @throws IOException - */ - @Test - public void testWritePack1NoObjectReuse() throws IOException { - config.setReuseDeltas(false); - config.setReuseObjects(false); - writeVerifyPack1(); - } - - /** - * Create pack basing on both interesting and uninteresting objects, then - * precisely verify content. No delta reuse here. - * - * @throws IOException - */ - @Test - public void testWritePack2() throws IOException { - writeVerifyPack2(false); - } - - /** - * Test pack writing with deltas reuse, delta-base first rule. Pack - * content/preparation as in {@link #testWritePack2()}. - * - * @throws IOException - */ - @Test - public void testWritePack2DeltasReuseRefs() throws IOException { - writeVerifyPack2(true); - } - - /** - * Test pack writing with delta reuse. Delta bases referred as offsets. Pack - * configuration as in {@link #testWritePack2DeltasReuseRefs()}. - * - * @throws IOException - */ - @Test - public void testWritePack2DeltasReuseOffsets() throws IOException { - config.setDeltaBaseAsOffset(true); - writeVerifyPack2(true); - } - - /** - * Test pack writing with delta reuse. Raw-data copy (reuse) is made on a - * pack with CRC32 index. Pack configuration as in - * {@link #testWritePack2DeltasReuseRefs()}. - * - * @throws IOException - */ - @Test - public void testWritePack2DeltasCRC32Copy() throws IOException { - final File packDir = db.getObjectDatabase().getPackDirectory(); - final PackFile crc32Pack = new PackFile(packDir, - "pack-34be9032ac282b11fa9babdc2b2a93ca996c9c2f.pack"); - final PackFile crc32Idx = new PackFile(packDir, - "pack-34be9032ac282b11fa9babdc2b2a93ca996c9c2f.idx"); - copyFile(JGitTestUtil.getTestResourceFile( - "pack-34be9032ac282b11fa9babdc2b2a93ca996c9c2f.idxV2"), - crc32Idx); - db.openPack(crc32Pack); - - writeVerifyPack2(true); - } - - /** - * Create pack basing on fixed objects list, then precisely verify content. - * No delta reuse here. - * - * @throws IOException - * @throws MissingObjectException - * - */ - @Test - public void testWritePack3() throws MissingObjectException, IOException { - config.setReuseDeltas(false); - final ObjectId forcedOrder[] = new ObjectId[] { - ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"), - ObjectId.fromString("c59759f143fb1fe21c197981df75a7ee00290799"), - ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035"), - ObjectId.fromString("902d5476fa249b7abc9d84c611577a81381f0327"), - ObjectId.fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3") , - ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259") }; - try (RevWalk parser = new RevWalk(db)) { - final RevObject forcedOrderRevs[] = new RevObject[forcedOrder.length]; - for (int i = 0; i < forcedOrder.length; i++) - forcedOrderRevs[i] = parser.parseAny(forcedOrder[i]); - - createVerifyOpenPack(Arrays.asList(forcedOrderRevs)); - } - - assertEquals(forcedOrder.length, writer.getObjectCount()); - verifyObjectsOrder(forcedOrder); - assertEquals("ed3f96b8327c7c66b0f8f70056129f0769323d86", writer - .computeName().name()); - } - - /** - * Another pack creation: basing on both interesting and uninteresting - * objects. No delta reuse possible here, as this is a specific case when we - * write only 1 commit, associated with 1 tree, 1 blob. - * - * @throws IOException - */ - @Test - public void testWritePack4() throws IOException { - writeVerifyPack4(false); - } - - /** - * Test thin pack writing: 1 blob delta base is on objects edge. Pack - * configuration as in {@link #testWritePack4()}. - * - * @throws IOException - */ - @Test - public void testWritePack4ThinPack() throws IOException { - writeVerifyPack4(true); - } - - /** - * Compare sizes of packs created using {@link #testWritePack2()} and - * {@link #testWritePack2DeltasReuseRefs()}. The pack using deltas should - * be smaller. - * - * @throws Exception - */ - @Test - public void testWritePack2SizeDeltasVsNoDeltas() throws Exception { - config.setReuseDeltas(false); - config.setDeltaCompress(false); - testWritePack2(); - final long sizePack2NoDeltas = os.size(); - tearDown(); - setUp(); - testWritePack2DeltasReuseRefs(); - final long sizePack2DeltasRefs = os.size(); - - assertTrue(sizePack2NoDeltas > sizePack2DeltasRefs); - } - - /** - * Compare sizes of packs created using - * {@link #testWritePack2DeltasReuseRefs()} and - * {@link #testWritePack2DeltasReuseOffsets()}. The pack with delta bases - * written as offsets should be smaller. - * - * @throws Exception - */ - @Test - public void testWritePack2SizeOffsetsVsRefs() throws Exception { - testWritePack2DeltasReuseRefs(); - final long sizePack2DeltasRefs = os.size(); - tearDown(); - setUp(); - testWritePack2DeltasReuseOffsets(); - final long sizePack2DeltasOffsets = os.size(); - - assertTrue(sizePack2DeltasRefs > sizePack2DeltasOffsets); - } - - /** - * Compare sizes of packs created using {@link #testWritePack4()} and - * {@link #testWritePack4ThinPack()}. Obviously, the thin pack should be - * smaller. - * - * @throws Exception - */ - @Test - public void testWritePack4SizeThinVsNoThin() throws Exception { - testWritePack4(); - final long sizePack4 = os.size(); - tearDown(); - setUp(); - testWritePack4ThinPack(); - final long sizePack4Thin = os.size(); - - assertTrue(sizePack4 > sizePack4Thin); - } - - @Test - public void testDeltaStatistics() throws Exception { - config.setDeltaCompress(true); - // TestRepository will close repo - FileRepository repo = createBareRepository(); - ArrayList blobs = new ArrayList<>(); - try (TestRepository testRepo = new TestRepository<>( - repo)) { - blobs.add(testRepo.blob(genDeltableData(1000))); - blobs.add(testRepo.blob(genDeltableData(1005))); - try (PackWriter pw = new PackWriter(repo)) { - NullProgressMonitor m = NullProgressMonitor.INSTANCE; - pw.preparePack(blobs.iterator()); - pw.writePack(m, m, os); - PackStatistics stats = pw.getStatistics(); - assertEquals(1, stats.getTotalDeltas()); - assertTrue("Delta bytes not set.", - stats.byObjectType(OBJ_BLOB).getDeltaBytes() > 0); - } - } - } - - // Generate consistent junk data for building files that delta well - private String genDeltableData(int length) { - assertTrue("Generated data must have a length > 0", length > 0); - char[] data = {'a', 'b', 'c', '\n'}; - StringBuilder builder = new StringBuilder(length); - for (int i = 0; i < length; i++) { - builder.append(data[i % 4]); - } - return builder.toString(); - } - - - @Test - public void testWriteIndex() throws Exception { - config.setIndexVersion(2); - writeVerifyPack4(false); - - PackFile packFile = pack.getPackFile(); - PackFile indexFile = packFile.create(PackExt.INDEX); - - // Validate that IndexPack came up with the right CRC32 value. - final PackIndex idx1 = PackIndex.open(indexFile); - assertTrue(idx1 instanceof PackIndexV2); - assertEquals(0x4743F1E4L, idx1.findCRC32(ObjectId - .fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"))); - - // Validate that an index written by PackWriter is the same. - final File idx2File = new File(indexFile.getAbsolutePath() + ".2"); - try (FileOutputStream is = new FileOutputStream(idx2File)) { - writer.writeIndex(is); - } - final PackIndex idx2 = PackIndex.open(idx2File); - assertTrue(idx2 instanceof PackIndexV2); - assertEquals(idx1.getObjectCount(), idx2.getObjectCount()); - assertEquals(idx1.getOffset64Count(), idx2.getOffset64Count()); - - for (int i = 0; i < idx1.getObjectCount(); i++) { - final ObjectId id = idx1.getObjectId(i); - assertEquals(id, idx2.getObjectId(i)); - assertEquals(idx1.findOffset(id), idx2.findOffset(id)); - assertEquals(idx1.findCRC32(id), idx2.findCRC32(id)); - } - } - - @Test - public void testWriteObjectSizeIndex_noDeltas() throws Exception { - config.setMinBytesForObjSizeIndex(0); - HashSet interesting = new HashSet<>(); - interesting.add(ObjectId - .fromString("82c6b885ff600be425b4ea96dee75dca255b69e7")); - - NullProgressMonitor m1 = NullProgressMonitor.INSTANCE; - writer = new PackWriter(config, db.newObjectReader()); - writer.setUseBitmaps(false); - writer.setThin(false); - writer.setIgnoreMissingUninteresting(false); - writer.preparePack(m1, interesting, NONE); - writer.writePack(m1, m1, os); - - PackIndex idx; - try (ByteArrayOutputStream is = new ByteArrayOutputStream()) { - writer.writeIndex(is); - idx = PackIndex.read(new ByteArrayInputStream(is.toByteArray())); - } - - PackObjectSizeIndex objSizeIdx; - try (ByteArrayOutputStream objSizeStream = new ByteArrayOutputStream()) { - writer.writeObjectSizeIndex(objSizeStream); - objSizeIdx = PackObjectSizeIndexLoader.load( - new ByteArrayInputStream(objSizeStream.toByteArray())); - } - writer.close(); - - ObjectId knownBlob1 = ObjectId - .fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259"); - ObjectId knownBlob2 = ObjectId - .fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3"); - assertEquals(18009, objSizeIdx.getSize(idx.findPosition(knownBlob1))); - assertEquals(18787, objSizeIdx.getSize(idx.findPosition(knownBlob2))); - } - - @Test - public void testWriteReverseIndexConfig() { - assertFalse(config.isWriteReverseIndex()); - config.setWriteReverseIndex(true); - assertTrue(config.isWriteReverseIndex()); - } - - @Test - public void testWriteReverseIndexOff() throws Exception { - config.setWriteReverseIndex(false); - writer = new PackWriter(config, db.newObjectReader()); - ByteArrayOutputStream reverseIndexOutput = new ByteArrayOutputStream(); - - writer.writeReverseIndex(reverseIndexOutput); - - assertEquals(0, reverseIndexOutput.size()); - } - - @Test - public void testWriteReverseIndexOn() throws Exception { - config.setWriteReverseIndex(true); - writeVerifyPack4(false); - ByteArrayOutputStream reverseIndexOutput = new ByteArrayOutputStream(); - int headerBytes = 12; - int bodyBytes = 12; - int footerBytes = 40; - - writer.writeReverseIndex(reverseIndexOutput); - - assertTrue(reverseIndexOutput.size() == headerBytes + bodyBytes - + footerBytes); - } - - @Test - public void testExclude() throws Exception { - // TestRepository closes repo - FileRepository repo = createBareRepository(); - - try (TestRepository testRepo = new TestRepository<>( - repo)) { - BranchBuilder bb = testRepo.branch("refs/heads/master"); - contentA = testRepo.blob("A"); - c1 = bb.commit().add("f", contentA).create(); - testRepo.getRevWalk().parseHeaders(c1); - PackIndex pf1 = writePack(repo, wants(c1), EMPTY_ID_SET); - assertContent(pf1, Arrays.asList(c1.getId(), c1.getTree().getId(), - contentA.getId())); - contentB = testRepo.blob("B"); - c2 = bb.commit().add("f", contentB).create(); - testRepo.getRevWalk().parseHeaders(c2); - PackIndex pf2 = writePack(repo, wants(c2), - Sets.of((ObjectIdSet) pf1)); - assertContent(pf2, Arrays.asList(c2.getId(), c2.getTree().getId(), - contentB.getId())); - } - } - - private static void assertContent(PackIndex pi, List expected) { - assertEquals("Pack index has wrong size.", expected.size(), - pi.getObjectCount()); - for (int i = 0; i < pi.getObjectCount(); i++) - assertTrue( - "Pack index didn't contain the expected id " - + pi.getObjectId(i), - expected.contains(pi.getObjectId(i))); - } - - @Test - public void testShallowIsMinimalDepth1() throws Exception { - try (FileRepository repo = setupRepoForShallowFetch()) { - PackIndex idx = writeShallowPack(repo, 1, wants(c2), NONE, NONE); - assertContent(idx, Arrays.asList(c2.getId(), c2.getTree().getId(), - contentA.getId(), contentB.getId())); - - // Client already has blobs A and B, verify those are not packed. - idx = writeShallowPack(repo, 1, wants(c5), haves(c2), shallows(c2)); - assertContent(idx, Arrays.asList(c5.getId(), c5.getTree().getId(), - contentC.getId(), contentD.getId(), contentE.getId())); - } - } - - @Test - public void testShallowIsMinimalDepth2() throws Exception { - try (FileRepository repo = setupRepoForShallowFetch()) { - PackIndex idx = writeShallowPack(repo, 2, wants(c2), NONE, NONE); - assertContent(idx, - Arrays.asList(c1.getId(), c2.getId(), c1.getTree().getId(), - c2.getTree().getId(), contentA.getId(), - contentB.getId())); - - // Client already has blobs A and B, verify those are not packed. - idx = writeShallowPack(repo, 2, wants(c5), haves(c1, c2), - shallows(c1)); - assertContent(idx, - Arrays.asList(c4.getId(), c5.getId(), c4.getTree().getId(), - c5.getTree().getId(), contentC.getId(), - contentD.getId(), contentE.getId())); - } - } - - @Test - public void testShallowFetchShallowParentDepth1() throws Exception { - try (FileRepository repo = setupRepoForShallowFetch()) { - PackIndex idx = writeShallowPack(repo, 1, wants(c5), NONE, NONE); - assertContent(idx, Arrays.asList(c5.getId(), c5.getTree().getId(), - contentA.getId(), contentB.getId(), contentC.getId(), - contentD.getId(), contentE.getId())); - - idx = writeShallowPack(repo, 1, wants(c4), haves(c5), shallows(c5)); - assertContent(idx, Arrays.asList(c4.getId(), c4.getTree().getId())); - } - } - - @Test - public void testShallowFetchShallowParentDepth2() throws Exception { - try (FileRepository repo = setupRepoForShallowFetch()) { - PackIndex idx = writeShallowPack(repo, 2, wants(c5), NONE, NONE); - assertContent(idx, - Arrays.asList(c4.getId(), c5.getId(), c4.getTree().getId(), - c5.getTree().getId(), contentA.getId(), - contentB.getId(), contentC.getId(), - contentD.getId(), contentE.getId())); - - idx = writeShallowPack(repo, 2, wants(c3), haves(c4, c5), - shallows(c4)); - assertContent(idx, Arrays.asList(c2.getId(), c3.getId(), - c2.getTree().getId(), c3.getTree().getId())); - } - } - - @Test - public void testShallowFetchShallowAncestorDepth1() throws Exception { - try (FileRepository repo = setupRepoForShallowFetch()) { - PackIndex idx = writeShallowPack(repo, 1, wants(c5), NONE, NONE); - assertContent(idx, Arrays.asList(c5.getId(), c5.getTree().getId(), - contentA.getId(), contentB.getId(), contentC.getId(), - contentD.getId(), contentE.getId())); - - idx = writeShallowPack(repo, 1, wants(c3), haves(c5), shallows(c5)); - assertContent(idx, Arrays.asList(c3.getId(), c3.getTree().getId())); - } - } - - @Test - public void testShallowFetchShallowAncestorDepth2() throws Exception { - try (FileRepository repo = setupRepoForShallowFetch()) { - PackIndex idx = writeShallowPack(repo, 2, wants(c5), NONE, NONE); - assertContent(idx, - Arrays.asList(c4.getId(), c5.getId(), c4.getTree().getId(), - c5.getTree().getId(), contentA.getId(), - contentB.getId(), contentC.getId(), - contentD.getId(), contentE.getId())); - - idx = writeShallowPack(repo, 2, wants(c2), haves(c4, c5), - shallows(c4)); - assertContent(idx, Arrays.asList(c1.getId(), c2.getId(), - c1.getTree().getId(), c2.getTree().getId())); - } - } - - @Test - public void testTotalPackFilesScanWhenSearchForReuseTimeoutNotSet() - throws Exception { - FileRepository fileRepository = setUpRepoWithMultiplePackfiles(); - PackWriter mockedPackWriter = Mockito - .spy(new PackWriter(config, fileRepository.newObjectReader())); - - doNothing().when(mockedPackWriter).select(any(), any()); - - try (FileOutputStream packOS = new FileOutputStream( - getPackFileToWrite(fileRepository, mockedPackWriter))) { - mockedPackWriter.writePack(NullProgressMonitor.INSTANCE, - NullProgressMonitor.INSTANCE, packOS); - } - - long numberOfPackFiles = new GC(fileRepository) - .getStatistics().numberOfPackFiles; - int expectedSelectCalls = - // Objects contained in multiple packfiles * number of packfiles - 2 * (int) numberOfPackFiles + - // Objects in single packfile - 1; - verify(mockedPackWriter, times(expectedSelectCalls)).select(any(), - any()); - } - - @Test - public void testTotalPackFilesScanWhenSkippingSearchForReuseTimeoutCheck() - throws Exception { - FileRepository fileRepository = setUpRepoWithMultiplePackfiles(); - PackConfig packConfig = new PackConfig(); - packConfig.setSearchForReuseTimeout(Duration.ofSeconds(-1)); - PackWriter mockedPackWriter = Mockito.spy( - new PackWriter(packConfig, fileRepository.newObjectReader())); - - doNothing().when(mockedPackWriter).select(any(), any()); - - try (FileOutputStream packOS = new FileOutputStream( - getPackFileToWrite(fileRepository, mockedPackWriter))) { - mockedPackWriter.writePack(NullProgressMonitor.INSTANCE, - NullProgressMonitor.INSTANCE, packOS); - } - - long numberOfPackFiles = new GC(fileRepository) - .getStatistics().numberOfPackFiles; - int expectedSelectCalls = - // Objects contained in multiple packfiles * number of packfiles - 2 * (int) numberOfPackFiles + - // Objects contained in single packfile - 1; - verify(mockedPackWriter, times(expectedSelectCalls)).select(any(), - any()); - } - - @Test - public void testPartialPackFilesScanWhenDoingSearchForReuseTimeoutCheck() - throws Exception { - FileRepository fileRepository = setUpRepoWithMultiplePackfiles(); - PackConfig packConfig = new PackConfig(); - packConfig.setSearchForReuseTimeout(Duration.ofSeconds(-1)); - PackWriter mockedPackWriter = Mockito.spy( - new PackWriter(packConfig, fileRepository.newObjectReader())); - mockedPackWriter.enableSearchForReuseTimeout(); - - doNothing().when(mockedPackWriter).select(any(), any()); - - try (FileOutputStream packOS = new FileOutputStream( - getPackFileToWrite(fileRepository, mockedPackWriter))) { - mockedPackWriter.writePack(NullProgressMonitor.INSTANCE, - NullProgressMonitor.INSTANCE, packOS); - } - - int expectedSelectCalls = 3; // Objects in packfiles - verify(mockedPackWriter, times(expectedSelectCalls)).select(any(), - any()); - } - - /** - * Creates objects and packfiles in the following order: - *
    - *
  • Creates 2 objects (C1 = commit, T1 = tree) - *
  • Creates packfile P1 (containing C1, T1) - *
  • Creates 1 object (C2 commit) - *
  • Creates packfile P2 (containing C1, T1, C2) - *
  • Create 1 object (C3 commit) - *
- * - * @throws Exception - */ - private FileRepository setUpRepoWithMultiplePackfiles() throws Exception { - FileRepository fileRepository = createWorkRepository(); - addRepoToClose(fileRepository); - try (Git git = new Git(fileRepository)) { - // Creates 2 objects (C1 = commit, T1 = tree) - git.commit().setMessage("First commit").call(); - GC gc = new GC(fileRepository); - gc.setPackExpireAgeMillis(Long.MAX_VALUE); - gc.setExpireAgeMillis(Long.MAX_VALUE); - // Creates packfile P1 (containing C1, T1) - gc.gc().get(); - // Creates 1 object (C2 commit) - git.commit().setMessage("Second commit").call(); - // Creates packfile P2 (containing C1, T1, C2) - gc.gc().get(); - // Create 1 object (C3 commit) - git.commit().setMessage("Third commit").call(); - } - return fileRepository; - } - - private PackFile getPackFileToWrite(FileRepository fileRepository, - PackWriter mockedPackWriter) throws IOException { - File packdir = fileRepository.getObjectDatabase().getPackDirectory(); - PackFile packFile = new PackFile(packdir, - mockedPackWriter.computeName(), PackExt.PACK); - - Set all = new HashSet<>(); - for (Ref r : fileRepository.getRefDatabase().getRefs()) { - all.add(r.getObjectId()); - } - - mockedPackWriter.preparePack(NullProgressMonitor.INSTANCE, all, - PackWriter.NONE); - return packFile; - } - - private FileRepository setupRepoForShallowFetch() throws Exception { - FileRepository repo = createBareRepository(); - // TestRepository will close the repo, but we need to return an open - // one! - repo.incrementOpen(); - try (TestRepository r = new TestRepository<>(repo)) { - BranchBuilder bb = r.branch("refs/heads/master"); - contentA = r.blob("A"); - contentB = r.blob("B"); - contentC = r.blob("C"); - contentD = r.blob("D"); - contentE = r.blob("E"); - c1 = bb.commit().add("a", contentA).create(); - c2 = bb.commit().add("b", contentB).create(); - c3 = bb.commit().add("c", contentC).create(); - c4 = bb.commit().add("d", contentD).create(); - c5 = bb.commit().add("e", contentE).create(); - r.getRevWalk().parseHeaders(c5); // fully initialize the tip RevCommit - return repo; - } - } - - private static PackIndex writePack(FileRepository repo, - Set want, Set excludeObjects) - throws IOException { - try (RevWalk walk = new RevWalk(repo)) { - return writePack(repo, walk, 0, want, NONE, excludeObjects); - } - } - - private static PackIndex writeShallowPack(FileRepository repo, int depth, - Set want, Set have, - Set shallow) throws IOException { - // During negotiation, UploadPack would have set up a DepthWalk and - // marked the client's "shallow" commits. Emulate that here. - try (DepthWalk.RevWalk walk = new DepthWalk.RevWalk(repo, depth - 1)) { - walk.assumeShallow(shallow); - return writePack(repo, walk, depth, want, have, EMPTY_ID_SET); - } - } - - private static PackIndex writePack(FileRepository repo, RevWalk walk, - int depth, Set want, - Set have, Set excludeObjects) - throws IOException { - try (PackWriter pw = new PackWriter(repo)) { - pw.setDeltaBaseAsOffset(true); - pw.setReuseDeltaCommits(false); - for (ObjectIdSet idx : excludeObjects) { - pw.excludeObjects(idx); - } - if (depth > 0) { - pw.setShallowPack(depth, null); - } - // ow doesn't need to be closed; caller closes walk. - ObjectWalk ow = walk.toObjectWalkWithSameObjects(); - - pw.preparePack(NullProgressMonitor.INSTANCE, ow, want, have, NONE); - File packdir = repo.getObjectDatabase().getPackDirectory(); - PackFile packFile = new PackFile(packdir, pw.computeName(), - PackExt.PACK); - try (FileOutputStream packOS = new FileOutputStream(packFile)) { - pw.writePack(NullProgressMonitor.INSTANCE, - NullProgressMonitor.INSTANCE, packOS); - } - PackFile idxFile = packFile.create(PackExt.INDEX); - try (FileOutputStream idxOS = new FileOutputStream(idxFile)) { - pw.writeIndex(idxOS); - } - return PackIndex.open(idxFile); - } - } - - // TODO: testWritePackDeltasCycle() - // TODO: testWritePackDeltasDepth() - - private void writeVerifyPack1() throws IOException { - final HashSet interestings = new HashSet<>(); - interestings.add(ObjectId - .fromString("82c6b885ff600be425b4ea96dee75dca255b69e7")); - createVerifyOpenPack(interestings, NONE, false, false); - - final ObjectId expectedOrder[] = new ObjectId[] { - ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"), - ObjectId.fromString("c59759f143fb1fe21c197981df75a7ee00290799"), - ObjectId.fromString("540a36d136cf413e4b064c2b0e0a4db60f77feab"), - ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035"), - ObjectId.fromString("902d5476fa249b7abc9d84c611577a81381f0327"), - ObjectId.fromString("4b825dc642cb6eb9a060e54bf8d69288fbee4904"), - ObjectId.fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3"), - ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259") }; - - assertEquals(expectedOrder.length, writer.getObjectCount()); - verifyObjectsOrder(expectedOrder); - assertEquals("34be9032ac282b11fa9babdc2b2a93ca996c9c2f", writer - .computeName().name()); - } - - private void writeVerifyPack2(boolean deltaReuse) throws IOException { - config.setReuseDeltas(deltaReuse); - final HashSet interestings = new HashSet<>(); - interestings.add(ObjectId - .fromString("82c6b885ff600be425b4ea96dee75dca255b69e7")); - final HashSet uninterestings = new HashSet<>(); - uninterestings.add(ObjectId - .fromString("540a36d136cf413e4b064c2b0e0a4db60f77feab")); - createVerifyOpenPack(interestings, uninterestings, false, false); - - final ObjectId expectedOrder[] = new ObjectId[] { - ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"), - ObjectId.fromString("c59759f143fb1fe21c197981df75a7ee00290799"), - ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035"), - ObjectId.fromString("902d5476fa249b7abc9d84c611577a81381f0327"), - ObjectId.fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3") , - ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259") }; - if (!config.isReuseDeltas() && !config.isDeltaCompress()) { - // If no deltas are in the file the final two entries swap places. - swap(expectedOrder, 4, 5); - } - assertEquals(expectedOrder.length, writer.getObjectCount()); - verifyObjectsOrder(expectedOrder); - assertEquals("ed3f96b8327c7c66b0f8f70056129f0769323d86", writer - .computeName().name()); - } - - private static void swap(ObjectId[] arr, int a, int b) { - ObjectId tmp = arr[a]; - arr[a] = arr[b]; - arr[b] = tmp; - } - - private void writeVerifyPack4(final boolean thin) throws IOException { - final HashSet interestings = new HashSet<>(); - interestings.add(ObjectId - .fromString("82c6b885ff600be425b4ea96dee75dca255b69e7")); - final HashSet uninterestings = new HashSet<>(); - uninterestings.add(ObjectId - .fromString("c59759f143fb1fe21c197981df75a7ee00290799")); - createVerifyOpenPack(interestings, uninterestings, thin, false); - - final ObjectId writtenObjects[] = new ObjectId[] { - ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"), - ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035"), - ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259") }; - assertEquals(writtenObjects.length, writer.getObjectCount()); - ObjectId expectedObjects[]; - if (thin) { - expectedObjects = new ObjectId[4]; - System.arraycopy(writtenObjects, 0, expectedObjects, 0, - writtenObjects.length); - expectedObjects[3] = ObjectId - .fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3"); - - } else { - expectedObjects = writtenObjects; - } - verifyObjectsOrder(expectedObjects); - assertEquals("cded4b74176b4456afa456768b2b5aafb41c44fc", writer - .computeName().name()); - } - - private void createVerifyOpenPack(final Set interestings, - final Set uninterestings, final boolean thin, - final boolean ignoreMissingUninteresting) - throws MissingObjectException, IOException { - createVerifyOpenPack(interestings, uninterestings, thin, - ignoreMissingUninteresting, false); - } - - private void createVerifyOpenPack(final Set interestings, - final Set uninterestings, final boolean thin, - final boolean ignoreMissingUninteresting, boolean useBitmaps) - throws MissingObjectException, IOException { - NullProgressMonitor m = NullProgressMonitor.INSTANCE; - writer = new PackWriter(config, db.newObjectReader()); - writer.setUseBitmaps(useBitmaps); - writer.setThin(thin); - writer.setIgnoreMissingUninteresting(ignoreMissingUninteresting); - writer.preparePack(m, interestings, uninterestings); - writer.writePack(m, m, os); - writer.close(); - verifyOpenPack(thin); - } - - private void createVerifyOpenPack(List objectSource) - throws MissingObjectException, IOException { - NullProgressMonitor m = NullProgressMonitor.INSTANCE; - writer = new PackWriter(config, db.newObjectReader()); - writer.preparePack(objectSource.iterator()); - assertEquals(objectSource.size(), writer.getObjectCount()); - writer.writePack(m, m, os); - writer.close(); - verifyOpenPack(false); - } - - private void verifyOpenPack(boolean thin) throws IOException { - final byte[] packData = os.toByteArray(); - - if (thin) { - PackParser p = index(packData); - try { - p.parse(NullProgressMonitor.INSTANCE); - fail("indexer should grumble about missing object"); - } catch (IOException x) { - // expected - } - } - - ObjectDirectoryPackParser p = (ObjectDirectoryPackParser) index(packData); - p.setKeepEmpty(true); - p.setAllowThin(thin); - p.setIndexVersion(2); - p.parse(NullProgressMonitor.INSTANCE); - pack = p.getPack(); - assertNotNull("have PackFile after parsing", pack); - } - - private PackParser index(byte[] packData) throws IOException { - if (inserter == null) - inserter = dst.newObjectInserter(); - return inserter.newPackParser(new ByteArrayInputStream(packData)); - } - - private void verifyObjectsOrder(ObjectId objectsOrder[]) { - final List entries = new ArrayList<>(); - - for (MutableEntry me : pack) { - entries.add(me.cloneEntry()); - } - Collections.sort(entries, (MutableEntry o1, MutableEntry o2) -> Long - .signum(o1.getOffset() - o2.getOffset())); - - int i = 0; - for (MutableEntry me : entries) { - assertEquals(objectsOrder[i++].toObjectId(), me.toObjectId()); - } - } - - private static Set haves(ObjectId... objects) { - return Sets.of(objects); - } - - private static Set wants(ObjectId... objects) { - return Sets.of(objects); - } - - private static Set shallows(ObjectId... objects) { - return Sets.of(objects); - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsInserter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsInserter.java index 86fbd445be..8f09261674 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsInserter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsInserter.java @@ -42,7 +42,7 @@ import org.eclipse.jgit.errors.IncorrectObjectTypeException; import org.eclipse.jgit.errors.LargeObjectException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.file.PackIndex; -import org.eclipse.jgit.internal.storage.file.PackIndexWriter; +import org.eclipse.jgit.internal.storage.file.BasePackIndexWriter; import org.eclipse.jgit.internal.storage.file.PackObjectSizeIndexWriter; import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.lib.AbbreviatedObjectId; @@ -333,7 +333,7 @@ public class DfsInserter extends ObjectInserter { private static void index(OutputStream out, byte[] packHash, List list) throws IOException { - PackIndexWriter.createVersion(out, INDEX_VERSION).write(list, packHash); + BasePackIndexWriter.createVersion(out, INDEX_VERSION).write(list, packHash); } void writeObjectSizeIndex(DfsPackDescription pack, diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/BasePackIndexWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/BasePackIndexWriter.java new file mode 100644 index 0000000000..b89cc1ebf4 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/BasePackIndexWriter.java @@ -0,0 +1,269 @@ +/* + * Copyright (C) 2008, Robin Rosenberg + * Copyright (C) 2008, Shawn O. Pearce and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.storage.file; + +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.security.DigestOutputStream; +import java.text.MessageFormat; +import java.util.List; + +import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.internal.storage.pack.PackIndexWriter; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.transport.PackedObjectInfo; +import org.eclipse.jgit.util.NB; + +/** + * Creates a table of contents to support random access by + * {@link org.eclipse.jgit.internal.storage.file.Pack}. + *

+ * Pack index files (the .idx suffix in a pack file pair) provides + * random access to any object in the pack by associating an ObjectId to the + * byte offset within the pack where the object's data can be read. + */ +public abstract class BasePackIndexWriter implements PackIndexWriter { + /** Magic constant indicating post-version 1 format. */ + protected static final byte[] TOC = { -1, 't', 'O', 'c' }; + + /** + * Create a new writer for the oldest (most widely understood) format. + *

+ * This method selects an index format that can accurate describe the + * supplied objects and that will be the most compatible format with older + * Git implementations. + *

+ * Index version 1 is widely recognized by all Git implementations, but + * index version 2 (and later) is not as well recognized as it was + * introduced more than a year later. Index version 1 can only be used if + * the resulting pack file is under 4 gigabytes in size; packs larger than + * that limit must use index version 2. + * + * @param dst + * the stream the index data will be written to. If not already + * buffered it will be automatically wrapped in a buffered + * stream. Callers are always responsible for closing the stream. + * @param objs + * the objects the caller needs to store in the index. Entries + * will be examined until a format can be conclusively selected. + * @return a new writer to output an index file of the requested format to + * the supplied stream. + * @throws java.lang.IllegalArgumentException + * no recognized pack index version can support the supplied + * objects. This is likely a bug in the implementation. + * @see #oldestPossibleFormat(List) + */ + public static PackIndexWriter createOldestPossible(final OutputStream dst, + final List objs) { + return createVersion(dst, oldestPossibleFormat(objs)); + } + + /** + * Return the oldest (most widely understood) index format. + *

+ * This method selects an index format that can accurate describe the + * supplied objects and that will be the most compatible format with older + * Git implementations. + *

+ * Index version 1 is widely recognized by all Git implementations, but + * index version 2 (and later) is not as well recognized as it was + * introduced more than a year later. Index version 1 can only be used if + * the resulting pack file is under 4 gigabytes in size; packs larger than + * that limit must use index version 2. + * + * @param objs + * the objects the caller needs to store in the index. Entries + * will be examined until a format can be conclusively selected. + * @return the index format. + * @throws java.lang.IllegalArgumentException + * no recognized pack index version can support the supplied + * objects. This is likely a bug in the implementation. + */ + public static int oldestPossibleFormat( + final List objs) { + for (PackedObjectInfo oe : objs) { + if (!PackIndexWriterV1.canStore(oe)) + return 2; + } + return 1; + } + + + /** + * Create a new writer instance for a specific index format version. + * + * @param dst + * the stream the index data will be written to. If not already + * buffered it will be automatically wrapped in a buffered + * stream. Callers are always responsible for closing the stream. + * @param version + * index format version number required by the caller. Exactly + * this formatted version will be written. + * @return a new writer to output an index file of the requested format to + * the supplied stream. + * @throws java.lang.IllegalArgumentException + * the version requested is not supported by this + * implementation. + */ + public static PackIndexWriter createVersion(final OutputStream dst, + final int version) { + switch (version) { + case 1: + return new PackIndexWriterV1(dst); + case 2: + return new PackIndexWriterV2(dst); + default: + throw new IllegalArgumentException(MessageFormat.format( + JGitText.get().unsupportedPackIndexVersion, + Integer.valueOf(version))); + } + } + + /** The index data stream we are responsible for creating. */ + protected final DigestOutputStream out; + + /** A temporary buffer for use during IO to {link #out}. */ + protected final byte[] tmp; + + /** The entries this writer must pack. */ + protected List entries; + + /** SHA-1 checksum for the entire pack data. */ + protected byte[] packChecksum; + + /** + * Create a new writer instance. + * + * @param dst + * the stream this instance outputs to. If not already buffered + * it will be automatically wrapped in a buffered stream. + */ + protected BasePackIndexWriter(OutputStream dst) { + out = new DigestOutputStream(dst instanceof BufferedOutputStream ? dst + : new BufferedOutputStream(dst), + Constants.newMessageDigest()); + tmp = new byte[4 + Constants.OBJECT_ID_LENGTH]; + } + + /** + * Write all object entries to the index stream. + *

+ * After writing the stream passed to the factory is flushed but remains + * open. Callers are always responsible for closing the output stream. + * + * @param toStore + * sorted list of objects to store in the index. The caller must + * have previously sorted the list using + * {@link org.eclipse.jgit.transport.PackedObjectInfo}'s native + * {@link java.lang.Comparable} implementation. + * @param packDataChecksum + * checksum signature of the entire pack data content. This is + * traditionally the last 20 bytes of the pack file's own stream. + * @throws java.io.IOException + * an error occurred while writing to the output stream, or this + * index format cannot store the object data supplied. + */ + @Override + public void write(final List toStore, + final byte[] packDataChecksum) throws IOException { + entries = toStore; + packChecksum = packDataChecksum; + writeImpl(); + out.flush(); + } + + /** + * Writes the index file to {@link #out}. + *

+ * Implementations should go something like: + * + * 

+	 * writeFanOutTable();
+	 * for (final PackedObjectInfo po : entries)
+	 * 	writeOneEntry(po);
+	 * writeChecksumFooter();
+	 *
+ * + *

+ * Where the logic for writeOneEntry is specific to the index + * format in use. Additional headers/footers may be used if necessary and + * the {@link #entries} collection may be iterated over more than once if + * necessary. Implementors therefore have complete control over the data. + * + * @throws java.io.IOException + * an error occurred while writing to the output stream, or this + * index format cannot store the object data supplied. + */ + protected abstract void writeImpl() throws IOException; + + /** + * Output the version 2 (and later) TOC header, with version number. + *

+ * Post version 1 all index files start with a TOC header that makes the + * file an invalid version 1 file, and then includes the version number. + * This header is necessary to recognize a version 1 from a version 2 + * formatted index. + * + * @param version + * version number of this index format being written. + * @throws java.io.IOException + * an error occurred while writing to the output stream. + */ + protected void writeTOC(int version) throws IOException { + out.write(TOC); + NB.encodeInt32(tmp, 0, version); + out.write(tmp, 0, 4); + } + + /** + * Output the standard 256 entry first-level fan-out table. + *

+ * The fan-out table is 4 KB in size, holding 256 32-bit unsigned integer + * counts. Each count represents the number of objects within this index + * whose {@link org.eclipse.jgit.lib.ObjectId#getFirstByte()} matches the + * count's position in the fan-out table. + * + * @throws java.io.IOException + * an error occurred while writing to the output stream. + */ + protected void writeFanOutTable() throws IOException { + final int[] fanout = new int[256]; + for (PackedObjectInfo po : entries) + fanout[po.getFirstByte() & 0xff]++; + for (int i = 1; i < 256; i++) + fanout[i] += fanout[i - 1]; + for (int n : fanout) { + NB.encodeInt32(tmp, 0, n); + out.write(tmp, 0, 4); + } + } + + /** + * Output the standard two-checksum index footer. + *

+ * The standard footer contains two checksums (20 byte SHA-1 values): + *

    + *
  1. Pack data checksum - taken from the last 20 bytes of the pack file.
    2. + *
  2. Index data checksum - checksum of all index bytes written, including + * the pack data checksum above.
    3. + *
+ * + * @throws java.io.IOException + * an error occurred while writing to the output stream. + */ + protected void writeChecksumFooter() throws IOException { + out.write(packChecksum); + out.on(false); + out.write(out.getMessageDigest().digest()); + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java index 9f27f4bd6e..746e124e1f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java @@ -28,6 +28,7 @@ import java.util.zip.Deflater; import org.eclipse.jgit.errors.LockFailedException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.pack.PackExt; +import org.eclipse.jgit.internal.storage.pack.PackIndexWriter; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.CoreConfig; @@ -110,7 +111,7 @@ public class ObjectDirectoryPackParser extends PackParser { * @param version * the version to write. The special version 0 designates the * oldest (most compatible) format available for the objects. - * @see PackIndexWriter + * @see BasePackIndexWriter */ public void setIndexVersion(int version) { indexVersion = version; @@ -386,9 +387,9 @@ public class ObjectDirectoryPackParser extends PackParser { try (FileOutputStream os = new FileOutputStream(tmpIdx)) { final PackIndexWriter iw; if (indexVersion <= 0) - iw = PackIndexWriter.createOldestPossible(os, list); + iw = BasePackIndexWriter.createOldestPossible(os, list); else - iw = PackIndexWriter.createVersion(os, indexVersion); + iw = BasePackIndexWriter.createVersion(os, indexVersion); iw.write(list, packHash); os.getChannel().force(true); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java index dfea5c1c80..7189ce20a6 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java @@ -109,7 +109,7 @@ public interface PackIndex } private static boolean isTOC(byte[] h) { - final byte[] toc = PackIndexWriter.TOC; + final byte[] toc = BasePackIndexWriter.TOC; for (int i = 0; i < toc.length; i++) if (h[i] != toc[i]) return false; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriter.java deleted file mode 100644 index 87e0b44d46..0000000000 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriter.java +++ /dev/null @@ -1,267 +0,0 @@ -/* - * Copyright (C) 2008, Robin Rosenberg - * Copyright (C) 2008, Shawn O. Pearce and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.storage.file; - -import java.io.BufferedOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.security.DigestOutputStream; -import java.text.MessageFormat; -import java.util.List; - -import org.eclipse.jgit.internal.JGitText; -import org.eclipse.jgit.lib.Constants; -import org.eclipse.jgit.transport.PackedObjectInfo; -import org.eclipse.jgit.util.NB; - -/** - * Creates a table of contents to support random access by - * {@link org.eclipse.jgit.internal.storage.file.Pack}. - *

- * Pack index files (the .idx suffix in a pack file pair) provides - * random access to any object in the pack by associating an ObjectId to the - * byte offset within the pack where the object's data can be read. - */ -public abstract class PackIndexWriter { - /** Magic constant indicating post-version 1 format. */ - protected static final byte[] TOC = { -1, 't', 'O', 'c' }; - - /** - * Create a new writer for the oldest (most widely understood) format. - *

- * This method selects an index format that can accurate describe the - * supplied objects and that will be the most compatible format with older - * Git implementations. - *

- * Index version 1 is widely recognized by all Git implementations, but - * index version 2 (and later) is not as well recognized as it was - * introduced more than a year later. Index version 1 can only be used if - * the resulting pack file is under 4 gigabytes in size; packs larger than - * that limit must use index version 2. - * - * @param dst - * the stream the index data will be written to. If not already - * buffered it will be automatically wrapped in a buffered - * stream. Callers are always responsible for closing the stream. - * @param objs - * the objects the caller needs to store in the index. Entries - * will be examined until a format can be conclusively selected. - * @return a new writer to output an index file of the requested format to - * the supplied stream. - * @throws java.lang.IllegalArgumentException - * no recognized pack index version can support the supplied - * objects. This is likely a bug in the implementation. - * @see #oldestPossibleFormat(List) - */ - public static PackIndexWriter createOldestPossible(final OutputStream dst, - final List objs) { - return createVersion(dst, oldestPossibleFormat(objs)); - } - - /** - * Return the oldest (most widely understood) index format. - *

- * This method selects an index format that can accurate describe the - * supplied objects and that will be the most compatible format with older - * Git implementations. - *

- * Index version 1 is widely recognized by all Git implementations, but - * index version 2 (and later) is not as well recognized as it was - * introduced more than a year later. Index version 1 can only be used if - * the resulting pack file is under 4 gigabytes in size; packs larger than - * that limit must use index version 2. - * - * @param objs - * the objects the caller needs to store in the index. Entries - * will be examined until a format can be conclusively selected. - * @return the index format. - * @throws java.lang.IllegalArgumentException - * no recognized pack index version can support the supplied - * objects. This is likely a bug in the implementation. - */ - public static int oldestPossibleFormat( - final List objs) { - for (PackedObjectInfo oe : objs) { - if (!PackIndexWriterV1.canStore(oe)) - return 2; - } - return 1; - } - - - /** - * Create a new writer instance for a specific index format version. - * - * @param dst - * the stream the index data will be written to. If not already - * buffered it will be automatically wrapped in a buffered - * stream. Callers are always responsible for closing the stream. - * @param version - * index format version number required by the caller. Exactly - * this formatted version will be written. - * @return a new writer to output an index file of the requested format to - * the supplied stream. - * @throws java.lang.IllegalArgumentException - * the version requested is not supported by this - * implementation. - */ - public static PackIndexWriter createVersion(final OutputStream dst, - final int version) { - switch (version) { - case 1: - return new PackIndexWriterV1(dst); - case 2: - return new PackIndexWriterV2(dst); - default: - throw new IllegalArgumentException(MessageFormat.format( - JGitText.get().unsupportedPackIndexVersion, - Integer.valueOf(version))); - } - } - - /** The index data stream we are responsible for creating. */ - protected final DigestOutputStream out; - - /** A temporary buffer for use during IO to {link #out}. */ - protected final byte[] tmp; - - /** The entries this writer must pack. */ - protected List entries; - - /** SHA-1 checksum for the entire pack data. */ - protected byte[] packChecksum; - - /** - * Create a new writer instance. - * - * @param dst - * the stream this instance outputs to. If not already buffered - * it will be automatically wrapped in a buffered stream. - */ - protected PackIndexWriter(OutputStream dst) { - out = new DigestOutputStream(dst instanceof BufferedOutputStream ? dst - : new BufferedOutputStream(dst), - Constants.newMessageDigest()); - tmp = new byte[4 + Constants.OBJECT_ID_LENGTH]; - } - - /** - * Write all object entries to the index stream. - *

- * After writing the stream passed to the factory is flushed but remains - * open. Callers are always responsible for closing the output stream. - * - * @param toStore - * sorted list of objects to store in the index. The caller must - * have previously sorted the list using - * {@link org.eclipse.jgit.transport.PackedObjectInfo}'s native - * {@link java.lang.Comparable} implementation. - * @param packDataChecksum - * checksum signature of the entire pack data content. This is - * traditionally the last 20 bytes of the pack file's own stream. - * @throws java.io.IOException - * an error occurred while writing to the output stream, or this - * index format cannot store the object data supplied. - */ - public void write(final List toStore, - final byte[] packDataChecksum) throws IOException { - entries = toStore; - packChecksum = packDataChecksum; - writeImpl(); - out.flush(); - } - - /** - * Writes the index file to {@link #out}. - *

- * Implementations should go something like: - * - * 

-	 * writeFanOutTable();
-	 * for (final PackedObjectInfo po : entries)
-	 * 	writeOneEntry(po);
-	 * writeChecksumFooter();
-	 *
- * - *

- * Where the logic for writeOneEntry is specific to the index - * format in use. Additional headers/footers may be used if necessary and - * the {@link #entries} collection may be iterated over more than once if - * necessary. Implementors therefore have complete control over the data. - * - * @throws java.io.IOException - * an error occurred while writing to the output stream, or this - * index format cannot store the object data supplied. - */ - protected abstract void writeImpl() throws IOException; - - /** - * Output the version 2 (and later) TOC header, with version number. - *

- * Post version 1 all index files start with a TOC header that makes the - * file an invalid version 1 file, and then includes the version number. - * This header is necessary to recognize a version 1 from a version 2 - * formatted index. - * - * @param version - * version number of this index format being written. - * @throws java.io.IOException - * an error occurred while writing to the output stream. - */ - protected void writeTOC(int version) throws IOException { - out.write(TOC); - NB.encodeInt32(tmp, 0, version); - out.write(tmp, 0, 4); - } - - /** - * Output the standard 256 entry first-level fan-out table. - *

- * The fan-out table is 4 KB in size, holding 256 32-bit unsigned integer - * counts. Each count represents the number of objects within this index - * whose {@link org.eclipse.jgit.lib.ObjectId#getFirstByte()} matches the - * count's position in the fan-out table. - * - * @throws java.io.IOException - * an error occurred while writing to the output stream. - */ - protected void writeFanOutTable() throws IOException { - final int[] fanout = new int[256]; - for (PackedObjectInfo po : entries) - fanout[po.getFirstByte() & 0xff]++; - for (int i = 1; i < 256; i++) - fanout[i] += fanout[i - 1]; - for (int n : fanout) { - NB.encodeInt32(tmp, 0, n); - out.write(tmp, 0, 4); - } - } - - /** - * Output the standard two-checksum index footer. - *

- * The standard footer contains two checksums (20 byte SHA-1 values): - *

    - *
  1. Pack data checksum - taken from the last 20 bytes of the pack file.
    2. - *
  2. Index data checksum - checksum of all index bytes written, including - * the pack data checksum above.
    3. - *
- * - * @throws java.io.IOException - * an error occurred while writing to the output stream. - */ - protected void writeChecksumFooter() throws IOException { - out.write(packChecksum); - out.on(false); - out.write(out.getMessageDigest().digest()); - } -} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriterV1.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriterV1.java index 7e28b5eb2b..f0b6193066 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriterV1.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriterV1.java @@ -21,10 +21,10 @@ import org.eclipse.jgit.util.NB; /** * Creates the version 1 (old style) pack table of contents files. * - * @see PackIndexWriter + * @see BasePackIndexWriter * @see PackIndexV1 */ -class PackIndexWriterV1 extends PackIndexWriter { +class PackIndexWriterV1 extends BasePackIndexWriter { static boolean canStore(PackedObjectInfo oe) { // We are limited to 4 GB per pack as offset is 32 bit unsigned int. // diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriterV2.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriterV2.java index fc5ef61912..b72b35a464 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriterV2.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndexWriterV2.java @@ -19,10 +19,10 @@ import org.eclipse.jgit.util.NB; /** * Creates the version 2 pack table of contents files. * - * @see PackIndexWriter + * @see BasePackIndexWriter * @see PackIndexV2 */ -class PackIndexWriterV2 extends PackIndexWriter { +class PackIndexWriterV2 extends BasePackIndexWriter { private static final int MAX_OFFSET_32 = 0x7fffffff; private static final int IS_OFFSET_64 = 0x80000000; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java index 1b092a3332..55e047bd43 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java @@ -77,6 +77,7 @@ import org.eclipse.jgit.errors.LargeObjectException; import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.pack.PackExt; +import org.eclipse.jgit.internal.storage.pack.PackIndexWriter; import org.eclipse.jgit.lib.AbbreviatedObjectId; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.Constants; @@ -320,7 +321,8 @@ public class PackInserter extends ObjectInserter { private static void writePackIndex(File idx, byte[] packHash, List list) throws IOException { try (OutputStream os = new FileOutputStream(idx)) { - PackIndexWriter w = PackIndexWriter.createVersion(os, INDEX_VERSION); + PackIndexWriter w = BasePackIndexWriter.createVersion(os, + INDEX_VERSION); w.write(list, packHash); } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackIndexWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackIndexWriter.java new file mode 100644 index 0000000000..f69e68d4ba --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackIndexWriter.java @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2024, Google LLC. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.storage.pack; + +import java.io.IOException; +import java.util.List; + +import org.eclipse.jgit.transport.PackedObjectInfo; + +/** + * Represents a function that accepts a collection of objects to write into a + * primary pack index storage format. + */ +public interface PackIndexWriter { + /** + * Write all object entries to the index stream. + * + * @param toStore + * sorted list of objects to store in the index. The caller must + * have previously sorted the list using + * {@link org.eclipse.jgit.transport.PackedObjectInfo}'s native + * {@link java.lang.Comparable} implementation. + * @param packDataChecksum + * checksum signature of the entire pack data content. This is + * traditionally the last 20 bytes of the pack file's own stream. + * @throws java.io.IOException + * an error occurred while writing to the output stream, or the + * underlying format cannot store the object data supplied. + */ + void write(List toStore, + byte[] packDataChecksum) throws IOException; +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java index d3e30f3f6c..4fd2eb5798 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java @@ -58,7 +58,7 @@ import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.errors.SearchForReuseTimeout; import org.eclipse.jgit.errors.StoredObjectRepresentationNotAvailableException; import org.eclipse.jgit.internal.JGitText; -import org.eclipse.jgit.internal.storage.file.PackIndexWriter; +import org.eclipse.jgit.internal.storage.file.BasePackIndexWriter; import org.eclipse.jgit.internal.storage.file.PackObjectSizeIndexWriter; import org.eclipse.jgit.internal.storage.file.PackReverseIndexWriter; import org.eclipse.jgit.internal.storage.file.PackBitmapIndexBuilder; @@ -1078,7 +1078,7 @@ public class PackWriter implements AutoCloseable { if (indexVersion <= 0) { for (BlockList objs : objectsLists) indexVersion = Math.max(indexVersion, - PackIndexWriter.oldestPossibleFormat(objs)); + BasePackIndexWriter.oldestPossibleFormat(objs)); } return indexVersion; } @@ -1103,8 +1103,8 @@ public class PackWriter implements AutoCloseable { throw new IOException(JGitText.get().cachedPacksPreventsIndexCreation); long writeStart = System.currentTimeMillis(); - final PackIndexWriter iw = PackIndexWriter.createVersion( - indexStream, getIndexVersion()); + PackIndexWriter iw = BasePackIndexWriter.createVersion(indexStream, + getIndexVersion()); iw.write(sortByName(), packcsum); stats.timeWriting += System.currentTimeMillis() - writeStart; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java index 8373d6809a..863b79466a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/storage/pack/PackConfig.java @@ -50,7 +50,7 @@ import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import java.util.zip.Deflater; -import org.eclipse.jgit.internal.storage.file.PackIndexWriter; +import org.eclipse.jgit.internal.storage.file.BasePackIndexWriter; import org.eclipse.jgit.lib.Config; import org.eclipse.jgit.lib.Repository; @@ -995,7 +995,7 @@ public class PackConfig { * * @return the index version, the special version 0 designates the oldest * (most compatible) format available for the objects. - * @see PackIndexWriter + * @see BasePackIndexWriter */ public int getIndexVersion() { return indexVersion; @@ -1009,7 +1009,7 @@ public class PackConfig { * @param version * the version to write. The special version 0 designates the * oldest (most compatible) format available for the objects. - * @see PackIndexWriter + * @see BasePackIndexWriter */ public void setIndexVersion(int version) { indexVersion = version; -- cgit v1.2.3 From 4402f10247169cce87fe950c7e6647bf44298ebd Mon Sep 17 00:00:00 2001 From: Laura Hamelin Date: Wed, 24 Jul 2024 14:01:38 -0700 Subject: dfs: update getBlockCacheStats to return a List of BlockCacheStats Make available all underlying cache table stats for the used cache table implementation. The existing cache table stats implementation only allows a "global" view of the cache table statistics; it does not differentiate between all possible underlying cache tables used. This change allows callers to get the block cache stats broken down per underlying table. These cache stats are intended to be used for monitoring all cache tables independently. Existing usages of getBlockCacheStats now make use of AggregatedBlockCacheStats.fromStatsList to aggregate the list of BlockCacheStats into a single BlockCacheStats instance. Change-Id: I261b3f2849857172397657e5c674b11e09807f27 --- .../storage/dfs/AggregatedBlockCacheStatsTest.java | 23 +++--- .../storage/dfs/ClockBlockCacheTableTest.java | 22 ++++- .../storage/dfs/PackExtBlockCacheTableTest.java | 93 +++++++++++++++++----- .../storage/dfs/AggregatedBlockCacheStats.java | 12 +-- .../internal/storage/dfs/ClockBlockCacheTable.java | 5 +- .../jgit/internal/storage/dfs/DfsBlockCache.java | 31 ++++++-- .../internal/storage/dfs/DfsBlockCacheTable.java | 15 ++-- .../storage/dfs/PackExtBlockCacheTable.java | 9 +-- 8 files changed, 150 insertions(+), 60 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/AggregatedBlockCacheStatsTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/AggregatedBlockCacheStatsTest.java index ac769498e2..2c4b432a01 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/AggregatedBlockCacheStatsTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/AggregatedBlockCacheStatsTest.java @@ -24,9 +24,10 @@ public class AggregatedBlockCacheStatsTest { @Test public void getName() { BlockCacheStats aggregatedBlockCacheStats = AggregatedBlockCacheStats - .fromStatsList("name", List.of()); + .fromStatsList(List.of()); - assertThat(aggregatedBlockCacheStats.getName(), equalTo("name")); + assertThat(aggregatedBlockCacheStats.getName(), + equalTo(AggregatedBlockCacheStats.class.getName())); } @Test @@ -46,8 +47,7 @@ public class AggregatedBlockCacheStatsTest { currentSizes[PackExt.INDEX.getPosition()] = 7; BlockCacheStats aggregatedBlockCacheStats = AggregatedBlockCacheStats - .fromStatsList("name", - List.of(packStats, bitmapStats, indexStats)); + .fromStatsList(List.of(packStats, bitmapStats, indexStats)); assertArrayEquals(aggregatedBlockCacheStats.getCurrentSize(), currentSizes); @@ -73,8 +73,7 @@ public class AggregatedBlockCacheStatsTest { hitCounts[PackExt.INDEX.getPosition()] = 7; BlockCacheStats aggregatedBlockCacheStats = AggregatedBlockCacheStats - .fromStatsList("name", - List.of(packStats, bitmapStats, indexStats)); + .fromStatsList(List.of(packStats, bitmapStats, indexStats)); assertArrayEquals(aggregatedBlockCacheStats.getHitCount(), hitCounts); } @@ -99,8 +98,7 @@ public class AggregatedBlockCacheStatsTest { missCounts[PackExt.INDEX.getPosition()] = 7; BlockCacheStats aggregatedBlockCacheStats = AggregatedBlockCacheStats - .fromStatsList("name", - List.of(packStats, bitmapStats, indexStats)); + .fromStatsList(List.of(packStats, bitmapStats, indexStats)); assertArrayEquals(aggregatedBlockCacheStats.getMissCount(), missCounts); } @@ -131,8 +129,7 @@ public class AggregatedBlockCacheStatsTest { totalRequestCounts[PackExt.INDEX.getPosition()] = 14; BlockCacheStats aggregatedBlockCacheStats = AggregatedBlockCacheStats - .fromStatsList("name", - List.of(packStats, bitmapStats, indexStats)); + .fromStatsList(List.of(packStats, bitmapStats, indexStats)); assertArrayEquals(aggregatedBlockCacheStats.getTotalRequestCount(), totalRequestCounts); @@ -160,8 +157,7 @@ public class AggregatedBlockCacheStatsTest { hitRatios[PackExt.INDEX.getPosition()] = 0; BlockCacheStats aggregatedBlockCacheStats = AggregatedBlockCacheStats - .fromStatsList("Name", - List.of(packStats, bitmapStats, indexStats)); + .fromStatsList(List.of(packStats, bitmapStats, indexStats)); assertArrayEquals(aggregatedBlockCacheStats.getHitRatio(), hitRatios); } @@ -186,8 +182,7 @@ public class AggregatedBlockCacheStatsTest { evictions[PackExt.INDEX.getPosition()] = 7; BlockCacheStats aggregatedBlockCacheStats = AggregatedBlockCacheStats - .fromStatsList("Name", - List.of(packStats, bitmapStats, indexStats)); + .fromStatsList(List.of(packStats, bitmapStats, indexStats)); assertArrayEquals(aggregatedBlockCacheStats.getEvictions(), evictions); } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/ClockBlockCacheTableTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/ClockBlockCacheTableTest.java index cb68bbc515..2e2f86bf80 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/ClockBlockCacheTableTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/ClockBlockCacheTableTest.java @@ -1,8 +1,14 @@ package org.eclipse.jgit.internal.storage.dfs; import static org.eclipse.jgit.internal.storage.dfs.DfsBlockCacheConfig.DEFAULT_NAME; +import static org.eclipse.jgit.internal.storage.dfs.DfsBlockCacheTable.BlockCacheStats; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.isA; + +import java.util.List; import org.junit.Test; @@ -30,7 +36,8 @@ public class ClockBlockCacheTableTest { ClockBlockCacheTable cacheTable = new ClockBlockCacheTable( createBlockCacheConfig()); - assertThat(cacheTable.getBlockCacheStats().getName(), + assertThat(cacheTable.getBlockCacheStats(), hasSize(1)); + assertThat(cacheTable.getBlockCacheStats().get(0).getName(), equalTo(DEFAULT_NAME)); } @@ -39,7 +46,18 @@ public class ClockBlockCacheTableTest { ClockBlockCacheTable cacheTable = new ClockBlockCacheTable( createBlockCacheConfig().setName(NAME)); - assertThat(cacheTable.getBlockCacheStats().getName(), equalTo(NAME)); + assertThat(cacheTable.getBlockCacheStats(), hasSize(1)); + assertThat(cacheTable.getBlockCacheStats().get(0).getName(), + equalTo(NAME)); + } + + @Test + public void getAllBlockCacheStats() { + ClockBlockCacheTable cacheTable = new ClockBlockCacheTable( + createBlockCacheConfig()); + + List blockCacheStats = cacheTable.getBlockCacheStats(); + assertThat(blockCacheStats, contains(isA(BlockCacheStats.class))); } private static DfsBlockCacheConfig createBlockCacheConfig() { diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/PackExtBlockCacheTableTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/PackExtBlockCacheTableTest.java index c5c964bcab..e7627bc4ab 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/PackExtBlockCacheTableTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/PackExtBlockCacheTableTest.java @@ -10,8 +10,10 @@ package org.eclipse.jgit.internal.storage.dfs; +import static org.eclipse.jgit.internal.storage.dfs.DfsBlockCacheTable.BlockCacheStats; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.sameInstance; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertNotNull; @@ -398,15 +400,51 @@ public class PackExtBlockCacheTableTest { } @Test - public void getBlockCacheStats_getName_returnsPackExtCacheTableName() { - DfsBlockCacheStats packStats = new DfsBlockCacheStats(); - PackExtBlockCacheTable tables = PackExtBlockCacheTable.fromCacheTables( - cacheTableWithStats(/* name= */ "defaultName", packStats), - Map.of(PackExt.PACK, cacheTableWithStats(/* name= */ "packName", - packStats))); + public void getAllBlockCacheStats() { + String defaultTableName = "default table"; + DfsBlockCacheStats defaultStats = new DfsBlockCacheStats( + defaultTableName); + incrementCounter(4, + () -> defaultStats.incrementHit(new TestKey(PackExt.REFTABLE))); + + String packTableName = "pack table"; + DfsBlockCacheStats packStats = new DfsBlockCacheStats(packTableName); + incrementCounter(5, + () -> packStats.incrementHit(new TestKey(PackExt.PACK))); - assertThat(tables.getBlockCacheStats().getName(), - equalTo("defaultName,packName")); + String bitmapTableName = "bitmap table"; + DfsBlockCacheStats bitmapStats = new DfsBlockCacheStats( + bitmapTableName); + incrementCounter(6, () -> bitmapStats + .incrementHit(new TestKey(PackExt.BITMAP_INDEX))); + + DfsBlockCacheTable defaultTable = cacheTableWithStats(defaultStats); + DfsBlockCacheTable packTable = cacheTableWithStats(packStats); + DfsBlockCacheTable bitmapTable = cacheTableWithStats(bitmapStats); + PackExtBlockCacheTable tables = PackExtBlockCacheTable + .fromCacheTables(defaultTable, Map.of(PackExt.PACK, packTable, + PackExt.BITMAP_INDEX, bitmapTable)); + + List statsList = tables.getBlockCacheStats(); + assertThat(statsList, hasSize(3)); + + long[] defaultTableHitCounts = createEmptyStatsArray(); + defaultTableHitCounts[PackExt.REFTABLE.getPosition()] = 4; + assertArrayEquals( + getCacheStatsByName(statsList, defaultTableName).getHitCount(), + defaultTableHitCounts); + + long[] packTableHitCounts = createEmptyStatsArray(); + packTableHitCounts[PackExt.PACK.getPosition()] = 5; + assertArrayEquals( + getCacheStatsByName(statsList, packTableName).getHitCount(), + packTableHitCounts); + + long[] bitmapHitCounts = createEmptyStatsArray(); + bitmapHitCounts[PackExt.BITMAP_INDEX.getPosition()] = 6; + assertArrayEquals( + getCacheStatsByName(statsList, bitmapTableName).getHitCount(), + bitmapHitCounts); } @Test @@ -431,7 +469,8 @@ public class PackExtBlockCacheTableTest { cacheTableWithStats(bitmapStats), PackExt.INDEX, cacheTableWithStats(indexStats))); - assertArrayEquals(tables.getBlockCacheStats().getCurrentSize(), + assertArrayEquals(AggregatedBlockCacheStats + .fromStatsList(tables.getBlockCacheStats()).getCurrentSize(), currentSizes); } @@ -460,7 +499,9 @@ public class PackExtBlockCacheTableTest { cacheTableWithStats(bitmapStats), PackExt.INDEX, cacheTableWithStats(indexStats))); - assertArrayEquals(tables.getBlockCacheStats().getHitCount(), hitCounts); + assertArrayEquals(AggregatedBlockCacheStats + .fromStatsList(tables.getBlockCacheStats()).getHitCount(), + hitCounts); } @Test @@ -488,7 +529,8 @@ public class PackExtBlockCacheTableTest { cacheTableWithStats(bitmapStats), PackExt.INDEX, cacheTableWithStats(indexStats))); - assertArrayEquals(tables.getBlockCacheStats().getMissCount(), + assertArrayEquals(AggregatedBlockCacheStats + .fromStatsList(tables.getBlockCacheStats()).getMissCount(), missCounts); } @@ -523,8 +565,9 @@ public class PackExtBlockCacheTableTest { cacheTableWithStats(bitmapStats), PackExt.INDEX, cacheTableWithStats(indexStats))); - assertArrayEquals(tables.getBlockCacheStats().getTotalRequestCount(), - totalRequestCounts); + assertArrayEquals(AggregatedBlockCacheStats + .fromStatsList(tables.getBlockCacheStats()) + .getTotalRequestCount(), totalRequestCounts); } @Test @@ -554,7 +597,9 @@ public class PackExtBlockCacheTableTest { cacheTableWithStats(bitmapStats), PackExt.INDEX, cacheTableWithStats(indexStats))); - assertArrayEquals(tables.getBlockCacheStats().getHitRatio(), hitRatios); + assertArrayEquals(AggregatedBlockCacheStats + .fromStatsList(tables.getBlockCacheStats()).getHitRatio(), + hitRatios); } @Test @@ -582,10 +627,21 @@ public class PackExtBlockCacheTableTest { cacheTableWithStats(bitmapStats), PackExt.INDEX, cacheTableWithStats(indexStats))); - assertArrayEquals(tables.getBlockCacheStats().getEvictions(), + assertArrayEquals(AggregatedBlockCacheStats + .fromStatsList(tables.getBlockCacheStats()).getEvictions(), evictions); } + private BlockCacheStats getCacheStatsByName( + List blockCacheStats, String name) { + for (BlockCacheStats entry : blockCacheStats) { + if (entry.getName().equals(name)) { + return entry; + } + } + return null; + } + private static void incrementCounter(int amount, Runnable fn) { for (int i = 0; i < amount; i++) { fn.run(); @@ -597,15 +653,16 @@ public class PackExtBlockCacheTableTest { } private static DfsBlockCacheTable cacheTableWithStats( - DfsBlockCacheStats dfsBlockCacheStats) { + BlockCacheStats dfsBlockCacheStats) { return cacheTableWithStats(CACHE_NAME, dfsBlockCacheStats); } private static DfsBlockCacheTable cacheTableWithStats(String name, - DfsBlockCacheStats dfsBlockCacheStats) { + BlockCacheStats dfsBlockCacheStats) { DfsBlockCacheTable cacheTable = mock(DfsBlockCacheTable.class); when(cacheTable.getName()).thenReturn(name); - when(cacheTable.getBlockCacheStats()).thenReturn(dfsBlockCacheStats); + when(cacheTable.getBlockCacheStats()) + .thenReturn(List.of(dfsBlockCacheStats)); return cacheTable; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/AggregatedBlockCacheStats.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/AggregatedBlockCacheStats.java index 743f4606c4..295b702fa7 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/AggregatedBlockCacheStats.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/AggregatedBlockCacheStats.java @@ -20,27 +20,23 @@ import org.eclipse.jgit.internal.storage.pack.PackExt; * Aggregates values for all given {@link BlockCacheStats}. */ class AggregatedBlockCacheStats implements BlockCacheStats { - private final String name; - private final List blockCacheStats; - static BlockCacheStats fromStatsList(String name, + static BlockCacheStats fromStatsList( List blockCacheStats) { if (blockCacheStats.size() == 1) { return blockCacheStats.get(0); } - return new AggregatedBlockCacheStats(name, blockCacheStats); + return new AggregatedBlockCacheStats(blockCacheStats); } - private AggregatedBlockCacheStats(String name, - List blockCacheStats) { - this.name = name; + private AggregatedBlockCacheStats(List blockCacheStats) { this.blockCacheStats = blockCacheStats; } @Override public String getName() { - return name; + return AggregatedBlockCacheStats.class.getName(); } @Override diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/ClockBlockCacheTable.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/ClockBlockCacheTable.java index bbee23b061..587d482583 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/ClockBlockCacheTable.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/ClockBlockCacheTable.java @@ -12,6 +12,7 @@ package org.eclipse.jgit.internal.storage.dfs; import java.io.IOException; import java.time.Duration; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicReferenceArray; @@ -141,8 +142,8 @@ final class ClockBlockCacheTable implements DfsBlockCacheTable { } @Override - public BlockCacheStats getBlockCacheStats() { - return dfsBlockCacheStats; + public List getBlockCacheStats() { + return List.of(dfsBlockCacheStats); } @Override diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCache.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCache.java index 0334450fbe..f8e0831e1f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCache.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCache.java @@ -11,7 +11,10 @@ package org.eclipse.jgit.internal.storage.dfs; +import static org.eclipse.jgit.internal.storage.dfs.DfsBlockCacheTable.BlockCacheStats; + import java.io.IOException; +import java.util.List; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.LongStream; @@ -124,7 +127,7 @@ public final class DfsBlockCache { * @return total number of bytes in the cache, per pack file extension. */ public long[] getCurrentSize() { - return dfsBlockCacheTable.getBlockCacheStats().getCurrentSize(); + return getAggregatedBlockCacheStats().getCurrentSize(); } /** @@ -143,7 +146,7 @@ public final class DfsBlockCache { * extension. */ public long[] getHitCount() { - return dfsBlockCacheTable.getBlockCacheStats().getHitCount(); + return getAggregatedBlockCacheStats().getHitCount(); } /** @@ -154,7 +157,7 @@ public final class DfsBlockCache { * extension. */ public long[] getMissCount() { - return dfsBlockCacheTable.getBlockCacheStats().getMissCount(); + return getAggregatedBlockCacheStats().getMissCount(); } /** @@ -163,7 +166,7 @@ public final class DfsBlockCache { * @return total number of requests (hit + miss), per pack file extension. */ public long[] getTotalRequestCount() { - return dfsBlockCacheTable.getBlockCacheStats().getTotalRequestCount(); + return getAggregatedBlockCacheStats().getTotalRequestCount(); } /** @@ -172,7 +175,7 @@ public final class DfsBlockCache { * @return hit ratios */ public long[] getHitRatio() { - return dfsBlockCacheTable.getBlockCacheStats().getHitRatio(); + return getAggregatedBlockCacheStats().getHitRatio(); } /** @@ -183,7 +186,18 @@ public final class DfsBlockCache { * file extension. */ public long[] getEvictions() { - return dfsBlockCacheTable.getBlockCacheStats().getEvictions(); + return getAggregatedBlockCacheStats().getEvictions(); + } + + /** + * Get the list of {@link BlockCacheStats} for all underlying caches. + *

+ * Useful in monitoring caches with breakdown. + * + * @return the list of {@link BlockCacheStats} for all underlying caches. + */ + public List getAllBlockCacheStats() { + return dfsBlockCacheTable.getBlockCacheStats(); } /** @@ -263,6 +277,11 @@ public final class DfsBlockCache { return dfsBlockCacheTable.get(key, position); } + private BlockCacheStats getAggregatedBlockCacheStats() { + return AggregatedBlockCacheStats + .fromStatsList(dfsBlockCacheTable.getBlockCacheStats()); + } + static final class Ref { final DfsStreamKey key; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheTable.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheTable.java index 43ba8c36ab..c3fd07b7bb 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheTable.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheTable.java @@ -11,6 +11,7 @@ package org.eclipse.jgit.internal.storage.dfs; import java.io.IOException; +import java.util.List; /** * Block cache table. @@ -125,13 +126,17 @@ public interface DfsBlockCacheTable { T get(DfsStreamKey key, long position); /** - * Get the {@link BlockCacheStats} object for this block cache table's - * statistics. + * Get the list of {@link BlockCacheStats} held by this cache. + *

+ * The returned list has a {@link BlockCacheStats} per configured cache + * table, with a minimum of 1 {@link BlockCacheStats} object returned. + * + * Use {@link AggregatedBlockCacheStats} to combine the results of the stats + * in the list for an aggregated view of the cache's stats. * - * @return the {@link BlockCacheStats} tracking this block cache table's - * statistics. + * @return the list of {@link BlockCacheStats} held by this cache. */ - BlockCacheStats getBlockCacheStats(); + List getBlockCacheStats(); /** * Get the name of the table. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/PackExtBlockCacheTable.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/PackExtBlockCacheTable.java index e45643be84..28b021c68f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/PackExtBlockCacheTable.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/PackExtBlockCacheTable.java @@ -188,11 +188,10 @@ class PackExtBlockCacheTable implements DfsBlockCacheTable { } @Override - public BlockCacheStats getBlockCacheStats() { - return AggregatedBlockCacheStats.fromStatsList(name, - blockCacheTableList.stream() - .map(DfsBlockCacheTable::getBlockCacheStats) - .collect(Collectors.toList())); + public List getBlockCacheStats() { + return blockCacheTableList.stream() + .flatMap(cacheTable -> cacheTable.getBlockCacheStats().stream()) + .collect(Collectors.toList()); } @Override -- cgit v1.2.3 From e5f42ae18e45e5c263ba510924e09cb2bb4bc822 Mon Sep 17 00:00:00 2001 From: Sam Delmerico Date: Mon, 7 Oct 2024 14:34:03 -0700 Subject: PackWriter: make PackWriter.writeIndex() take a PackIndexWriter Previously, the PackWriter implementation required that indexes and their extensions be writable to an OutputStream with a fixed binary format. To support more general index storage formats, allow PackWriter to accept an PackIndexWriter interface which accepts only the objects to store. This allows implementors to choose their storage format. The implementation will be provided by the DfsObjectDatabase. The DfsObjectDatabase is already responsible for providing the OutputStream that was previously used to write indexes. Having it provide a writing interface would be a natural generalization. This idea was previously implemented for PackBitmapIndex writing in https://gerrithub.io/c/eclipse-jgit/jgit/+/1177722. Change-Id: I582d2f3d25d6adb2da243d6d0d7bc405a97d6183 --- .../internal/storage/dfs/DfsGarbageCollector.java | 10 +------ .../jgit/internal/storage/dfs/DfsObjDatabase.java | 34 ++++++++++++++++++++++ .../internal/storage/dfs/DfsPackCompactor.java | 11 +------ .../jgit/internal/storage/pack/PackWriter.java | 26 +++++++++++++---- 4 files changed, 57 insertions(+), 24 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java index a177669788..b933e942d9 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java @@ -18,7 +18,6 @@ import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.RE import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE; import static org.eclipse.jgit.internal.storage.dfs.DfsPackCompactor.configureReftable; import static org.eclipse.jgit.internal.storage.pack.PackExt.COMMIT_GRAPH; -import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.OBJECT_SIZE_INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK; import static org.eclipse.jgit.internal.storage.pack.PackExt.REFTABLE; @@ -687,14 +686,7 @@ public class DfsGarbageCollector { pack.setBlockSize(PACK, out.blockSize()); } - try (DfsOutputStream out = objdb.writeFile(pack, INDEX)) { - CountingOutputStream cnt = new CountingOutputStream(out); - pw.writeIndex(cnt); - pack.addFileExt(INDEX); - pack.setFileSize(INDEX, cnt.getCount()); - pack.setBlockSize(INDEX, out.blockSize()); - pack.setIndexVersion(pw.getIndexVersion()); - } + pw.writeIndex(objdb.getPackIndexWriter(pack, pw.getIndexVersion())); if (source != UNREACHABLE_GARBAGE && packConfig.getMinBytesForObjSizeIndex() >= 0) { try (DfsOutputStream out = objdb.writeFile(pack, diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java index 616563ffdd..efd666ff27 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java @@ -12,6 +12,7 @@ package org.eclipse.jgit.internal.storage.dfs; import static java.util.stream.Collectors.joining; import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX; +import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX; import java.io.FileNotFoundException; import java.io.IOException; @@ -27,7 +28,9 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicReference; +import org.eclipse.jgit.internal.storage.file.BasePackIndexWriter; import org.eclipse.jgit.internal.storage.file.PackBitmapIndexWriterV1; +import org.eclipse.jgit.internal.storage.pack.PackIndexWriter; import org.eclipse.jgit.internal.storage.pack.PackBitmapIndexWriter; import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.lib.AnyObjectId; @@ -771,4 +774,35 @@ public abstract class DfsObjDatabase extends ObjectDatabase { } }; } + + /** + * Returns a writer to store the pack index in this object database. + * + * @param pack + * Pack file to which the index is associated. + * @param indexVersion + * which version of the index to write + * @return a writer to store the index associated with the pack + * @throws IOException + * when some I/O problem occurs while creating or writing to + * output stream + */ + public PackIndexWriter getPackIndexWriter( + DfsPackDescription pack, int indexVersion) + throws IOException { + return (objectsToStore, packDataChecksum) -> { + try (DfsOutputStream out = writeFile(pack, INDEX); + CountingOutputStream cnt = new CountingOutputStream(out)) { + final PackIndexWriter iw = BasePackIndexWriter + .createVersion(cnt, + indexVersion); + iw.write(objectsToStore, packDataChecksum); + pack.addFileExt(INDEX); + pack.setFileSize(INDEX, cnt.getCount()); + pack.setBlockSize(INDEX, out.blockSize()); + pack.setIndexVersion(indexVersion); + } + }; + } + } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java index 86144b389c..b32cddae77 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java @@ -12,7 +12,6 @@ package org.eclipse.jgit.internal.storage.dfs; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.COMPACT; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC; -import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK; import static org.eclipse.jgit.internal.storage.pack.PackExt.REFTABLE; import static org.eclipse.jgit.internal.storage.pack.StoredObjectRepresentation.PACK_DELTA; @@ -45,7 +44,6 @@ import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.storage.pack.PackConfig; import org.eclipse.jgit.storage.pack.PackStatistics; import org.eclipse.jgit.util.BlockList; -import org.eclipse.jgit.util.io.CountingOutputStream; /** * Combine several pack files into one pack. @@ -458,14 +456,7 @@ public class DfsPackCompactor { private static void writeIndex(DfsObjDatabase objdb, DfsPackDescription pack, PackWriter pw) throws IOException { - try (DfsOutputStream out = objdb.writeFile(pack, INDEX)) { - CountingOutputStream cnt = new CountingOutputStream(out); - pw.writeIndex(cnt); - pack.addFileExt(INDEX); - pack.setFileSize(INDEX, cnt.getCount()); - pack.setBlockSize(INDEX, out.blockSize()); - pack.setIndexVersion(pw.getIndexVersion()); - } + pw.writeIndex(objdb.getPackIndexWriter(pack, pw.getIndexVersion())); } static ReftableConfig configureReftable(ReftableConfig cfg, diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java index 4fd2eb5798..4c262b0e3a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java @@ -118,7 +118,7 @@ import org.eclipse.jgit.util.TemporaryBuffer; * {@link #preparePack(ProgressMonitor, Set, Set)}, and streaming with * {@link #writePack(ProgressMonitor, ProgressMonitor, OutputStream)}. If the * pack is being stored as a file the matching index can be written out after - * writing the pack by {@link #writeIndex(OutputStream)}. An optional bitmap + * writing the pack by {@link #writeIndex(PackIndexWriter)}. An optional bitmap * index can be made by calling {@link #prepareBitmapIndex(ProgressMonitor)} * followed by {@link #writeBitmapIndex(PackBitmapIndexWriter)}. *

@@ -1099,12 +1099,28 @@ public class PackWriter implements AutoCloseable { * the index data could not be written to the supplied stream. */ public void writeIndex(OutputStream indexStream) throws IOException { + writeIndex(BasePackIndexWriter.createVersion(indexStream, + getIndexVersion())); + } + + /** + * Create an index file to match the pack file just written. + *

+ * Called after + * {@link #writePack(ProgressMonitor, ProgressMonitor, OutputStream)}. + *

+ * Writing an index is only required for local pack storage. Packs sent on + * the network do not need to create an index. + * + * @param iw + * an @code{PackIndexWriter} instance to write the index + * @throws java.io.IOException + * the index data could not be written to the supplied stream. + */ + public void writeIndex(PackIndexWriter iw) throws IOException { if (isIndexDisabled()) throw new IOException(JGitText.get().cachedPacksPreventsIndexCreation); - long writeStart = System.currentTimeMillis(); - PackIndexWriter iw = BasePackIndexWriter.createVersion(indexStream, - getIndexVersion()); iw.write(sortByName(), packcsum); stats.timeWriting += System.currentTimeMillis() - writeStart; } @@ -2448,7 +2464,7 @@ public class PackWriter implements AutoCloseable { * object graph at selected commits. Writing a bitmap index is an optional * feature that not all pack users may require. *

- * Called after {@link #writeIndex(OutputStream)}. + * Called after {@link #writeIndex(PackIndexWriter)}. *

* To reduce memory internal state is cleared during this method, rendering * the PackWriter instance useless for anything further than a call to write -- cgit v1.2.3 From 9c6b36e97ba561d4737130c6427d6febc0a7d431 Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Tue, 29 Oct 2024 10:57:28 -0700 Subject: DfsInserter: Read minBytesForObjectSizeIndex only from repo config In general, JGit reads the configuration it needs from the repository configuration. minBytesForObjectSizeIndex is a special case with a setter for subclasses but that is unnecessary. Remove the setter and read the conf from the repo. Make the property final and read it directly from the conf (it is clearer than parsing a whole PackConfig to read a single value). Change-Id: I8d77247452ff65e6c431fdcfebb90ed2ce40aed1 --- .../jgit/internal/storage/dfs/DfsInserter.java | 28 ++++++---------------- 1 file changed, 7 insertions(+), 21 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsInserter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsInserter.java index 8f09261674..16315bf4f2 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsInserter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsInserter.java @@ -41,12 +41,13 @@ import org.eclipse.jgit.errors.CorruptObjectException; import org.eclipse.jgit.errors.IncorrectObjectTypeException; import org.eclipse.jgit.errors.LargeObjectException; import org.eclipse.jgit.internal.JGitText; -import org.eclipse.jgit.internal.storage.file.PackIndex; import org.eclipse.jgit.internal.storage.file.BasePackIndexWriter; +import org.eclipse.jgit.internal.storage.file.PackIndex; import org.eclipse.jgit.internal.storage.file.PackObjectSizeIndexWriter; import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.lib.AbbreviatedObjectId; import org.eclipse.jgit.lib.AnyObjectId; +import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectIdOwnerMap; @@ -54,7 +55,6 @@ import org.eclipse.jgit.lib.ObjectInserter; import org.eclipse.jgit.lib.ObjectLoader; import org.eclipse.jgit.lib.ObjectReader; import org.eclipse.jgit.lib.ObjectStream; -import org.eclipse.jgit.storage.pack.PackConfig; import org.eclipse.jgit.transport.PackedObjectInfo; import org.eclipse.jgit.util.BlockList; import org.eclipse.jgit.util.IO; @@ -71,6 +71,8 @@ public class DfsInserter extends ObjectInserter { private static final int INDEX_VERSION = 2; final DfsObjDatabase db; + + private final int minBytesForObjectSizeIndex; int compression = Deflater.BEST_COMPRESSION; List objectList; @@ -83,8 +85,6 @@ public class DfsInserter extends ObjectInserter { private boolean rollback; private boolean checkExisting = true; - private int minBytesForObjectSizeIndex = -1; - /** * Initialize a new inserter. * @@ -93,8 +93,9 @@ public class DfsInserter extends ObjectInserter { */ protected DfsInserter(DfsObjDatabase db) { this.db = db; - PackConfig pc = new PackConfig(db.getRepository()); - this.minBytesForObjectSizeIndex = pc.getMinBytesForObjSizeIndex(); + this.minBytesForObjectSizeIndex = db.getRepository().getConfig().getInt( + ConfigConstants.CONFIG_PACK_SECTION, + ConfigConstants.CONFIG_KEY_MIN_BYTES_OBJ_SIZE_INDEX, -1); } /** @@ -112,21 +113,6 @@ public class DfsInserter extends ObjectInserter { void setCompressionLevel(int compression) { this.compression = compression; } - - /** - * Set minimum size for an object to be included in the object size index. - * - *

- * Use 0 for all and -1 for nothing (the pack won't have object size index). - * - * @param minBytes - * only objects with size bigger or equal to this are included in - * the index. - */ - protected void setMinBytesForObjectSizeIndex(int minBytes) { - this.minBytesForObjectSizeIndex = minBytes; - } - @Override public DfsPackParser newPackParser(InputStream in) throws IOException { return new DfsPackParser(db, this, in); -- cgit v1.2.3 From d861ad1f5a1fdaf49b9c4b641e7d298b7acea6ad Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Tue, 9 Jul 2024 11:10:02 -0700 Subject: [errorprone] util.Stats: Add summary fragment to javadoc Errorprone complains about missing summary in these javadocs. [MissingSummary] A summary fragment is required; consider using the value of the @return b lock as a summary fragment instead. * @return variance of the added values ^ (see https://google.github.io/styleguide/javaguide.html#s7.2-summary-fragment) Did you mean '*Returns variance of the added values.'? Change-Id: I29d633ec95c18b17cc92b069dd1a94fbb2a75c94 --- .../src/org/eclipse/jgit/util/Stats.java | 25 ++++++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/Stats.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/Stats.java index d957deb34c..efa6e7ddc3 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/Stats.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/Stats.java @@ -43,14 +43,18 @@ public class Stats { } /** - * @return number of the added values + * Returns the number of added values + * + * @return the number of added values */ public int count() { return n; } /** - * @return minimum of the added values + * Returns the smallest value added + * + * @return the smallest value added */ public double min() { if (n < 1) { @@ -60,7 +64,9 @@ public class Stats { } /** - * @return maximum of the added values + * Returns the biggest value added + * + * @return the biggest value added */ public double max() { if (n < 1) { @@ -70,9 +76,10 @@ public class Stats { } /** - * @return average of the added values + * Returns the average of the added values + * + * @return the average of the added values */ - public double avg() { if (n < 1) { return Double.NaN; @@ -81,7 +88,9 @@ public class Stats { } /** - * @return variance of the added values + * Returns the variance of the added values + * + * @return the variance of the added values */ public double var() { if (n < 2) { @@ -91,7 +100,9 @@ public class Stats { } /** - * @return standard deviation of the added values + * Returns the standard deviation of the added values + * + * @return the standard deviation of the added values */ public double stddev() { return Math.sqrt(this.var()); -- cgit v1.2.3 From 793df21ed5c75613b0eb7f300857d56779d5c502 Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Tue, 9 Jul 2024 13:29:15 -0700 Subject: [errorprone] ByteArraySet: Add summary fragment to javadoc Reported by error prone. Change-Id: Icaa69c37d0cde19fc605cb3f3c5f9ed9abfb37d3 --- .../src/org/eclipse/jgit/treewalk/filter/ByteArraySet.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/filter/ByteArraySet.java b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/filter/ByteArraySet.java index bcf79a285d..33db6ea661 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/filter/ByteArraySet.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/filter/ByteArraySet.java @@ -13,12 +13,12 @@ package org.eclipse.jgit.treewalk.filter; -import org.eclipse.jgit.util.RawParseUtils; - import java.util.Arrays; import java.util.Set; import java.util.stream.Collectors; +import org.eclipse.jgit.util.RawParseUtils; + /** * Specialized set for byte arrays, interpreted as strings for use in * {@link PathFilterGroup.Group}. Most methods assume the hash is already know @@ -141,13 +141,19 @@ class ByteArraySet { } /** + * Returns number of arrays in the set + * * @return number of arrays in the set */ int size() { return size; } - /** @return true if {@link #size()} is 0. */ + /** + * Returns true if {@link #size()} is 0 + * + * @return true if {@link #size()} is 0 + */ boolean isEmpty() { return size == 0; } -- cgit v1.2.3 From c9752b0125c84248c468065eba64bf007abae81c Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Wed, 30 Oct 2024 15:36:09 -0700 Subject: [errorprone] PackWriter: Fix javadoc tag in new #writeIndex method We introduced this method recently and the javadoc is not correct: error: [InvalidInlineTag] This tag is invalid. @code{PackIndexWriter} instance to write the index Change-Id: I34ed3d8b5a121fea9b8163627b46ae4a289c9462 --- .../src/org/eclipse/jgit/internal/storage/pack/PackWriter.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java index 4c262b0e3a..f025d4e3d5 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java @@ -59,9 +59,9 @@ import org.eclipse.jgit.errors.SearchForReuseTimeout; import org.eclipse.jgit.errors.StoredObjectRepresentationNotAvailableException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.file.BasePackIndexWriter; +import org.eclipse.jgit.internal.storage.file.PackBitmapIndexBuilder; import org.eclipse.jgit.internal.storage.file.PackObjectSizeIndexWriter; import org.eclipse.jgit.internal.storage.file.PackReverseIndexWriter; -import org.eclipse.jgit.internal.storage.file.PackBitmapIndexBuilder; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.AsyncObjectSizeQueue; import org.eclipse.jgit.lib.BatchingProgressMonitor; @@ -1113,7 +1113,7 @@ public class PackWriter implements AutoCloseable { * the network do not need to create an index. * * @param iw - * an @code{PackIndexWriter} instance to write the index + * an {@link PackIndexWriter} instance to write the index * @throws java.io.IOException * the index data could not be written to the supplied stream. */ -- cgit v1.2.3 From 92a9641026306af3136530bc351da1c755df0c27 Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Wed, 30 Oct 2024 15:45:33 -0700 Subject: [errorprone] HttpConnection: Add missing summary in java The constants don't have summary and errorprone complains about it. Change-Id: Id1470ed9fd54cf7fd684045c5631acc1a8d450c2 --- .../org/eclipse/jgit/transport/http/HttpConnection.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/http/HttpConnection.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/http/HttpConnection.java index 125ee6cbe9..95b8221a8b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/http/HttpConnection.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/http/HttpConnection.java @@ -36,29 +36,39 @@ import org.eclipse.jgit.annotations.NonNull; */ public interface HttpConnection { /** + * HttpURLConnection#HTTP_OK + * * @see HttpURLConnection#HTTP_OK */ int HTTP_OK = java.net.HttpURLConnection.HTTP_OK; /** + * HttpURLConnection#HTTP_NOT_AUTHORITATIVE + * * @see HttpURLConnection#HTTP_NOT_AUTHORITATIVE * @since 5.8 */ int HTTP_NOT_AUTHORITATIVE = java.net.HttpURLConnection.HTTP_NOT_AUTHORITATIVE; /** + * HttpURLConnection#HTTP_MOVED_PERM + * * @see HttpURLConnection#HTTP_MOVED_PERM * @since 4.7 */ int HTTP_MOVED_PERM = java.net.HttpURLConnection.HTTP_MOVED_PERM; /** + * HttpURLConnection#HTTP_MOVED_TEMP + * * @see HttpURLConnection#HTTP_MOVED_TEMP * @since 4.9 */ int HTTP_MOVED_TEMP = java.net.HttpURLConnection.HTTP_MOVED_TEMP; /** + * HttpURLConnection#HTTP_SEE_OTHER + * * @see HttpURLConnection#HTTP_SEE_OTHER * @since 4.9 */ @@ -85,16 +95,22 @@ public interface HttpConnection { int HTTP_11_MOVED_PERM = 308; /** + * HttpURLConnection#HTTP_NOT_FOUND + * * @see HttpURLConnection#HTTP_NOT_FOUND */ int HTTP_NOT_FOUND = java.net.HttpURLConnection.HTTP_NOT_FOUND; /** + * HttpURLConnection#HTTP_UNAUTHORIZED + * * @see HttpURLConnection#HTTP_UNAUTHORIZED */ int HTTP_UNAUTHORIZED = java.net.HttpURLConnection.HTTP_UNAUTHORIZED; /** + * HttpURLConnection#HTTP_FORBIDDEN + * * @see HttpURLConnection#HTTP_FORBIDDEN */ int HTTP_FORBIDDEN = java.net.HttpURLConnection.HTTP_FORBIDDEN; -- cgit v1.2.3 From 7c092ab66e27b3ae68c7fed6d300b8645f7e4173 Mon Sep 17 00:00:00 2001 From: Saril Sudhakaran Date: Tue, 29 Oct 2024 00:17:01 -0500 Subject: DfsGarbageCollector: Add setter for reflog expiration time. JGit reftable writer/compator knows how to prune the history, but the DfsGarbageCollector doesn't expose the time limit. Add a method to DfsGarbageCollector to set the reflog time limit. This value is then passed to the reftable compactor. Callers usually pass here the value from gc.reflogExpire. The reflog block length is stored in 24 bits [1], limiting the size to 16MB. I have observed that in repositories with frequent commits, reflogs hit that size in 6-12 months. [1] https://git-scm.com/docs/reftable Bug: jgit-96 Change-Id: I8b32d6d2b2e1d8af8fb7d9f86225d75f1877eb2f --- .../storage/dfs/DfsGarbageCollectorTest.java | 98 ++++++++++++++++++++++ .../internal/storage/dfs/DfsGarbageCollector.java | 21 +++++ 2 files changed, 119 insertions(+) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollectorTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollectorTest.java index 2be11d32ea..3edd1056be 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollectorTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollectorTest.java @@ -6,6 +6,7 @@ import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.IN import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE; import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK; import static org.eclipse.jgit.internal.storage.pack.PackExt.REFTABLE; +import static org.eclipse.jgit.lib.Constants.OBJECT_ID_LENGTH; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -17,6 +18,8 @@ import static org.junit.Assert.fail; import java.io.IOException; import java.util.Arrays; import java.util.Collections; +import java.util.Date; +import java.util.GregorianCalendar; import java.util.concurrent.TimeUnit; import org.eclipse.jgit.internal.storage.commitgraph.CommitGraph; @@ -24,6 +27,7 @@ import org.eclipse.jgit.internal.storage.commitgraph.CommitGraphWriter; import org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource; import org.eclipse.jgit.internal.storage.file.PackBitmapIndex; import org.eclipse.jgit.internal.storage.pack.PackExt; +import org.eclipse.jgit.internal.storage.reftable.LogCursor; import org.eclipse.jgit.internal.storage.reftable.RefCursor; import org.eclipse.jgit.internal.storage.reftable.ReftableConfig; import org.eclipse.jgit.internal.storage.reftable.ReftableReader; @@ -37,6 +41,7 @@ import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.NullProgressMonitor; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectIdRef; +import org.eclipse.jgit.lib.PersonIdent; import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.revwalk.RevBlob; @@ -44,6 +49,7 @@ import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.storage.pack.PackConfig; import org.eclipse.jgit.transport.ReceiveCommand; +import org.eclipse.jgit.util.GitDateParser; import org.eclipse.jgit.util.SystemReader; import org.junit.After; import org.junit.Before; @@ -1275,6 +1281,90 @@ public class DfsGarbageCollectorTest { bitmapIndex.getXorBitmapCount() > 0); } + @Test + public void gitGCWithRefLogExpire() throws Exception { + String master = "refs/heads/master"; + RevCommit commit0 = commit().message("0").create(); + RevCommit commit1 = commit().message("1").parent(commit0).create(); + git.update(master, commit1); + DfsGarbageCollector gc = new DfsGarbageCollector(repo); + gc.setReftableConfig(new ReftableConfig()); + run(gc); + DfsPackDescription t1 = odb.newPack(INSERT); + Ref next = new ObjectIdRef.PeeledNonTag(Ref.Storage.LOOSE, + "refs/heads/next", commit0.copy()); + long currentDay = new Date().getTime(); + GregorianCalendar cal = new GregorianCalendar(SystemReader + .getInstance().getTimeZone(), SystemReader.getInstance() + .getLocale()); + long ten_days_ago = GitDateParser.parse("10 days ago",cal,SystemReader.getInstance() + .getLocale()).getTime() ; + long twenty_days_ago = GitDateParser.parse("20 days ago",cal,SystemReader.getInstance() + .getLocale()).getTime() ; + long thirty_days_ago = GitDateParser.parse("30 days ago",cal,SystemReader.getInstance() + .getLocale()).getTime() ;; + long fifty_days_ago = GitDateParser.parse("50 days ago",cal,SystemReader.getInstance() + .getLocale()).getTime() ; + PersonIdent who2 = new PersonIdent("J.Author", "authemail", currentDay, -8 * 60); + PersonIdent who3 = new PersonIdent("J.Author", "authemail", ten_days_ago, -8 * 60); + PersonIdent who4 = new PersonIdent("J.Author", "authemail", twenty_days_ago, -8 * 60); + PersonIdent who5 = new PersonIdent("J.Author", "authemail", thirty_days_ago, -8 * 60); + PersonIdent who6 = new PersonIdent("J.Author", "authemail", fifty_days_ago, -8 * 60); + + try (DfsOutputStream out = odb.writeFile(t1, REFTABLE)) { + ReftableWriter w = new ReftableWriter(out); + w.setMinUpdateIndex(42); + w.setMaxUpdateIndex(42); + w.begin(); + w.sortAndWriteRefs(Collections.singleton(next)); + w.writeLog("refs/heads/branch", 1, who2, ObjectId.zeroId(),id(2), "Branch Message"); + w.writeLog("refs/heads/branch1", 2, who3, ObjectId.zeroId(),id(3), "Branch Message1"); + w.writeLog("refs/heads/branch2", 2, who4, ObjectId.zeroId(),id(4), "Branch Message2"); + w.writeLog("refs/heads/branch3", 2, who5, ObjectId.zeroId(),id(5), "Branch Message3"); + w.writeLog("refs/heads/branch4", 2, who6, ObjectId.zeroId(),id(6), "Branch Message4"); + w.finish(); + t1.addFileExt(REFTABLE); + t1.setReftableStats(w.getStats()); + } + odb.commitPack(Collections.singleton(t1), null); + + gc = new DfsGarbageCollector(repo); + gc.setReftableConfig(new ReftableConfig()); + // Expire ref log entries older than 30 days + gc.setRefLogExpire(new Date(thirty_days_ago)); + run(gc); + + // Single GC pack present with all objects. + assertEquals(1, odb.getPacks().length); + DfsPackFile pack = odb.getPacks()[0]; + DfsPackDescription desc = pack.getPackDescription(); + + DfsReftable table = new DfsReftable(DfsBlockCache.getInstance(), desc); + try (DfsReader ctx = odb.newReader(); + ReftableReader rr = table.open(ctx); + RefCursor rc = rr.allRefs(); + LogCursor lc = rr.allLogs()) { + assertTrue(rc.next()); + assertEquals(master, rc.getRef().getName()); + assertEquals(commit1, rc.getRef().getObjectId()); + assertTrue(rc.next()); + assertEquals(next.getName(), rc.getRef().getName()); + assertEquals(commit0, rc.getRef().getObjectId()); + assertFalse(rc.next()); + assertTrue(lc.next()); + assertEquals(lc.getRefName(),"refs/heads/branch"); + assertTrue(lc.next()); + assertEquals(lc.getRefName(),"refs/heads/branch1"); + assertTrue(lc.next()); + assertEquals(lc.getRefName(),"refs/heads/branch2"); + // Old entries are purged + assertFalse(lc.next()); + + } + + } + + private RevCommit commitChain(RevCommit parent, int length) throws Exception { for (int i = 0; i < length; i++) { @@ -1364,4 +1454,12 @@ public class DfsGarbageCollectorTest { } return cnt; } + private static ObjectId id(int i) { + byte[] buf = new byte[OBJECT_ID_LENGTH]; + buf[0] = (byte) (i & 0xff); + buf[1] = (byte) ((i >>> 8) & 0xff); + buf[2] = (byte) ((i >>> 16) & 0xff); + buf[3] = (byte) (i >>> 24); + return ObjectId.fromRaw(buf); + } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java index a177669788..91440a6498 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java @@ -29,6 +29,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; import java.util.Collection; +import java.util.Date; import java.util.EnumSet; import java.util.GregorianCalendar; import java.util.HashSet; @@ -100,6 +101,7 @@ public class DfsGarbageCollector { private Set allTags; private Set nonHeads; private Set tagTargets; + private Date refLogExpire; /** * Initialize a garbage collector. @@ -200,6 +202,22 @@ public class DfsGarbageCollector { return this; } + + /** + * Set time limit to the reflog history. + *

+ * Garbage Collector prunes entries from reflog history older than {@code refLogExpire} + *

+ * + * @param refLogExpire + * instant in time which defines refLog expiration + * @return {@code this} + */ + public DfsGarbageCollector setRefLogExpire(Date refLogExpire) { + this.refLogExpire = refLogExpire ; + return this; + } + /** * Set maxUpdateIndex for the initial reftable created during conversion. * @@ -741,6 +759,9 @@ public class DfsGarbageCollector { compact.addAll(stack.readers()); compact.setIncludeDeletes(includeDeletes); compact.setConfig(configureReftable(reftableConfig, out)); + if(refLogExpire != null ){ + compact.setReflogExpireOldestReflogTimeMillis(refLogExpire.getTime()); + } compact.compact(); pack.addFileExt(REFTABLE); pack.setReftableStats(compact.getStats()); -- cgit v1.2.3 From 4ec2413e14043f6d9a0ebd6616fdef2464fe4880 Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Thu, 31 Oct 2024 12:55:31 -0700 Subject: [errorprone] RefDatabase: #getConflictingNames immutable return Errorprone reports that: This method returns both mutable and immutable collections or maps from different paths. This may be confusing for users of the method. Return always an immutable collection. Change-Id: Id48f3645fd06c8bc72212af180d7d02c7e0b7632 --- org.eclipse.jgit/src/org/eclipse/jgit/lib/RefDatabase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefDatabase.java index 2cf24185c7..114246beb2 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefDatabase.java @@ -160,7 +160,7 @@ public abstract class RefDatabase { if (existing.startsWith(prefix)) conflicting.add(existing); - return conflicting; + return Collections.unmodifiableList(conflicting); } /** -- cgit v1.2.3 From 562d825f26134ee3db30c1791e918fa8c32095f1 Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Thu, 31 Oct 2024 12:17:09 -0700 Subject: [errorprone] Remove deprecated security manager Errorprone warns about this deprecated classes. The recommendation is stop using SecurityManager all together. The Security Manager is deprecated and subject to removal in a future release. There is no replacement for the Security Manager. See JEP 411 [1] for discussion and alternatives. [1] https://openjdk.org/jeps/411 Change-Id: I3c67136e97d13cf24b85e41d94408631c26e8be8 --- .../api/SecurityManagerMissingPermissionsTest.java | 126 --------------- .../org/eclipse/jgit/api/SecurityManagerTest.java | 173 --------------------- .../org/eclipse/jgit/internal/JGitText.properties | 2 - .../src/org/eclipse/jgit/internal/JGitText.java | 2 - .../eclipse/jgit/transport/SshSessionFactory.java | 7 +- org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java | 35 +---- .../src/org/eclipse/jgit/util/FS_Win32_Cygwin.java | 11 +- .../src/org/eclipse/jgit/util/SystemReader.java | 6 +- .../eclipse/jgit/util/io/ThrowingPrintWriter.java | 7 +- 9 files changed, 7 insertions(+), 362 deletions(-) delete mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/api/SecurityManagerMissingPermissionsTest.java delete mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/api/SecurityManagerTest.java (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/SecurityManagerMissingPermissionsTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/SecurityManagerMissingPermissionsTest.java deleted file mode 100644 index d0fbdbd090..0000000000 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/SecurityManagerMissingPermissionsTest.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright (c) 2019 Alex Jitianu and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ -package org.eclipse.jgit.api; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.IOException; -import java.io.PrintStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.security.Policy; -import java.util.Collections; - -import org.eclipse.jgit.junit.RepositoryTestCase; -import org.eclipse.jgit.util.FileUtils; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -/** - * Tests that using a SecurityManager does not result in errors logged. - */ -public class SecurityManagerMissingPermissionsTest extends RepositoryTestCase { - - /** - * Collects all logging sent to the logging system. - */ - private final ByteArrayOutputStream errorOutput = new ByteArrayOutputStream(); - - private SecurityManager originalSecurityManager; - - private PrintStream defaultErrorOutput; - - @Override - @Before - public void setUp() throws Exception { - originalSecurityManager = System.getSecurityManager(); - - // slf4j-simple logs to System.err, redirect it to enable asserting - // logged errors - defaultErrorOutput = System.err; - System.setErr(new PrintStream(errorOutput)); - - refreshPolicyAllPermission(Policy.getPolicy()); - System.setSecurityManager(new SecurityManager()); - super.setUp(); - } - - /** - * If a SecurityManager is active a lot of {@link java.io.FilePermission} - * errors are thrown and logged while initializing a repository. - * - * @throws Exception - */ - @Test - public void testCreateNewRepos_MissingPermissions() throws Exception { - File wcTree = new File(getTemporaryDirectory(), - "CreateNewRepositoryTest_testCreateNewRepos"); - - File marker = new File(getTemporaryDirectory(), "marker"); - Files.write(marker.toPath(), Collections.singletonList("Can write")); - assertTrue("Can write in test directory", marker.isFile()); - FileUtils.delete(marker); - assertFalse("Can delete in test direcory", marker.exists()); - - Git git = Git.init().setBare(false) - .setDirectory(new File(wcTree.getAbsolutePath())).call(); - - addRepoToClose(git.getRepository()); - - assertEquals("", errorOutput.toString()); - } - - @Override - @After - public void tearDown() throws Exception { - System.setSecurityManager(originalSecurityManager); - System.setErr(defaultErrorOutput); - super.tearDown(); - } - - /** - * Refresh the Java Security Policy. - * - * @param policy - * the policy object - * - * @throws IOException - * if the temporary file that contains the policy could not be - * created - */ - private static void refreshPolicyAllPermission(Policy policy) - throws IOException { - // Starting with an all permissions policy. - String policyString = "grant { permission java.security.AllPermission; };"; - - // Do not use TemporaryFilesFactory, it will create a dependency cycle - Path policyFile = Files.createTempFile("testpolicy", ".txt"); - - try { - Files.write(policyFile, Collections.singletonList(policyString)); - System.setProperty("java.security.policy", - policyFile.toUri().toURL().toString()); - policy.refresh(); - } finally { - try { - Files.delete(policyFile); - } catch (IOException e) { - // Do not log; the test tests for no logging having occurred - e.printStackTrace(); - } - } - } - -} diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/SecurityManagerTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/SecurityManagerTest.java deleted file mode 100644 index 2b930a1133..0000000000 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/SecurityManagerTest.java +++ /dev/null @@ -1,173 +0,0 @@ -/* - * Copyright (C) 2019 Nail Samatov and others - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ -package org.eclipse.jgit.api; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; - -import java.io.File; -import java.io.FilePermission; -import java.io.IOException; -import java.lang.reflect.ReflectPermission; -import java.nio.file.Files; -import java.security.Permission; -import java.security.SecurityPermission; -import java.util.ArrayList; -import java.util.List; -import java.util.PropertyPermission; -import java.util.logging.LoggingPermission; - -import javax.security.auth.AuthPermission; - -import org.eclipse.jgit.api.errors.GitAPIException; -import org.eclipse.jgit.junit.JGitTestUtil; -import org.eclipse.jgit.junit.MockSystemReader; -import org.eclipse.jgit.junit.SeparateClassloaderTestRunner; -import org.eclipse.jgit.revwalk.RevCommit; -import org.eclipse.jgit.treewalk.TreeWalk; -import org.eclipse.jgit.util.FileUtils; -import org.eclipse.jgit.util.SystemReader; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; - -/** - *

- * Tests if jgit works if SecurityManager is enabled. - *

- * - *

- * Note: JGit's classes shouldn't be used before SecurityManager is configured. - * If you use some JGit's class before SecurityManager is replaced then part of - * the code can be invoked outside of our custom SecurityManager and this test - * becomes useless. - *

- * - *

- * For example the class {@link org.eclipse.jgit.util.FS} is used widely in jgit - * sources. It contains DETECTED static field. At the first usage of the class - * FS the field DETECTED is initialized and during initialization many system - * operations that SecurityManager can forbid are invoked. - *

- * - *

- * For this reason this test doesn't extend LocalDiskRepositoryTestCase (it uses - * JGit's classes in setUp() method) and other JGit's utility classes. It's done - * to affect SecurityManager as less as possible. - *

- * - *

- * We use SeparateClassloaderTestRunner to isolate FS.DETECTED field - * initialization between different tests run. - *

- */ -@RunWith(SeparateClassloaderTestRunner.class) -public class SecurityManagerTest { - private File root; - - private SecurityManager originalSecurityManager; - - private List permissions = new ArrayList<>(); - - @Before - public void setUp() throws Exception { - // Create working directory - SystemReader.setInstance(new MockSystemReader()); - root = Files.createTempDirectory("jgit-security").toFile(); - - // Add system permissions - permissions.add(new RuntimePermission("*")); - permissions.add(new SecurityPermission("*")); - permissions.add(new AuthPermission("*")); - permissions.add(new ReflectPermission("*")); - permissions.add(new PropertyPermission("*", "read,write")); - permissions.add(new LoggingPermission("control", null)); - - permissions.add(new FilePermission( - System.getProperty("java.home") + "/-", "read")); - - String tempDir = System.getProperty("java.io.tmpdir"); - permissions.add(new FilePermission(tempDir, "read,write,delete")); - permissions - .add(new FilePermission(tempDir + "/-", "read,write,delete")); - - // Add permissions to dependent jar files. - String classPath = System.getProperty("java.class.path"); - if (classPath != null) { - for (String path : classPath.split(File.pathSeparator)) { - permissions.add(new FilePermission(path, "read")); - } - } - // Add permissions to jgit class files. - String jgitSourcesRoot = new File(System.getProperty("user.dir")) - .getParent(); - permissions.add(new FilePermission(jgitSourcesRoot + "/-", "read")); - - // Add permissions to working dir for jgit. Our git repositories will be - // initialized and cloned here. - permissions.add(new FilePermission(root.getPath() + "/-", - "read,write,delete,execute")); - - // Replace Security Manager - originalSecurityManager = System.getSecurityManager(); - System.setSecurityManager(new SecurityManager() { - - @Override - public void checkPermission(Permission requested) { - for (Permission permission : permissions) { - if (permission.implies(requested)) { - return; - } - } - - super.checkPermission(requested); - } - }); - } - - @After - public void tearDown() throws Exception { - System.setSecurityManager(originalSecurityManager); - - // Note: don't use this method before security manager is replaced in - // setUp() method. The method uses FS.DETECTED internally and can affect - // the test. - FileUtils.delete(root, FileUtils.RECURSIVE | FileUtils.RETRY); - } - - @Test - public void testInitAndClone() throws IOException, GitAPIException { - File remote = new File(root, "remote"); - File local = new File(root, "local"); - - try (Git git = Git.init().setDirectory(remote).call()) { - JGitTestUtil.write(new File(remote, "hello.txt"), "Hello world!"); - git.add().addFilepattern(".").call(); - git.commit().setMessage("Initial commit").call(); - } - - try (Git git = Git.cloneRepository().setURI(remote.toURI().toString()) - .setDirectory(local).call()) { - assertTrue(new File(local, ".git").exists()); - - JGitTestUtil.write(new File(local, "hi.txt"), "Hi!"); - git.add().addFilepattern(".").call(); - RevCommit commit1 = git.commit().setMessage("Commit on local repo") - .call(); - assertEquals("Commit on local repo", commit1.getFullMessage()); - assertNotNull(TreeWalk.forPath(git.getRepository(), "hello.txt", - commit1.getTree())); - } - - } - -} diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties index b1450ef057..024ca77f21 100644 --- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties +++ b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties @@ -640,8 +640,6 @@ readFileStoreAttributesFailed=Reading FileStore attributes from user config fail readerIsRequired=Reader is required readingObjectsFromLocalRepositoryFailed=reading objects from local repository failed: {0} readLastModifiedFailed=Reading lastModified of {0} failed -readPipeIsNotAllowed=FS.readPipe() isn't allowed for command ''{0}''. Working directory: ''{1}''. -readPipeIsNotAllowedRequiredPermission=FS.readPipe() isn't allowed for command ''{0}''. Working directory: ''{1}''. Required permission: {2}. readTimedOut=Read timed out after {0} ms receivePackObjectTooLarge1=Object too large, rejecting the pack. Max object size limit is {0} bytes. receivePackObjectTooLarge2=Object too large ({0} bytes), rejecting the pack. Max object size limit is {1} bytes. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java index 310962b967..0980219e25 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java @@ -670,8 +670,6 @@ public class JGitText extends TranslationBundle { /***/ public String readerIsRequired; /***/ public String readingObjectsFromLocalRepositoryFailed; /***/ public String readLastModifiedFailed; - /***/ public String readPipeIsNotAllowed; - /***/ public String readPipeIsNotAllowedRequiredPermission; /***/ public String readTimedOut; /***/ public String receivePackObjectTooLarge1; /***/ public String receivePackObjectTooLarge2; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/SshSessionFactory.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/SshSessionFactory.java index a0194ea8b1..8120df0698 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/SshSessionFactory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/SshSessionFactory.java @@ -11,8 +11,6 @@ package org.eclipse.jgit.transport; -import java.security.AccessController; -import java.security.PrivilegedAction; import java.util.Iterator; import java.util.ServiceLoader; @@ -99,9 +97,8 @@ public abstract class SshSessionFactory { * @since 5.2 */ public static String getLocalUserName() { - return AccessController - .doPrivileged((PrivilegedAction) () -> SystemReader - .getInstance().getProperty(Constants.OS_USER_NAME_KEY)); + return SystemReader.getInstance() + .getProperty(Constants.OS_USER_NAME_KEY); } /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java index 860c1c92fd..59bbacfa76 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java @@ -30,7 +30,6 @@ import java.nio.file.InvalidPathException; import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; import java.nio.file.attribute.FileTime; -import java.security.AccessControlException; import java.text.MessageFormat; import java.time.Duration; import java.time.Instant; @@ -261,31 +260,6 @@ public abstract class FS { private static final AtomicInteger threadNumber = new AtomicInteger(1); - /** - * Don't use the default thread factory of the ForkJoinPool for the - * CompletableFuture; it runs without any privileges, which causes - * trouble if a SecurityManager is present. - *

- * Instead use normal daemon threads. They'll belong to the - * SecurityManager's thread group, or use the one of the calling thread, - * as appropriate. - *

- * - * @see java.util.concurrent.Executors#newCachedThreadPool() - */ - private static final ExecutorService FUTURE_RUNNER = new ThreadPoolExecutor( - 5, 5, 30L, TimeUnit.SECONDS, - new LinkedBlockingQueue<>(), - runnable -> { - Thread t = new Thread(runnable, - "JGit-FileStoreAttributeReader-" //$NON-NLS-1$ - + threadNumber.getAndIncrement()); - // Make sure these threads don't prevent application/JVM - // shutdown. - t.setDaemon(true); - return t; - }); - /** * Use a separate executor with at most one thread to synchronize * writing to the config. We write asynchronously since the config @@ -463,7 +437,7 @@ public abstract class FS { locks.remove(s); } return attributes; - }, FUTURE_RUNNER); + }); f = f.exceptionally(e -> { LOG.error(e.getLocalizedMessage(), e); return Optional.empty(); @@ -1391,13 +1365,6 @@ public abstract class FS { } } catch (IOException e) { LOG.error("Caught exception in FS.readPipe()", e); //$NON-NLS-1$ - } catch (AccessControlException e) { - LOG.warn(MessageFormat.format( - JGitText.get().readPipeIsNotAllowedRequiredPermission, - command, dir, e.getPermission())); - } catch (SecurityException e) { - LOG.warn(MessageFormat.format(JGitText.get().readPipeIsNotAllowed, - command, dir)); } if (debug) { LOG.debug("readpipe returns null"); //$NON-NLS-1$ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_Win32_Cygwin.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_Win32_Cygwin.java index 635351ac84..237879110a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_Win32_Cygwin.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS_Win32_Cygwin.java @@ -14,8 +14,6 @@ import static java.nio.charset.StandardCharsets.UTF_8; import java.io.File; import java.io.OutputStream; -import java.security.AccessController; -import java.security.PrivilegedAction; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -43,10 +41,7 @@ public class FS_Win32_Cygwin extends FS_Win32 { * @return true if cygwin is found */ public static boolean isCygwin() { - final String path = AccessController - .doPrivileged((PrivilegedAction) () -> System - .getProperty("java.library.path") //$NON-NLS-1$ - ); + final String path = System.getProperty("java.library.path"); //$NON-NLS-1$ if (path == null) return false; File found = FS.searchPath(path, "cygpath.exe"); //$NON-NLS-1$ @@ -99,9 +94,7 @@ public class FS_Win32_Cygwin extends FS_Win32 { @Override protected File userHomeImpl() { - final String home = AccessController.doPrivileged( - (PrivilegedAction) () -> System.getenv("HOME") //$NON-NLS-1$ - ); + final String home = System.getenv("HOME"); //$NON-NLS-1$ if (home == null || home.length() == 0) return super.userHomeImpl(); return resolve(new File("."), home); //$NON-NLS-1$ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/SystemReader.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/SystemReader.java index ed62c71371..03ed925617 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/SystemReader.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/SystemReader.java @@ -23,8 +23,6 @@ import java.nio.charset.UnsupportedCharsetException; import java.nio.file.InvalidPathException; import java.nio.file.Path; import java.nio.file.Paths; -import java.security.AccessController; -import java.security.PrivilegedAction; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.Locale; @@ -670,9 +668,7 @@ public abstract class SystemReader { } private String getOsName() { - return AccessController.doPrivileged( - (PrivilegedAction) () -> getProperty("os.name") //$NON-NLS-1$ - ); + return getProperty("os.name"); //$NON-NLS-1$ } /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/ThrowingPrintWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/ThrowingPrintWriter.java index 4764676c88..13982b133c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/ThrowingPrintWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/ThrowingPrintWriter.java @@ -11,8 +11,6 @@ package org.eclipse.jgit.util.io; import java.io.IOException; import java.io.Writer; -import java.security.AccessController; -import java.security.PrivilegedAction; import org.eclipse.jgit.util.SystemReader; @@ -35,10 +33,7 @@ public class ThrowingPrintWriter extends Writer { */ public ThrowingPrintWriter(Writer out) { this.out = out; - LF = AccessController - .doPrivileged((PrivilegedAction) () -> SystemReader - .getInstance().getProperty("line.separator") //$NON-NLS-1$ - ); + LF = SystemReader.getInstance().getProperty("line.separator"); //$NON-NLS-1$ } @Override -- cgit v1.2.3 From facddaccf591c3f3da9384269e1b592ca179a794 Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Thu, 31 Oct 2024 12:50:00 -0700 Subject: [errorprone] BaseRepositoryBuilder: Use #split(sep, limit) String.split(String) and Pattern.split(CharSequence) have surprising behaviour [1]. We use one of the recommended replacements: #split(sep, limit). [1] https://errorprone.info/bugpattern/StringSplitter Change-Id: Ie1cf7590bd8660d21c79c5c3c1bc2765e5d9462b --- .../src/org/eclipse/jgit/lib/BaseRepositoryBuilder.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/BaseRepositoryBuilder.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/BaseRepositoryBuilder.java index d232be6276..0c1da83dfb 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/BaseRepositoryBuilder.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/BaseRepositoryBuilder.java @@ -15,6 +15,7 @@ import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_BARE; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_WORKTREE; import static org.eclipse.jgit.lib.Constants.CONFIG; import static org.eclipse.jgit.lib.Constants.DOT_GIT; +import static org.eclipse.jgit.lib.Constants.GITDIR_FILE; import static org.eclipse.jgit.lib.Constants.GIT_ALTERNATE_OBJECT_DIRECTORIES_KEY; import static org.eclipse.jgit.lib.Constants.GIT_CEILING_DIRECTORIES_KEY; import static org.eclipse.jgit.lib.Constants.GIT_COMMON_DIR_KEY; @@ -23,7 +24,6 @@ import static org.eclipse.jgit.lib.Constants.GIT_INDEX_FILE_KEY; import static org.eclipse.jgit.lib.Constants.GIT_OBJECT_DIRECTORY_KEY; import static org.eclipse.jgit.lib.Constants.GIT_WORK_TREE_KEY; import static org.eclipse.jgit.lib.Constants.OBJECTS; -import static org.eclipse.jgit.lib.Constants.GITDIR_FILE; import java.io.File; import java.io.IOException; @@ -485,7 +485,7 @@ public class BaseRepositoryBuilder Date: Fri, 1 Nov 2024 08:58:27 -0700 Subject: DfsPackCompactor: write object size index Currently the compactor is not writing the object size index for packs. As it is using PackWriter to generate the packs, it needs to explicitely call the writes of each extension. Invoke writeObjectSizeIndex in the compactor. The pack writer will write one if the configuration says so. Change-Id: I8d6bbbb5bd67bfc7dd511aa76463512b1e86a45d --- .../internal/storage/dfs/DfsPackCompacterTest.java | 44 ++++++++++++++++++++++ .../internal/storage/dfs/DfsPackCompactor.java | 17 +++++++++ 2 files changed, 61 insertions(+) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackCompacterTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackCompacterTest.java index c516e30f50..c3b6aa85a2 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackCompacterTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackCompacterTest.java @@ -12,13 +12,18 @@ package org.eclipse.jgit.internal.storage.dfs; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.COMPACT; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.INSERT; +import static org.eclipse.jgit.internal.storage.pack.PackExt.OBJECT_SIZE_INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import java.io.IOException; +import java.util.Arrays; +import java.util.Optional; import org.eclipse.jgit.junit.TestRepository; +import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.revwalk.RevCommit; import org.junit.Before; import org.junit.Test; @@ -98,6 +103,40 @@ public class DfsPackCompacterTest { pack.getPackDescription().getEstimatedPackSize()); } + @Test + public void testObjectSizeIndexWritten() throws Exception { + writeObjectSizeIndex(repo, true); + RevCommit commit0 = commit().message("0").create(); + RevCommit commit1 = commit().message("1").parent(commit0).create(); + git.update("master", commit1); + + compact(); + + Optional compactPack = Arrays.stream(odb.getPacks()) + .filter(pack -> pack.getPackDescription() + .getPackSource() == COMPACT) + .findFirst(); + assertTrue(compactPack.isPresent()); + assertTrue(compactPack.get().getPackDescription().hasFileExt(OBJECT_SIZE_INDEX)); + } + + @Test + public void testObjectSizeIndexNotWritten() throws Exception { + writeObjectSizeIndex(repo, false); + RevCommit commit0 = commit().message("0").create(); + RevCommit commit1 = commit().message("1").parent(commit0).create(); + git.update("master", commit1); + + compact(); + + Optional compactPack = Arrays.stream(odb.getPacks()) + .filter(pack -> pack.getPackDescription() + .getPackSource() == COMPACT) + .findFirst(); + assertTrue(compactPack.isPresent()); + assertFalse(compactPack.get().getPackDescription().hasFileExt(OBJECT_SIZE_INDEX)); + } + private TestRepository.CommitBuilder commit() { return git.commit(); } @@ -108,4 +147,9 @@ public class DfsPackCompacterTest { compactor.compact(null); odb.clearCache(); } + + private static void writeObjectSizeIndex(DfsRepository repo, boolean should) { + repo.getConfig().setInt(ConfigConstants.CONFIG_PACK_SECTION, null, + ConfigConstants.CONFIG_KEY_MIN_BYTES_OBJ_SIZE_INDEX, should ? 0 : -1); + } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java index b32cddae77..f9c01b9d6e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java @@ -12,6 +12,7 @@ package org.eclipse.jgit.internal.storage.dfs; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.COMPACT; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC; +import static org.eclipse.jgit.internal.storage.pack.PackExt.OBJECT_SIZE_INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK; import static org.eclipse.jgit.internal.storage.pack.PackExt.REFTABLE; import static org.eclipse.jgit.internal.storage.pack.StoredObjectRepresentation.PACK_DELTA; @@ -44,6 +45,7 @@ import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.storage.pack.PackConfig; import org.eclipse.jgit.storage.pack.PackStatistics; import org.eclipse.jgit.util.BlockList; +import org.eclipse.jgit.util.io.CountingOutputStream; /** * Combine several pack files into one pack. @@ -247,6 +249,7 @@ public class DfsPackCompactor { try { writePack(objdb, outDesc, pw, pm); writeIndex(objdb, outDesc, pw); + writeObjectSizeIndex(objdb, outDesc, pw); PackStatistics stats = pw.getStatistics(); @@ -459,6 +462,20 @@ public class DfsPackCompactor { pw.writeIndex(objdb.getPackIndexWriter(pack, pw.getIndexVersion())); } + private static void writeObjectSizeIndex(DfsObjDatabase objdb, + DfsPackDescription pack, + PackWriter pw) throws IOException { + try (DfsOutputStream out = objdb.writeFile(pack, OBJECT_SIZE_INDEX)) { + CountingOutputStream cnt = new CountingOutputStream(out); + pw.writeObjectSizeIndex(cnt); + if (cnt.getCount() > 0) { + pack.addFileExt(OBJECT_SIZE_INDEX); + pack.setFileSize(OBJECT_SIZE_INDEX, cnt.getCount()); + pack.setBlockSize(OBJECT_SIZE_INDEX, out.blockSize()); + } + } + } + static ReftableConfig configureReftable(ReftableConfig cfg, DfsOutputStream out) { int bs = out.blockSize(); -- cgit v1.2.3 From 4f48a5b1e3842aeeae6b8f0f5d517819373221a4 Mon Sep 17 00:00:00 2001 From: Nasser Grainawi Date: Mon, 21 Oct 2024 20:26:35 -0600 Subject: Add Union merge strategy support Allow users to specify the `union` strategy in their .gitattributes file in order to keep lines from both versions of a conflict [1]. [1] https://git-scm.com/docs/gitattributes.html#Documentation/gitattributes.txt-union Change-Id: I74cecceb2db819a8551b95fb10dfe7c2b160b709 --- .../attributes/merge/MergeGitAttributeTest.java | 45 +++ .../jgit/merge/MergeAlgorithmUnionTest.java | 328 +++++++++++++++++++++ org.eclipse.jgit/.settings/.api_filters | 16 + .../src/org/eclipse/jgit/lib/Constants.java | 7 + .../eclipse/jgit/merge/ContentMergeStrategy.java | 11 +- .../src/org/eclipse/jgit/merge/MergeAlgorithm.java | 13 +- .../src/org/eclipse/jgit/merge/ResolveMerger.java | 14 +- 7 files changed, 430 insertions(+), 4 deletions(-) create mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/merge/MergeAlgorithmUnionTest.java (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/attributes/merge/MergeGitAttributeTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/attributes/merge/MergeGitAttributeTest.java index 009ca8a152..ac30c6c526 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/attributes/merge/MergeGitAttributeTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/attributes/merge/MergeGitAttributeTest.java @@ -267,6 +267,51 @@ public class MergeGitAttributeTest extends RepositoryTestCase { } } + @Test + public void mergeTextualFile_SetUnionMerge() throws NoWorkTreeException, + NoFilepatternException, GitAPIException, IOException { + try (Git git = createRepositoryBinaryConflict(g -> { + try { + writeTrashFile(".gitattributes", "*.cat merge=union"); + writeTrashFile("main.cat", "A\n" + "B\n" + "C\n" + "D\n"); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }, g -> { + try { + writeTrashFile("main.cat", "A\n" + "G\n" + "C\n" + "F\n"); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }, g -> { + try { + writeTrashFile("main.cat", "A\n" + "E\n" + "C\n" + "D\n"); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + })) { + // Check that the merge attribute is set to union + assertAddMergeAttributeCustom(REFS_HEADS_LEFT, "main.cat", "union"); + assertAddMergeAttributeCustom(REFS_HEADS_RIGHT, "main.cat", + "union"); + + checkoutBranch(REFS_HEADS_LEFT); + // Merge refs/heads/left -> refs/heads/right + + MergeResult mergeResult = git.merge() + .include(git.getRepository().resolve(REFS_HEADS_RIGHT)) + .call(); + assertEquals(MergeStatus.MERGED, mergeResult.getMergeStatus()); + + // Check that the file is the union of both branches (no conflict + // marker added) + String result = read(writeTrashFile("res.cat", + "A\n" + "G\n" + "E\n" + "C\n" + "F\n")); + assertEquals(result, read(git.getRepository().getWorkTree().toPath() + .resolve("main.cat").toFile())); + } + } + @Test public void mergeBinaryFile_NoAttr_Conflict() throws IllegalStateException, IOException, NoHeadException, ConcurrentRefUpdateException, diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/merge/MergeAlgorithmUnionTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/merge/MergeAlgorithmUnionTest.java new file mode 100644 index 0000000000..3a8af7a00e --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/merge/MergeAlgorithmUnionTest.java @@ -0,0 +1,328 @@ +/* + * Copyright (C) 2024 Qualcomm Innovation Center, Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.merge; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.assertEquals; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +import org.eclipse.jgit.diff.RawText; +import org.eclipse.jgit.diff.RawTextComparator; +import org.eclipse.jgit.lib.Constants; +import org.junit.Assume; +import org.junit.Test; +import org.junit.experimental.theories.DataPoints; +import org.junit.experimental.theories.Theories; +import org.junit.runner.RunWith; + +@RunWith(Theories.class) +public class MergeAlgorithmUnionTest { + MergeFormatter fmt = new MergeFormatter(); + + private final boolean newlineAtEnd; + + @DataPoints + public static boolean[] newlineAtEndDataPoints = { false, true }; + + public MergeAlgorithmUnionTest(boolean newlineAtEnd) { + this.newlineAtEnd = newlineAtEnd; + } + + /** + * Check for a conflict where the second text was changed similar to the + * first one, but the second texts modification covers one more line. + * + * @throws java.io.IOException + */ + @Test + public void testTwoConflictingModifications() throws IOException { + assertEquals(t("abZZdefghij"), + merge("abcdefghij", "abZdefghij", "aZZdefghij")); + } + + /** + * Test a case where we have three consecutive chunks. The first text + * modifies all three chunks. The second text modifies the first and the + * last chunk. This should be reported as one conflicting region. + * + * @throws java.io.IOException + */ + @Test + public void testOneAgainstTwoConflictingModifications() throws IOException { + assertEquals(t("aZZcZefghij"), + merge("abcdefghij", "aZZZefghij", "aZcZefghij")); + } + + /** + * Test a merge where only the second text contains modifications. Expect as + * merge result the second text. + * + * @throws java.io.IOException + */ + @Test + public void testNoAgainstOneModification() throws IOException { + assertEquals(t("aZcZefghij"), + merge("abcdefghij", "abcdefghij", "aZcZefghij")); + } + + /** + * Both texts contain modifications but not on the same chunks. Expect a + * non-conflict merge result. + * + * @throws java.io.IOException + */ + @Test + public void testTwoNonConflictingModifications() throws IOException { + assertEquals(t("YbZdefghij"), + merge("abcdefghij", "abZdefghij", "Ybcdefghij")); + } + + /** + * Merge two complicated modifications. The merge algorithm has to extend + * and combine conflicting regions to get to the expected merge result. + * + * @throws java.io.IOException + */ + @Test + public void testTwoComplicatedModifications() throws IOException { + assertEquals(t("aZZZZfZhZjbYdYYYYiY"), + merge("abcdefghij", "aZZZZfZhZj", "abYdYYYYiY")); + } + + /** + * Merge two modifications with a shared delete at the end. The underlying + * diff algorithm has to provide consistent edit results to get the expected + * merge result. + * + * @throws java.io.IOException + */ + @Test + public void testTwoModificationsWithSharedDelete() throws IOException { + assertEquals(t("Cb}n}"), merge("ab}n}n}", "ab}n}", "Cb}n}")); + } + + /** + * Merge modifications with a shared insert in the middle. The underlying + * diff algorithm has to provide consistent edit results to get the expected + * merge result. + * + * @throws java.io.IOException + */ + @Test + public void testModificationsWithMiddleInsert() throws IOException { + assertEquals(t("aBcd123123uvwxPq"), + merge("abcd123uvwxpq", "aBcd123123uvwxPq", "abcd123123uvwxpq")); + } + + /** + * Merge modifications with a shared delete in the middle. The underlying + * diff algorithm has to provide consistent edit results to get the expected + * merge result. + * + * @throws java.io.IOException + */ + @Test + public void testModificationsWithMiddleDelete() throws IOException { + assertEquals(t("Abz}z123Q"), + merge("abz}z}z123q", "Abz}z123Q", "abz}z123q")); + } + + @Test + public void testInsertionAfterDeletion() throws IOException { + assertEquals(t("abcd"), merge("abd", "ad", "abcd")); + } + + @Test + public void testInsertionBeforeDeletion() throws IOException { + assertEquals(t("acbd"), merge("abd", "ad", "acbd")); + } + + /** + * Test a conflicting region at the very start of the text. + * + * @throws java.io.IOException + */ + @Test + public void testConflictAtStart() throws IOException { + assertEquals(t("ZYbcdefghij"), + merge("abcdefghij", "Zbcdefghij", "Ybcdefghij")); + } + + /** + * Test a conflicting region at the very end of the text. + * + * @throws java.io.IOException + */ + @Test + public void testConflictAtEnd() throws IOException { + assertEquals(t("abcdefghiZY"), + merge("abcdefghij", "abcdefghiZ", "abcdefghiY")); + } + + /** + * Check for a conflict where the second text was changed similar to the + * first one, but the second texts modification covers one more line. + * + * @throws java.io.IOException + */ + @Test + public void testSameModification() throws IOException { + assertEquals(t("abZdefghij"), + merge("abcdefghij", "abZdefghij", "abZdefghij")); + } + + /** + * Check that a deleted vs. a modified line shows up as conflict (see Bug + * 328551) + * + * @throws java.io.IOException + */ + @Test + public void testDeleteVsModify() throws IOException { + assertEquals(t("abZdefghij"), + merge("abcdefghij", "abdefghij", "abZdefghij")); + } + + @Test + public void testInsertVsModify() throws IOException { + assertEquals(t("abZXY"), merge("ab", "abZ", "aXY")); + } + + @Test + public void testAdjacentModifications() throws IOException { + assertEquals(t("aZcbYd"), merge("abcd", "aZcd", "abYd")); + } + + @Test + public void testSeparateModifications() throws IOException { + assertEquals(t("aZcYe"), merge("abcde", "aZcde", "abcYe")); + } + + @Test + public void testBlankLines() throws IOException { + assertEquals(t("aZc\nYe"), merge("abc\nde", "aZc\nde", "abc\nYe")); + } + + /** + * Test merging two contents which do one similar modification and one + * insertion is only done by one side, in the middle. Between modification + * and insertion is a block which is common between the two contents and the + * common base + * + * @throws java.io.IOException + */ + @Test + public void testTwoSimilarModsAndOneInsert() throws IOException { + assertEquals(t("aBcDde"), merge("abcde", "aBcde", "aBcDde")); + + assertEquals(t("IAAAJCAB"), merge("iACAB", "IACAB", "IAAAJCAB")); + + assertEquals(t("HIAAAJCAB"), merge("HiACAB", "HIACAB", "HIAAAJCAB")); + + assertEquals(t("AGADEFHIAAAJCAB"), + merge("AGADEFHiACAB", "AGADEFHIACAB", "AGADEFHIAAAJCAB")); + } + + /** + * Test merging two contents which do one similar modification and one + * insertion is only done by one side, at the end. Between modification and + * insertion is a block which is common between the two contents and the + * common base + * + * @throws java.io.IOException + */ + @Test + public void testTwoSimilarModsAndOneInsertAtEnd() throws IOException { + Assume.assumeTrue(newlineAtEnd); + assertEquals(t("IAAJ"), merge("iA", "IA", "IAAJ")); + + assertEquals(t("IAJ"), merge("iA", "IA", "IAJ")); + + assertEquals(t("IAAAJ"), merge("iA", "IA", "IAAAJ")); + } + + @Test + public void testTwoSimilarModsAndOneInsertAtEndNoNewlineAtEnd() + throws IOException { + Assume.assumeFalse(newlineAtEnd); + assertEquals(t("IAAAJ"), merge("iA", "IA", "IAAJ")); + + assertEquals(t("IAAJ"), merge("iA", "IA", "IAJ")); + + assertEquals(t("IAAAAJ"), merge("iA", "IA", "IAAAJ")); + } + + // Test situations where (at least) one input value is the empty text + + @Test + public void testEmptyTextModifiedAgainstDeletion() throws IOException { + // NOTE: git.git merge-file appends a '\n' to the end of the file even + // when the input files do not have a newline at the end. That appears + // to be a bug in git.git. + assertEquals(t("AB"), merge("A", "AB", "")); + assertEquals(t("AB"), merge("A", "", "AB")); + } + + @Test + public void testEmptyTextUnmodifiedAgainstDeletion() throws IOException { + assertEquals(t(""), merge("AB", "AB", "")); + + assertEquals(t(""), merge("AB", "", "AB")); + } + + @Test + public void testEmptyTextDeletionAgainstDeletion() throws IOException { + assertEquals(t(""), merge("AB", "", "")); + } + + private String merge(String commonBase, String ours, String theirs) + throws IOException { + MergeAlgorithm ma = new MergeAlgorithm(); + ma.setContentMergeStrategy(ContentMergeStrategy.UNION); + MergeResult r = ma.merge(RawTextComparator.DEFAULT, + T(commonBase), T(ours), T(theirs)); + ByteArrayOutputStream bo = new ByteArrayOutputStream(50); + fmt.formatMerge(bo, r, "B", "O", "T", UTF_8); + return bo.toString(UTF_8); + } + + public String t(String text) { + StringBuilder r = new StringBuilder(); + for (int i = 0; i < text.length(); i++) { + char c = text.charAt(i); + switch (c) { + case '<': + r.append("<<<<<<< O\n"); + break; + case '=': + r.append("=======\n"); + break; + case '|': + r.append("||||||| B\n"); + break; + case '>': + r.append(">>>>>>> T\n"); + break; + default: + r.append(c); + if (newlineAtEnd || i < text.length() - 1) + r.append('\n'); + } + } + return r.toString(); + } + + public RawText T(String text) { + return new RawText(Constants.encode(t(text))); + } +} diff --git a/org.eclipse.jgit/.settings/.api_filters b/org.eclipse.jgit/.settings/.api_filters index 50a04d20f0..28b20ab01f 100644 --- a/org.eclipse.jgit/.settings/.api_filters +++ b/org.eclipse.jgit/.settings/.api_filters @@ -16,4 +16,20 @@ + + + + + + + + + + + + + + + + diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java index 60a23dde05..1a97d111e3 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java @@ -493,6 +493,13 @@ public final class Constants { */ public static final String ATTR_BUILTIN_BINARY_MERGER = "binary"; //$NON-NLS-1$ + /** + * Union built-in merge driver + * + * @since 6.10.1 + */ + public static final String ATTR_BUILTIN_UNION_MERGE_DRIVER = "union"; //$NON-NLS-1$ + /** * Create a new digest function for objects. * diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/merge/ContentMergeStrategy.java b/org.eclipse.jgit/src/org/eclipse/jgit/merge/ContentMergeStrategy.java index 6d568643d5..a835a1dfc5 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/merge/ContentMergeStrategy.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/merge/ContentMergeStrategy.java @@ -23,5 +23,12 @@ public enum ContentMergeStrategy { OURS, /** Resolve the conflict hunk using the theirs version. */ - THEIRS -} \ No newline at end of file + THEIRS, + + /** + * Resolve the conflict hunk using a union of both ours and theirs versions. + * + * @since 6.10.1 + */ + UNION +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/merge/MergeAlgorithm.java b/org.eclipse.jgit/src/org/eclipse/jgit/merge/MergeAlgorithm.java index 5734a25276..d0d4d367b9 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/merge/MergeAlgorithm.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/merge/MergeAlgorithm.java @@ -120,6 +120,7 @@ public final class MergeAlgorithm { result.add(1, 0, 0, ConflictState.NO_CONFLICT); break; case THEIRS: + case UNION: result.add(2, 0, theirs.size(), ConflictState.NO_CONFLICT); break; @@ -148,6 +149,7 @@ public final class MergeAlgorithm { // we modified, they deleted switch (strategy) { case OURS: + case UNION: result.add(1, 0, ours.size(), ConflictState.NO_CONFLICT); break; case THEIRS: @@ -158,7 +160,7 @@ public final class MergeAlgorithm { result.add(1, 0, ours.size(), ConflictState.FIRST_CONFLICTING_RANGE); result.add(0, 0, base.size(), - ConflictState.BASE_CONFLICTING_RANGE); + ConflictState.BASE_CONFLICTING_RANGE); result.add(2, 0, 0, ConflictState.NEXT_CONFLICTING_RANGE); break; } @@ -329,6 +331,15 @@ public final class MergeAlgorithm { ConflictState.NO_CONFLICT); break; case THEIRS: + result.add(2, theirsBeginB + commonPrefix, + theirsEndB - commonSuffix, + ConflictState.NO_CONFLICT); + break; + case UNION: + result.add(1, oursBeginB + commonPrefix, + oursEndB - commonSuffix, + ConflictState.NO_CONFLICT); + result.add(2, theirsBeginB + commonPrefix, theirsEndB - commonSuffix, ConflictState.NO_CONFLICT); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java b/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java index 1ad41be423..85b85cf855 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java @@ -1489,11 +1489,23 @@ public class ResolveMerger extends ThreeWayMerger { : getRawText(ours.getEntryObjectId(), attributes[T_OURS]); RawText theirsText = theirs == null ? RawText.EMPTY_TEXT : getRawText(theirs.getEntryObjectId(), attributes[T_THEIRS]); - mergeAlgorithm.setContentMergeStrategy(strategy); + mergeAlgorithm.setContentMergeStrategy( + getAttributesContentMergeStrategy(attributes[T_OURS], + strategy)); return mergeAlgorithm.merge(RawTextComparator.DEFAULT, baseText, ourText, theirsText); } + private ContentMergeStrategy getAttributesContentMergeStrategy( + Attributes attributes, ContentMergeStrategy strategy) { + Attribute attr = attributes.get(Constants.ATTR_MERGE); + if (attr != null && attr.getValue() + .equals(Constants.ATTR_BUILTIN_UNION_MERGE_DRIVER)) { + return ContentMergeStrategy.UNION; + } + return strategy; + } + private boolean isIndexDirty() { if (inCore) { return false; -- cgit v1.2.3 From 3a7a9cb0e89de263d0a5133949c9c9bed5141916 Mon Sep 17 00:00:00 2001 From: Nasser Grainawi Date: Tue, 29 Oct 2024 17:22:15 -0600 Subject: ResolveMerger: Allow setting the TreeWalk AttributesNodeProvider When a merger is created without a Repository, no AttributesNodeProvider is created in the TreeWalk. Since mergers are often created with a custom ObjectInserter and no repo, they skip any lookups of attributes from any of the gitattributes files (within a tree, in the repo info/ dir, or user/global). Since there are potentially merge-affecting attributes in those files, callers might want to use both a custom ObjectInserter and an AttributesNodeProvider. Change-Id: I7997309003bbb598e1002261b3be7f2cc52066c8 --- org.eclipse.jgit/.settings/.api_filters | 20 +++++++++++++++++++ .../src/org/eclipse/jgit/merge/ResolveMerger.java | 23 ++++++++++++++++++++++ 2 files changed, 43 insertions(+) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/.settings/.api_filters b/org.eclipse.jgit/.settings/.api_filters index 28b20ab01f..2c22f0231c 100644 --- a/org.eclipse.jgit/.settings/.api_filters +++ b/org.eclipse.jgit/.settings/.api_filters @@ -32,4 +32,24 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java b/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java index 85b85cf855..a6a287bcf4 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java @@ -41,6 +41,7 @@ import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.attributes.Attribute; import org.eclipse.jgit.attributes.Attributes; +import org.eclipse.jgit.attributes.AttributesNodeProvider; import org.eclipse.jgit.diff.DiffAlgorithm; import org.eclipse.jgit.diff.DiffAlgorithm.SupportedAlgorithm; import org.eclipse.jgit.diff.RawText; @@ -837,6 +838,13 @@ public class ResolveMerger extends ThreeWayMerger { @NonNull private ContentMergeStrategy contentStrategy = ContentMergeStrategy.CONFLICT; + /** + * The {@link AttributesNodeProvider} to use while merging trees. + * + * @since 6.10.1 + */ + protected AttributesNodeProvider attributesNodeProvider; + private static MergeAlgorithm getMergeAlgorithm(Config config) { SupportedAlgorithm diffAlg = config.getEnum( CONFIG_DIFF_SECTION, null, CONFIG_KEY_ALGORITHM, @@ -1836,6 +1844,18 @@ public class ResolveMerger extends ThreeWayMerger { this.workingTreeIterator = workingTreeIterator; } + /** + * Sets the {@link AttributesNodeProvider} to be used by this merger. + * + * @param attributesNodeProvider + * the attributeNodeProvider to set + * @since 6.10.1 + */ + public void setAttributesNodeProvider( + AttributesNodeProvider attributesNodeProvider) { + this.attributesNodeProvider = attributesNodeProvider; + } + /** * The resolve conflict way of three way merging @@ -1880,6 +1900,9 @@ public class ResolveMerger extends ThreeWayMerger { WorkTreeUpdater.createWorkTreeUpdater(db, dircache); dircache = workTreeUpdater.getLockedDirCache(); tw = new NameConflictTreeWalk(db, reader); + if (attributesNodeProvider != null) { + tw.setAttributesNodeProvider(attributesNodeProvider); + } tw.addTree(baseTree); tw.setHead(tw.addTree(headTree)); -- cgit v1.2.3 From 1342502ad0954526589437fd309ef70a36ee3099 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Tue, 5 Nov 2024 01:28:14 +0100 Subject: Add missing @since 7.1 to UploadPack#implies Change-Id: Iabbe1f18a5022b4669a3352493c6fd35920ef25f --- org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java index 216cde1e37..2270c5e5c5 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java @@ -155,8 +155,10 @@ public class UploadPack implements Closeable { /** * Check if the current policy implies another, based on its bitmask. * - * @param implied the implied policy based on its bitmask. + * @param implied + * the implied policy based on its bitmask. * @return true if the policy is implied. + * @since 7.1 */ public boolean implies(RequestPolicy implied) { return (bitmask & implied.bitmask) != 0; -- cgit v1.2.3 From 93ede18ffd17df936da4e8d0ef28e2ba69019c93 Mon Sep 17 00:00:00 2001 From: Jacek Centkowski Date: Fri, 20 Sep 2024 08:47:13 +0200 Subject: Add `numberOfPackFilesAfterBitmap` to RepoStatistics Introduce a `numberOfPackFilesAfterBitmap` that contains the number of packfiles created since the latest bitmap generation. Notes: * the `repo.getObjectDatabase().getPacks()` that obtains the list of packs (in the existing `getStatistics` function) uses `PackDirectory.scanPacks` that boils down to call `PackDirectory.scanPacksImpl` which is sorting packs prior returning them therefore the `numberOfPackFilesAfterBitmap` is just all packs before the one that has bitmap attached * the improved version of `packAndPrune` function (one that skips non-existent packfiles) was introduced for testing Change-Id: I608011462f104fc002ac527aa405f492a8a4b0c2 --- ...NumberOfPackFilesAfterBitmapStatisticsTest.java | 173 +++++++++++++++++++++ .../org/eclipse/jgit/internal/storage/file/GC.java | 13 +- 2 files changed, 185 insertions(+), 1 deletion(-) create mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcNumberOfPackFilesAfterBitmapStatisticsTest.java (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcNumberOfPackFilesAfterBitmapStatisticsTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcNumberOfPackFilesAfterBitmapStatisticsTest.java new file mode 100644 index 0000000000..e5a391f2e3 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcNumberOfPackFilesAfterBitmapStatisticsTest.java @@ -0,0 +1,173 @@ +/* + * Copyright (c) 2024 Jacek Centkowski and others. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.storage.file; + +import static org.junit.Assert.assertEquals; + +import java.io.BufferedOutputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.util.HashSet; +import java.util.Set; +import java.util.stream.StreamSupport; + +import org.eclipse.jgit.internal.storage.file.PackIndex.MutableEntry; +import org.eclipse.jgit.internal.storage.pack.PackExt; +import org.eclipse.jgit.internal.storage.pack.PackWriter; +import org.eclipse.jgit.junit.TestRepository; +import org.eclipse.jgit.lib.NullProgressMonitor; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.PersonIdent; +import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.util.FileUtils; +import org.junit.Test; + +public class GcNumberOfPackFilesAfterBitmapStatisticsTest extends GcTestCase { + @Test + public void testShouldReportZeroObjectsForInitializedRepo() + throws IOException { + assertEquals(0L, gc.getStatistics().numberOfPackFilesAfterBitmap); + } + + @Test + public void testShouldReportAllPackFilesWhenNoGcWasPerformed() + throws Exception { + packAndPrune(); + long result = gc.getStatistics().numberOfPackFilesAfterBitmap; + + assertEquals(repo.getObjectDatabase().getPacks().size(), result); + } + + @Test + public void testShouldReportNoObjectsDirectlyAfterGc() throws Exception { + // given + addCommit(null); + gc.gc().get(); + assertEquals(1L, repositoryBitmapFiles()); + assertEquals(0L, gc.getStatistics().numberOfPackFilesAfterBitmap); + } + + @Test + public void testShouldReportNewObjectsAfterGcWhenRepositoryProgresses() + throws Exception { + // commit & gc + RevCommit parent = addCommit(null); + gc.gc().get(); + assertEquals(1L, repositoryBitmapFiles()); + + // progress & pack + addCommit(parent); + packAndPrune(); + + assertEquals(1L, gc.getStatistics().numberOfPackFilesAfterBitmap); + } + + @Test + public void testShouldReportNewObjectsFromTheLatestBitmapWhenRepositoryProgresses() + throws Exception { + // commit & gc + RevCommit parent = addCommit(null); + gc.gc().get(); + assertEquals(1L, repositoryBitmapFiles()); + + // progress & gc + parent = addCommit(parent); + gc.gc().get(); + assertEquals(2L, repositoryBitmapFiles()); + + // progress & pack + addCommit(parent); + packAndPrune(); + + assertEquals(1L, gc.getStatistics().numberOfPackFilesAfterBitmap); + } + + private void packAndPrune() throws Exception { + try (SkipNonExistingFilesTestRepository testRepo = new SkipNonExistingFilesTestRepository( + repo)) { + testRepo.packAndPrune(); + } + } + + private RevCommit addCommit(RevCommit parent) throws Exception { + return tr.branch("master").commit() + .author(new PersonIdent("repo-metrics", "repo@metrics.com")) + .parent(parent).create(); + } + + private long repositoryBitmapFiles() throws IOException { + return StreamSupport + .stream(Files + .newDirectoryStream(repo.getObjectDatabase() + .getPackDirectory().toPath(), "pack-*.bitmap") + .spliterator(), false) + .count(); + } + + /** + * The TestRepository has a {@link TestRepository#packAndPrune()} function + * but it fails in the last step after GC was performed as it doesn't + * SKIP_MISSING files. In order to circumvent it was copied and improved + * here. + */ + private static class SkipNonExistingFilesTestRepository + extends TestRepository { + private final FileRepository repo; + + private SkipNonExistingFilesTestRepository(FileRepository db) throws IOException { + super(db); + repo = db; + } + + @Override + public void packAndPrune() throws Exception { + ObjectDirectory odb = repo.getObjectDatabase(); + NullProgressMonitor m = NullProgressMonitor.INSTANCE; + + final PackFile pack, idx; + try (PackWriter pw = new PackWriter(repo)) { + Set all = new HashSet<>(); + for (Ref r : repo.getRefDatabase().getRefs()) + all.add(r.getObjectId()); + pw.preparePack(m, all, PackWriter.NONE); + + pack = new PackFile(odb.getPackDirectory(), pw.computeName(), + PackExt.PACK); + try (OutputStream out = new BufferedOutputStream( + new FileOutputStream(pack))) { + pw.writePack(m, m, out); + } + pack.setReadOnly(); + + idx = pack.create(PackExt.INDEX); + try (OutputStream out = new BufferedOutputStream( + new FileOutputStream(idx))) { + pw.writeIndex(out); + } + idx.setReadOnly(); + } + + odb.openPack(pack); + updateServerInfo(); + + // alternative packAndPrune implementation that skips missing files + // after GC. + for (Pack p : odb.getPacks()) { + for (MutableEntry e : p) + FileUtils.delete(odb.fileFor(e.toObjectId()), + FileUtils.SKIP_MISSING); + } + } + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java index cf26f8d284..ee6abc372c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java @@ -1508,6 +1508,12 @@ public class GC { */ public long numberOfPackFiles; + /** + * The number of pack files that were created after the last bitmap + * generation. + */ + public long numberOfPackFilesAfterBitmap; + /** * The number of objects stored as loose objects. */ @@ -1543,6 +1549,8 @@ public class GC { final StringBuilder b = new StringBuilder(); b.append("numberOfPackedObjects=").append(numberOfPackedObjects); //$NON-NLS-1$ b.append(", numberOfPackFiles=").append(numberOfPackFiles); //$NON-NLS-1$ + b.append(", numberOfPackFilesAfterBitmap=") //$NON-NLS-1$ + .append(numberOfPackFilesAfterBitmap); b.append(", numberOfLooseObjects=").append(numberOfLooseObjects); //$NON-NLS-1$ b.append(", numberOfLooseRefs=").append(numberOfLooseRefs); //$NON-NLS-1$ b.append(", numberOfPackedRefs=").append(numberOfPackedRefs); //$NON-NLS-1$ @@ -1567,8 +1575,11 @@ public class GC { ret.numberOfPackedObjects += p.getIndex().getObjectCount(); ret.numberOfPackFiles++; ret.sizeOfPackedObjects += p.getPackFile().length(); - if (p.getBitmapIndex() != null) + if (p.getBitmapIndex() != null) { ret.numberOfBitmaps += p.getBitmapIndex().getBitmapCount(); + } else { + ret.numberOfPackFilesAfterBitmap++; + } } File objDir = repo.getObjectsDirectory(); String[] fanout = objDir.list(); -- cgit v1.2.3 From 88f8321bafeb9476bbb9b42b349521c68e375e4f Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Mon, 4 Nov 2024 14:21:24 -0800 Subject: SystemReader: Offer methods with java.time API Error prone explains: The Date API is full of major design flaws and pitfalls and should be avoided at all costs. Prefer the java.time APIs, specifically, java.time.Instant (for physical time) and java.time.LocalDate[Time] (for civil time). Add to SystemReader methods to get the time and timezone in the new java.time classes (Instant/ZoneId) and mark as deprecated their old counterparts. The mapping of methods is: * #getCurrentTime -> #now (returns Instant instead of int) * #getTimezone -> #getTimeZoneAt (returns ZoneOffset intead of int) * #getTimeZone -> #getTimeZoneId (return ZoneId instead of TimeZone) Change-Id: Ic55b2f442a40046ff0ed24f61f566fc7416471be --- .../org/eclipse/jgit/junit/MockSystemReader.java | 18 +++++++ org.eclipse.jgit/.settings/.api_filters | 14 +++++ .../src/org/eclipse/jgit/util/SystemReader.java | 61 ++++++++++++++++++++++ 3 files changed, 93 insertions(+) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/MockSystemReader.java b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/MockSystemReader.java index 419fdb1966..b0365aa7e1 100644 --- a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/MockSystemReader.java +++ b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/MockSystemReader.java @@ -18,6 +18,9 @@ import java.lang.reflect.Field; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.time.Duration; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZoneOffset; import java.util.HashMap; import java.util.Locale; import java.util.Map; @@ -201,6 +204,11 @@ public class MockSystemReader extends SystemReader { return now; } + @Override + public Instant now() { + return Instant.ofEpochMilli(now); + } + @Override public MonotonicClock getClock() { return () -> { @@ -236,11 +244,21 @@ public class MockSystemReader extends SystemReader { return getTimeZone().getOffset(when) / (60 * 1000); } + @Override + public ZoneOffset getTimeZoneAt(Instant when) { + return getTimeZoneId().getRules().getOffset(when); + } + @Override public TimeZone getTimeZone() { return TimeZone.getTimeZone("GMT-03:30"); } + @Override + public ZoneId getTimeZoneId() { + return ZoneOffset.ofHoursMinutes(-3, 30); + } + @Override public Locale getLocale() { return Locale.US; diff --git a/org.eclipse.jgit/.settings/.api_filters b/org.eclipse.jgit/.settings/.api_filters index f2c73f5c48..aed6683062 100644 --- a/org.eclipse.jgit/.settings/.api_filters +++ b/org.eclipse.jgit/.settings/.api_filters @@ -36,4 +36,18 @@ + + + + + + + + + + + + + + diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/SystemReader.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/SystemReader.java index 03ed925617..7150e471bc 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/SystemReader.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/SystemReader.java @@ -25,6 +25,9 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.text.DateFormat; import java.text.SimpleDateFormat; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZoneOffset; import java.util.Locale; import java.util.TimeZone; import java.util.concurrent.atomic.AtomicReference; @@ -166,10 +169,20 @@ public abstract class SystemReader { return System.currentTimeMillis(); } + @Override + public Instant now() { + return Instant.now(); + } + @Override public int getTimezone(long when) { return getTimeZone().getOffset(when) / (60 * 1000); } + + @Override + public ZoneOffset getTimeZoneAt(Instant when) { + return getTimeZoneId().getRules().getOffset(when); + } } /** @@ -227,10 +240,20 @@ public abstract class SystemReader { return delegate.getCurrentTime(); } + @Override + public Instant now() { + return delegate.now(); + } + @Override public int getTimezone(long when) { return delegate.getTimezone(when); } + + @Override + public ZoneOffset getTimeZoneAt(Instant when) { + return delegate.getTimeZoneAt(when); + } } private static volatile SystemReader INSTANCE = DEFAULT; @@ -501,9 +524,21 @@ public abstract class SystemReader { * Get the current system time * * @return the current system time + * + * @deprecated Use {@link #now()} */ + @Deprecated public abstract long getCurrentTime(); + /** + * Get the current system time + * + * @return the current system time + * + * @since 7.1 + */ + public abstract Instant now(); + /** * Get clock instance preferred by this system. * @@ -520,19 +555,45 @@ public abstract class SystemReader { * @param when * a system timestamp * @return the local time zone + * + * @deprecated Use {@link #getTimeZoneAt(Instant)} instead. */ + @Deprecated public abstract int getTimezone(long when); + /** + * Get the local time zone offset at "when" time + * + * @param when + * a system timestamp + * @return the local time zone + * @since 7.1 + */ + public abstract ZoneOffset getTimeZoneAt(Instant when); + /** * Get system time zone, possibly mocked for testing * * @return system time zone, possibly mocked for testing * @since 1.2 + * + * @deprecated Use {@link #getTimeZoneId()} */ + @Deprecated public TimeZone getTimeZone() { return TimeZone.getDefault(); } + /** + * Get system time zone, possibly mocked for testing + * + * @return system time zone, possibly mocked for testing + * @since 7.1 + */ + public ZoneId getTimeZoneId() { + return ZoneId.systemDefault(); + } + /** * Get the locale to use * -- cgit v1.2.3 From a7b5f056d8ff68a0df8968fdd10d3f7faa66d290 Mon Sep 17 00:00:00 2001 From: Laura Hamelin Date: Wed, 6 Nov 2024 13:41:30 -0800 Subject: DfsBlockCacheConfig: propagate hotmap configs to pack ext cache configs CacheHotMap is currently only set on the base DfsBlockCacheConfig and is not propagated down to PackExt specific caches. Because CacheHotMap is set from a method call rather than from Configs, this change sets per-PackExt CacheHotMap configs on PackExt cache configs both when DfsBlockCacheConfig#setCacheHotMap(...) is called, and when DfsBlockCacheConfig#configure(...) is called after setCacheHotMap. The outer DfsBlockCacheConfig keeps the full CacheHotMap for the same reason that the CacheHotMap config is propagated in both setCacheHotMap and configure: the order of operations setting the configuration from Configs and calling setCacheHotMap is not guaranteed. Change-Id: Id9dc32fedca99ecc83c9dc90c24d9616873a202e --- .../storage/dfs/DfsBlockCacheConfigTest.java | 71 ++++++++++++++++++++++ .../internal/storage/dfs/DfsBlockCacheConfig.java | 22 +++++++ 2 files changed, 93 insertions(+) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfigTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfigTest.java index 65774e6d6c..afa3179cde 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfigTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfigTest.java @@ -179,6 +179,76 @@ public class DfsBlockCacheConfigTest { is(indexConfig)); } + @Test + public void fromConfig_withExistingCacheHotMap_configWithPackExtConfigsHasHotMaps() { + Config config = new Config(); + addPackExtConfigEntry(config, "pack", List.of(PackExt.PACK), + /* blockLimit= */ 20 * 512, /* blockSize= */ 512); + + addPackExtConfigEntry(config, "bitmap", List.of(PackExt.BITMAP_INDEX), + /* blockLimit= */ 25 * 1024, /* blockSize= */ 1024); + + addPackExtConfigEntry(config, "index", + List.of(PackExt.INDEX, PackExt.OBJECT_SIZE_INDEX, + PackExt.REVERSE_INDEX), + /* blockLimit= */ 30 * 1024, /* blockSize= */ 1024); + + Map cacheHotMap = Map.of(PackExt.PACK, 1, + PackExt.BITMAP_INDEX, 2, PackExt.INDEX, 3, PackExt.REFTABLE, 4); + + DfsBlockCacheConfig cacheConfig = new DfsBlockCacheConfig(); + cacheConfig.setCacheHotMap(cacheHotMap); + cacheConfig.fromConfig(config); + + var configs = cacheConfig.getPackExtCacheConfigurations(); + assertThat(cacheConfig.getCacheHotMap(), is(cacheHotMap)); + assertThat(configs, hasSize(3)); + var packConfig = getConfigForExt(configs, PackExt.PACK); + assertThat(packConfig.getCacheHotMap(), is(Map.of(PackExt.PACK, 1))); + + var bitmapConfig = getConfigForExt(configs, PackExt.BITMAP_INDEX); + assertThat(bitmapConfig.getCacheHotMap(), + is(Map.of(PackExt.BITMAP_INDEX, 2))); + + var indexConfig = getConfigForExt(configs, PackExt.INDEX); + assertThat(indexConfig.getCacheHotMap(), is(Map.of(PackExt.INDEX, 3))); + } + + @Test + public void setCacheHotMap_configWithPackExtConfigs_setsHotMaps() { + Config config = new Config(); + addPackExtConfigEntry(config, "pack", List.of(PackExt.PACK), + /* blockLimit= */ 20 * 512, /* blockSize= */ 512); + + addPackExtConfigEntry(config, "bitmap", List.of(PackExt.BITMAP_INDEX), + /* blockLimit= */ 25 * 1024, /* blockSize= */ 1024); + + addPackExtConfigEntry(config, "index", + List.of(PackExt.INDEX, PackExt.OBJECT_SIZE_INDEX, + PackExt.REVERSE_INDEX), + /* blockLimit= */ 30 * 1024, /* blockSize= */ 1024); + + Map cacheHotMap = Map.of(PackExt.PACK, 1, + PackExt.BITMAP_INDEX, 2, PackExt.INDEX, 3, PackExt.REFTABLE, 4); + + DfsBlockCacheConfig cacheConfig = new DfsBlockCacheConfig() + .fromConfig(config); + cacheConfig.setCacheHotMap(cacheHotMap); + + var configs = cacheConfig.getPackExtCacheConfigurations(); + assertThat(cacheConfig.getCacheHotMap(), is(cacheHotMap)); + assertThat(configs, hasSize(3)); + var packConfig = getConfigForExt(configs, PackExt.PACK); + assertThat(packConfig.getCacheHotMap(), is(Map.of(PackExt.PACK, 1))); + + var bitmapConfig = getConfigForExt(configs, PackExt.BITMAP_INDEX); + assertThat(bitmapConfig.getCacheHotMap(), + is(Map.of(PackExt.BITMAP_INDEX, 2))); + + var indexConfig = getConfigForExt(configs, PackExt.INDEX); + assertThat(indexConfig.getCacheHotMap(), is(Map.of(PackExt.INDEX, 3))); + } + @Test public void fromConfigs_baseConfigOnly_nameSetFromConfigDfsSubSection() { Config config = new Config(); @@ -291,6 +361,7 @@ public class DfsBlockCacheConfigTest { " Name: dfs.pack", " BlockLimit: " + 20 * 512, " BlockSize: 512", " StreamRatio: 0.3", " ConcurrencyLevel: 32", + " CacheHotMapEntry: " + PackExt.PACK + " : " + 10, " PackExts: " + List.of(PackExt.PACK)))); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java index acd7add5a9..d63c208bbb 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java @@ -30,6 +30,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.function.Consumer; +import java.util.function.Function; import java.util.stream.Collectors; import org.eclipse.jgit.internal.JGitText; @@ -300,12 +301,24 @@ public class DfsBlockCacheConfig { * map of hot count per pack extension for {@code DfsBlockCache}. * @return {@code this} */ + /* + * TODO The cache HotMap configuration should be set as a config option and + * not passed in through a setter. + */ public DfsBlockCacheConfig setCacheHotMap( Map cacheHotMap) { this.cacheHotMap = Collections.unmodifiableMap(cacheHotMap); + setCacheHotMapToPackExtConfigs(this.cacheHotMap); return this; } + private void setCacheHotMapToPackExtConfigs( + Map cacheHotMap) { + for (DfsBlockCachePackExtConfig packExtConfig : packExtCacheConfigurations) { + packExtConfig.setCacheHotMap(cacheHotMap); + } + } + /** * Get the consumer of cache index events. * @@ -433,6 +446,7 @@ public class DfsBlockCacheConfig { cacheConfigs.add(cacheConfig); } packExtCacheConfigurations = cacheConfigs; + setCacheHotMapToPackExtConfigs(this.cacheHotMap); } private static Set intersection(Set first, Set second) { @@ -551,6 +565,14 @@ public class DfsBlockCacheConfig { return packExtCacheConfiguration; } + void setCacheHotMap(Map cacheHotMap) { + Map packExtHotMap = packExts.stream() + .filter(cacheHotMap::containsKey) + .collect(Collectors.toUnmodifiableMap(Function.identity(), + cacheHotMap::get)); + packExtCacheConfiguration.setCacheHotMap(packExtHotMap); + } + private static DfsBlockCachePackExtConfig fromConfig(Config config, String section, String subSection) { String packExtensions = config.getString(section, subSection, -- cgit v1.2.3 From 9ea537d5c44876c12d154331c94a7ba14eaa64f8 Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Fri, 8 Nov 2024 08:49:09 -0800 Subject: DfsGarbageCollector: #setReflogExpire with Instant instead of Date The Date API is full of major design flaws and pitfalls and should be avoided at all costs. Prefer the java.time APIs, specifically, java.time.Instant (for physical time) and java.time.LocalDate[Time] (for civil time). [1] Replace the Date with Instant in the DfsGarbageCollector#setReflogExpire method. [1] https://errorprone.info/bugpattern/JavaUtilDate Change-Id: Ie98e426e63621e8bef96c31bca56aec0c8eef5a6 --- .../jgit/internal/storage/dfs/DfsGarbageCollectorTest.java | 3 ++- .../jgit/internal/storage/dfs/DfsGarbageCollector.java | 11 ++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollectorTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollectorTest.java index 3edd1056be..f9fbfe8db0 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollectorTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollectorTest.java @@ -16,6 +16,7 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.IOException; +import java.time.Instant; import java.util.Arrays; import java.util.Collections; import java.util.Date; @@ -1331,7 +1332,7 @@ public class DfsGarbageCollectorTest { gc = new DfsGarbageCollector(repo); gc.setReftableConfig(new ReftableConfig()); // Expire ref log entries older than 30 days - gc.setRefLogExpire(new Date(thirty_days_ago)); + gc.setRefLogExpire(Instant.ofEpochMilli(thirty_days_ago)); run(gc); // Single GC pack present with all objects. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java index 6861fe8099..e6068a15ec 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java @@ -24,11 +24,11 @@ import static org.eclipse.jgit.internal.storage.pack.PackExt.REFTABLE; import static org.eclipse.jgit.internal.storage.pack.PackWriter.NONE; import java.io.IOException; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; import java.util.Collection; -import java.util.Date; import java.util.EnumSet; import java.util.GregorianCalendar; import java.util.HashSet; @@ -100,7 +100,7 @@ public class DfsGarbageCollector { private Set allTags; private Set nonHeads; private Set tagTargets; - private Date refLogExpire; + private Instant refLogExpire; /** * Initialize a garbage collector. @@ -212,8 +212,8 @@ public class DfsGarbageCollector { * instant in time which defines refLog expiration * @return {@code this} */ - public DfsGarbageCollector setRefLogExpire(Date refLogExpire) { - this.refLogExpire = refLogExpire ; + public DfsGarbageCollector setRefLogExpire(Instant refLogExpire) { + this.refLogExpire = refLogExpire; return this; } @@ -752,7 +752,8 @@ public class DfsGarbageCollector { compact.setIncludeDeletes(includeDeletes); compact.setConfig(configureReftable(reftableConfig, out)); if(refLogExpire != null ){ - compact.setReflogExpireOldestReflogTimeMillis(refLogExpire.getTime()); + compact.setReflogExpireOldestReflogTimeMillis( + refLogExpire.toEpochMilli()); } compact.compact(); pack.addFileExt(REFTABLE); -- cgit v1.2.3 From e8c414b9cc72cd0e4f8c66b836de4c43f134b102 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 6 Sep 2024 12:26:44 +0200 Subject: Replace custom encoder `Constants#encode` by JDK implementation Using the implementation provided in the JDK since Java 1.6 by `String#getBytes(Charset)` reduces JGit maintenance effort and improves performance. The method Constants#encode was implemented when JGit still used Java 1.5. See [1]. Kudos to Marcin for proposing to use this improvement in RefWriter [2]. I think it should be used generally. [1] https://repo.or.cz/jgit.git?a=commit;h=bfa3da225f198b19061158499b1135aff07d85b3 [2] https://eclipse.gerrithub.io/c/eclipse-jgit/jgit/+/1195180 Also-By: Marcin Czech Change-Id: I361ed6286b98351a315b8a8ffc3cb845831d35b2 (cherry picked from commit e5d2898997462e0f2409c09497ab62c6cda2dbaf) --- org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java index 1a97d111e3..5d7eacb4b8 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java @@ -14,7 +14,6 @@ package org.eclipse.jgit.lib; import static java.nio.charset.StandardCharsets.UTF_8; -import java.nio.ByteBuffer; import java.nio.charset.Charset; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; @@ -688,17 +687,7 @@ public final class Constants { * @see #CHARACTER_ENCODING */ public static byte[] encode(String str) { - final ByteBuffer bb = UTF_8.encode(str); - final int len = bb.limit(); - if (bb.hasArray() && bb.arrayOffset() == 0) { - final byte[] arr = bb.array(); - if (arr.length == len) - return arr; - } - - final byte[] arr = new byte[len]; - bb.get(arr); - return arr; + return str.getBytes(UTF_8); } static { -- cgit v1.2.3 From d34f8b523638fc95b2e7006d02c9f6a756cbba85 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 6 Sep 2024 13:42:27 +0200 Subject: Replace custom encoder Constants#encodeASCII by JDK implementation Ensure that the method still throws an IllegalArgumentException for malformed input or if the String contains unmappable characters. Change-Id: I6a340aa1af60c315272ff13b6bf2041ba30c94ca (cherry picked from commit 0fd76114e3436ac635641d06371fd8833179312d) --- .../src/org/eclipse/jgit/lib/Constants.java | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java index 5d7eacb4b8..1a2f735622 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Constants.java @@ -12,9 +12,14 @@ package org.eclipse.jgit.lib; +import static java.nio.charset.StandardCharsets.US_ASCII; import static java.nio.charset.StandardCharsets.UTF_8; import java.nio.charset.Charset; +import java.nio.CharBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.CharsetEncoder; +import java.nio.charset.CodingErrorAction; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.text.MessageFormat; @@ -667,14 +672,15 @@ public final class Constants { * the 7-bit ASCII character space. */ public static byte[] encodeASCII(String s) { - final byte[] r = new byte[s.length()]; - for (int k = r.length - 1; k >= 0; k--) { - final char c = s.charAt(k); - if (c > 127) - throw new IllegalArgumentException(MessageFormat.format(JGitText.get().notASCIIString, s)); - r[k] = (byte) c; + try { + CharsetEncoder encoder = US_ASCII.newEncoder() + .onUnmappableCharacter(CodingErrorAction.REPORT) + .onMalformedInput(CodingErrorAction.REPORT); + return encoder.encode(CharBuffer.wrap(s)).array(); + } catch (CharacterCodingException e) { + throw new IllegalArgumentException( + MessageFormat.format(JGitText.get().notASCIIString, s), e); } - return r; } /** -- cgit v1.2.3 From ce3a7ca3bf0b5f8ad9d14efb730c8d3d204e802a Mon Sep 17 00:00:00 2001 From: Jacek Centkowski Date: Mon, 11 Nov 2024 12:36:03 +0100 Subject: Rename numberOfPackFilesAfterBitmap to numberOfPackFilesSinceBitmap As sugested in I608011462f1. Change-Id: If66226dd7b08ae768413fa614df5dcb6b44dc118 --- ...NumberOfPackFilesAfterBitmapStatisticsTest.java | 180 --------------------- ...NumberOfPackFilesSinceBitmapStatisticsTest.java | 180 +++++++++++++++++++++ .../org/eclipse/jgit/internal/storage/file/GC.java | 8 +- 3 files changed, 184 insertions(+), 184 deletions(-) delete mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcNumberOfPackFilesAfterBitmapStatisticsTest.java create mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcNumberOfPackFilesSinceBitmapStatisticsTest.java (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcNumberOfPackFilesAfterBitmapStatisticsTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcNumberOfPackFilesAfterBitmapStatisticsTest.java deleted file mode 100644 index 820f0c768d..0000000000 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcNumberOfPackFilesAfterBitmapStatisticsTest.java +++ /dev/null @@ -1,180 +0,0 @@ -/* - * Copyright (c) 2024 Jacek Centkowski and others. - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.storage.file; - -import static org.junit.Assert.assertEquals; - -import java.io.BufferedOutputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.nio.file.Files; -import java.util.HashSet; -import java.util.Set; -import java.util.stream.StreamSupport; - -import org.eclipse.jgit.internal.storage.file.PackIndex.MutableEntry; -import org.eclipse.jgit.internal.storage.pack.PackExt; -import org.eclipse.jgit.internal.storage.pack.PackWriter; -import org.eclipse.jgit.junit.TestRepository; -import org.eclipse.jgit.lib.NullProgressMonitor; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.PersonIdent; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.revwalk.RevCommit; -import org.eclipse.jgit.util.FileUtils; -import org.junit.Test; - -public class GcNumberOfPackFilesAfterBitmapStatisticsTest extends GcTestCase { - @Test - public void testShouldReportZeroObjectsForInitializedRepo() - throws IOException { - assertEquals(0L, gc.getStatistics().numberOfPackFilesAfterBitmap); - } - - @Test - public void testShouldReportAllPackFilesWhenNoGcWasPerformed() - throws Exception { - packAndPrune(); - long result = gc.getStatistics().numberOfPackFilesAfterBitmap; - - assertEquals(repo.getObjectDatabase().getPacks().size(), result); - } - - @Test - public void testShouldReportNoObjectsDirectlyAfterGc() throws Exception { - // given - addCommit(null); - gc.gc().get(); - assertEquals(1L, repositoryBitmapFiles()); - assertEquals(0L, gc.getStatistics().numberOfPackFilesAfterBitmap); - } - - @Test - public void testShouldReportNewObjectsAfterGcWhenRepositoryProgresses() - throws Exception { - // commit & gc - RevCommit parent = addCommit(null); - gc.gc().get(); - assertEquals(1L, repositoryBitmapFiles()); - - // progress & pack - addCommit(parent); - packAndPrune(); - - assertEquals(1L, gc.getStatistics().numberOfPackFilesAfterBitmap); - } - - @Test - public void testShouldReportNewObjectsFromTheLatestBitmapWhenRepositoryProgresses() - throws Exception { - // commit & gc - RevCommit parent = addCommit(null); - gc.gc().get(); - assertEquals(1L, repositoryBitmapFiles()); - - // progress & gc - parent = addCommit(parent); - gc.gc().get(); - assertEquals(2L, repositoryBitmapFiles()); - - // progress & pack - addCommit(parent); - packAndPrune(); - - assertEquals(1L, gc.getStatistics().numberOfPackFilesAfterBitmap); - } - - private void packAndPrune() throws Exception { - try (SkipNonExistingFilesTestRepository testRepo = new SkipNonExistingFilesTestRepository( - repo)) { - testRepo.packAndPrune(); - } - } - - private RevCommit addCommit(RevCommit parent) throws Exception { - PersonIdent ident = new PersonIdent("repo-metrics", "repo@metrics.com"); - TestRepository.CommitBuilder builder = tr.commit() - .author(ident); - if (parent != null) { - builder.parent(parent); - } - RevCommit commit = builder.create(); - tr.update("master", commit); - parent = commit; - return parent; - } - - private long repositoryBitmapFiles() throws IOException { - return StreamSupport - .stream(Files - .newDirectoryStream(repo.getObjectDatabase() - .getPackDirectory().toPath(), "pack-*.bitmap") - .spliterator(), false) - .count(); - } - - /** - * The TestRepository has a {@link TestRepository#packAndPrune()} function - * but it fails in the last step after GC was performed as it doesn't - * SKIP_MISSING files. In order to circumvent it was copied and improved - * here. - */ - private static class SkipNonExistingFilesTestRepository - extends TestRepository { - private final FileRepository repo; - - private SkipNonExistingFilesTestRepository(FileRepository db) throws IOException { - super(db); - repo = db; - } - - @Override - public void packAndPrune() throws Exception { - ObjectDirectory odb = repo.getObjectDatabase(); - NullProgressMonitor m = NullProgressMonitor.INSTANCE; - - final PackFile pack, idx; - try (PackWriter pw = new PackWriter(repo)) { - Set all = new HashSet<>(); - for (Ref r : repo.getRefDatabase().getRefs()) - all.add(r.getObjectId()); - pw.preparePack(m, all, PackWriter.NONE); - - pack = new PackFile(odb.getPackDirectory(), pw.computeName(), - PackExt.PACK); - try (OutputStream out = new BufferedOutputStream( - new FileOutputStream(pack))) { - pw.writePack(m, m, out); - } - pack.setReadOnly(); - - idx = pack.create(PackExt.INDEX); - try (OutputStream out = new BufferedOutputStream( - new FileOutputStream(idx))) { - pw.writeIndex(out); - } - idx.setReadOnly(); - } - - odb.openPack(pack); - updateServerInfo(); - - // alternative packAndPrune implementation that skips missing files - // after GC. - for (Pack p : odb.getPacks()) { - for (MutableEntry e : p) - FileUtils.delete(odb.fileFor(e.toObjectId()), - FileUtils.SKIP_MISSING); - } - } - } -} diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcNumberOfPackFilesSinceBitmapStatisticsTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcNumberOfPackFilesSinceBitmapStatisticsTest.java new file mode 100644 index 0000000000..42b99ae512 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcNumberOfPackFilesSinceBitmapStatisticsTest.java @@ -0,0 +1,180 @@ +/* + * Copyright (c) 2024 Jacek Centkowski and others. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.storage.file; + +import static org.junit.Assert.assertEquals; + +import java.io.BufferedOutputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.util.HashSet; +import java.util.Set; +import java.util.stream.StreamSupport; + +import org.eclipse.jgit.internal.storage.file.PackIndex.MutableEntry; +import org.eclipse.jgit.internal.storage.pack.PackExt; +import org.eclipse.jgit.internal.storage.pack.PackWriter; +import org.eclipse.jgit.junit.TestRepository; +import org.eclipse.jgit.lib.NullProgressMonitor; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.PersonIdent; +import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.util.FileUtils; +import org.junit.Test; + +public class GcNumberOfPackFilesSinceBitmapStatisticsTest extends GcTestCase { + @Test + public void testShouldReportZeroObjectsForInitializedRepo() + throws IOException { + assertEquals(0L, gc.getStatistics().numberOfPackFilesSinceBitmap); + } + + @Test + public void testShouldReportAllPackFilesWhenNoGcWasPerformed() + throws Exception { + packAndPrune(); + long result = gc.getStatistics().numberOfPackFilesSinceBitmap; + + assertEquals(repo.getObjectDatabase().getPacks().size(), result); + } + + @Test + public void testShouldReportNoObjectsDirectlyAfterGc() throws Exception { + // given + addCommit(null); + gc.gc().get(); + assertEquals(1L, repositoryBitmapFiles()); + assertEquals(0L, gc.getStatistics().numberOfPackFilesSinceBitmap); + } + + @Test + public void testShouldReportNewObjectsSinceGcWhenRepositoryProgresses() + throws Exception { + // commit & gc + RevCommit parent = addCommit(null); + gc.gc().get(); + assertEquals(1L, repositoryBitmapFiles()); + + // progress & pack + addCommit(parent); + packAndPrune(); + + assertEquals(1L, gc.getStatistics().numberOfPackFilesSinceBitmap); + } + + @Test + public void testShouldReportNewObjectsFromTheLatestBitmapWhenRepositoryProgresses() + throws Exception { + // commit & gc + RevCommit parent = addCommit(null); + gc.gc().get(); + assertEquals(1L, repositoryBitmapFiles()); + + // progress & gc + parent = addCommit(parent); + gc.gc().get(); + assertEquals(2L, repositoryBitmapFiles()); + + // progress & pack + addCommit(parent); + packAndPrune(); + + assertEquals(1L, gc.getStatistics().numberOfPackFilesSinceBitmap); + } + + private void packAndPrune() throws Exception { + try (SkipNonExistingFilesTestRepository testRepo = new SkipNonExistingFilesTestRepository( + repo)) { + testRepo.packAndPrune(); + } + } + + private RevCommit addCommit(RevCommit parent) throws Exception { + PersonIdent ident = new PersonIdent("repo-metrics", "repo@metrics.com"); + TestRepository.CommitBuilder builder = tr.commit() + .author(ident); + if (parent != null) { + builder.parent(parent); + } + RevCommit commit = builder.create(); + tr.update("master", commit); + parent = commit; + return parent; + } + + private long repositoryBitmapFiles() throws IOException { + return StreamSupport + .stream(Files + .newDirectoryStream(repo.getObjectDatabase() + .getPackDirectory().toPath(), "pack-*.bitmap") + .spliterator(), false) + .count(); + } + + /** + * The TestRepository has a {@link TestRepository#packAndPrune()} function + * but it fails in the last step after GC was performed as it doesn't + * SKIP_MISSING files. In order to circumvent it was copied and improved + * here. + */ + private static class SkipNonExistingFilesTestRepository + extends TestRepository { + private final FileRepository repo; + + private SkipNonExistingFilesTestRepository(FileRepository db) throws IOException { + super(db); + repo = db; + } + + @Override + public void packAndPrune() throws Exception { + ObjectDirectory odb = repo.getObjectDatabase(); + NullProgressMonitor m = NullProgressMonitor.INSTANCE; + + final PackFile pack, idx; + try (PackWriter pw = new PackWriter(repo)) { + Set all = new HashSet<>(); + for (Ref r : repo.getRefDatabase().getRefs()) + all.add(r.getObjectId()); + pw.preparePack(m, all, PackWriter.NONE); + + pack = new PackFile(odb.getPackDirectory(), pw.computeName(), + PackExt.PACK); + try (OutputStream out = new BufferedOutputStream( + new FileOutputStream(pack))) { + pw.writePack(m, m, out); + } + pack.setReadOnly(); + + idx = pack.create(PackExt.INDEX); + try (OutputStream out = new BufferedOutputStream( + new FileOutputStream(idx))) { + pw.writeIndex(out); + } + idx.setReadOnly(); + } + + odb.openPack(pack); + updateServerInfo(); + + // alternative packAndPrune implementation that skips missing files + // after GC. + for (Pack p : odb.getPacks()) { + for (MutableEntry e : p) + FileUtils.delete(odb.fileFor(e.toObjectId()), + FileUtils.SKIP_MISSING); + } + } + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java index 34b5ec01e1..d8eb751136 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java @@ -1512,7 +1512,7 @@ public class GC { * The number of pack files that were created after the last bitmap * generation. */ - public long numberOfPackFilesAfterBitmap; + public long numberOfPackFilesSinceBitmap; /** * The number of objects stored as loose objects. @@ -1549,8 +1549,8 @@ public class GC { final StringBuilder b = new StringBuilder(); b.append("numberOfPackedObjects=").append(numberOfPackedObjects); //$NON-NLS-1$ b.append(", numberOfPackFiles=").append(numberOfPackFiles); //$NON-NLS-1$ - b.append(", numberOfPackFilesAfterBitmap=") //$NON-NLS-1$ - .append(numberOfPackFilesAfterBitmap); + b.append(", numberOfPackFilesSinceBitmap=") //$NON-NLS-1$ + .append(numberOfPackFilesSinceBitmap); b.append(", numberOfLooseObjects=").append(numberOfLooseObjects); //$NON-NLS-1$ b.append(", numberOfLooseRefs=").append(numberOfLooseRefs); //$NON-NLS-1$ b.append(", numberOfPackedRefs=").append(numberOfPackedRefs); //$NON-NLS-1$ @@ -1578,7 +1578,7 @@ public class GC { if (p.getBitmapIndex() != null) ret.numberOfBitmaps += p.getBitmapIndex().getBitmapCount(); else - ret.numberOfPackFilesAfterBitmap++; + ret.numberOfPackFilesSinceBitmap++; } File objDir = repo.getObjectsDirectory(); String[] fanout = objDir.list(); -- cgit v1.2.3 From e682a023936b190c25c7b708a85b6cf0db7c80c5 Mon Sep 17 00:00:00 2001 From: Martin Fick Date: Mon, 23 Sep 2024 12:10:17 -0700 Subject: Pack.java: Recover more often in Pack.copyAsIs2() The PACK class is designed to throw StoredObjectRepresentationNotAvailableException at times when it cannot find an object which previously was believed to be in its packfile and it is still possible for the caller, PackWriter.writeObjectImpl(), to retry copying the object from another file and potentially avoid causing a user facing error for this fairly common expected situation. This retry helps handle when repacking causes a packfile to be replaced by new files with the same objects. Improve copyAsIs2() to drastically make recovery possible in more situations. Once any data for a specific object, has been sent it is very difficult to recover from that object being relocated to another pack. But if a read error is detected in copyAsIs2() before sending the object header (and thus any data), then it should still be recoverable. Fix three places where we could have recovered because we hadn't sent the header yet, and adjust another place to send the header a bit later, after having read some data from the object successfully. Basically, if the header has not been written yet, throw StoredObjectRepresentationNotAvailableException to signal that this is still recoverable. These fixes should drastically improve the chances of recovery since due to unix file semantics, if the partial read succeeds, then the full read will very likely succeed. This is because while the file may no longer be open when the first read is done (the WindowCache may have evicted it), once the first read completes it will likely still be open and even if the file is deleted the WindowCache will continue to be able to read from it until it closes it. Change-Id: Ib87e294e0dbacf71b10db55be511e91963b4a84a Signed-off-by: Martin Fick --- .../eclipse/jgit/internal/storage/file/Pack.java | 295 +++++++++++---------- 1 file changed, 153 insertions(+), 142 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java index be457644d9..3518342f97 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java @@ -416,185 +416,196 @@ public class Pack implements Iterable { final CRC32 crc2 = validate ? new CRC32() : null; final byte[] buf = out.getCopyBuffer(); + boolean isHeaderWritten = false; // Rip apart the header so we can discover the size. // - readFully(src.offset, buf, 0, 20, curs); - int c = buf[0] & 0xff; - final int typeCode = (c >> 4) & 7; - long inflatedLength = c & 15; - int shift = 4; - int headerCnt = 1; - while ((c & 0x80) != 0) { - c = buf[headerCnt++] & 0xff; - inflatedLength += ((long) (c & 0x7f)) << shift; - shift += 7; - } - - if (typeCode == Constants.OBJ_OFS_DELTA) { - do { + try { + readFully(src.offset, buf, 0, 20, curs); + + int c = buf[0] & 0xff; + final int typeCode = (c >> 4) & 7; + long inflatedLength = c & 15; + int shift = 4; + int headerCnt = 1; + while ((c & 0x80) != 0) { c = buf[headerCnt++] & 0xff; - } while ((c & 128) != 0); - if (validate) { - assert(crc1 != null && crc2 != null); - crc1.update(buf, 0, headerCnt); - crc2.update(buf, 0, headerCnt); + inflatedLength += ((long) (c & 0x7f)) << shift; + shift += 7; } - } else if (typeCode == Constants.OBJ_REF_DELTA) { - if (validate) { + + if (typeCode == Constants.OBJ_OFS_DELTA) { + do { + c = buf[headerCnt++] & 0xff; + } while ((c & 128) != 0); + if (validate) { + assert(crc1 != null && crc2 != null); + crc1.update(buf, 0, headerCnt); + crc2.update(buf, 0, headerCnt); + } + } else if (typeCode == Constants.OBJ_REF_DELTA) { + if (validate) { + assert(crc1 != null && crc2 != null); + crc1.update(buf, 0, headerCnt); + crc2.update(buf, 0, headerCnt); + } + + readFully(src.offset + headerCnt, buf, 0, 20, curs); + if (validate) { + assert(crc1 != null && crc2 != null); + crc1.update(buf, 0, 20); + crc2.update(buf, 0, 20); + } + headerCnt += 20; + } else if (validate) { assert(crc1 != null && crc2 != null); crc1.update(buf, 0, headerCnt); crc2.update(buf, 0, headerCnt); } - readFully(src.offset + headerCnt, buf, 0, 20, curs); - if (validate) { - assert(crc1 != null && crc2 != null); - crc1.update(buf, 0, 20); - crc2.update(buf, 0, 20); - } - headerCnt += 20; - } else if (validate) { - assert(crc1 != null && crc2 != null); - crc1.update(buf, 0, headerCnt); - crc2.update(buf, 0, headerCnt); - } + final long dataOffset = src.offset + headerCnt; + final long dataLength = src.length; + final long expectedCRC; + final ByteArrayWindow quickCopy; - final long dataOffset = src.offset + headerCnt; - final long dataLength = src.length; - final long expectedCRC; - final ByteArrayWindow quickCopy; - - // Verify the object isn't corrupt before sending. If it is, - // we report it missing instead. - // - try { - quickCopy = curs.quickCopy(this, dataOffset, dataLength); + // Verify the object isn't corrupt before sending. If it is, + // we report it missing instead. + // + try { + quickCopy = curs.quickCopy(this, dataOffset, dataLength); - if (validate && idx().hasCRC32Support()) { - assert(crc1 != null); - // Index has the CRC32 code cached, validate the object. - // - expectedCRC = idx().findCRC32(src); - if (quickCopy != null) { - quickCopy.crc32(crc1, dataOffset, (int) dataLength); - } else { - long pos = dataOffset; - long cnt = dataLength; - while (cnt > 0) { - final int n = (int) Math.min(cnt, buf.length); - readFully(pos, buf, 0, n, curs); - crc1.update(buf, 0, n); - pos += n; - cnt -= n; + if (validate && idx().hasCRC32Support()) { + assert(crc1 != null); + // Index has the CRC32 code cached, validate the object. + // + expectedCRC = idx().findCRC32(src); + if (quickCopy != null) { + quickCopy.crc32(crc1, dataOffset, (int) dataLength); + } else { + long pos = dataOffset; + long cnt = dataLength; + while (cnt > 0) { + final int n = (int) Math.min(cnt, buf.length); + readFully(pos, buf, 0, n, curs); + crc1.update(buf, 0, n); + pos += n; + cnt -= n; + } } + if (crc1.getValue() != expectedCRC) { + setCorrupt(src.offset); + throw new CorruptObjectException(MessageFormat.format( + JGitText.get().objectAtHasBadZlibStream, + Long.valueOf(src.offset), getPackFile())); + } + } else if (validate) { + // We don't have a CRC32 code in the index, so compute it + // now while inflating the raw data to get zlib to tell us + // whether or not the data is safe. + // + Inflater inf = curs.inflater(); + byte[] tmp = new byte[1024]; + if (quickCopy != null) { + quickCopy.check(inf, tmp, dataOffset, (int) dataLength); + } else { + assert(crc1 != null); + long pos = dataOffset; + long cnt = dataLength; + while (cnt > 0) { + final int n = (int) Math.min(cnt, buf.length); + readFully(pos, buf, 0, n, curs); + crc1.update(buf, 0, n); + inf.setInput(buf, 0, n); + while (inf.inflate(tmp, 0, tmp.length) > 0) + continue; + pos += n; + cnt -= n; + } + } + if (!inf.finished() || inf.getBytesRead() != dataLength) { + setCorrupt(src.offset); + throw new EOFException(MessageFormat.format( + JGitText.get().shortCompressedStreamAt, + Long.valueOf(src.offset))); + } + assert(crc1 != null); + expectedCRC = crc1.getValue(); + } else { + expectedCRC = -1; } - if (crc1.getValue() != expectedCRC) { - setCorrupt(src.offset); - throw new CorruptObjectException(MessageFormat.format( - JGitText.get().objectAtHasBadZlibStream, - Long.valueOf(src.offset), getPackFile())); - } - } else if (validate) { - // We don't have a CRC32 code in the index, so compute it - // now while inflating the raw data to get zlib to tell us - // whether or not the data is safe. + } catch (DataFormatException dataFormat) { + setCorrupt(src.offset); + + CorruptObjectException corruptObject = new CorruptObjectException( + MessageFormat.format( + JGitText.get().objectAtHasBadZlibStream, + Long.valueOf(src.offset), getPackFile()), + dataFormat); + + throw new StoredObjectRepresentationNotAvailableException( + corruptObject); + } + + if (quickCopy != null) { + // The entire object fits into a single byte array window slice, + // and we have it pinned. Write this out without copying. // - Inflater inf = curs.inflater(); - byte[] tmp = new byte[1024]; - if (quickCopy != null) { - quickCopy.check(inf, tmp, dataOffset, (int) dataLength); - } else { - assert(crc1 != null); + out.writeHeader(src, inflatedLength); + isHeaderWritten = true; + quickCopy.write(out, dataOffset, (int) dataLength); + + } else if (dataLength <= buf.length) { + // Tiny optimization: Lots of objects are very small deltas or + // deflated commits that are likely to fit in the copy buffer. + // + if (!validate) { long pos = dataOffset; long cnt = dataLength; while (cnt > 0) { final int n = (int) Math.min(cnt, buf.length); readFully(pos, buf, 0, n, curs); - crc1.update(buf, 0, n); - inf.setInput(buf, 0, n); - while (inf.inflate(tmp, 0, tmp.length) > 0) - continue; pos += n; cnt -= n; } } - if (!inf.finished() || inf.getBytesRead() != dataLength) { - setCorrupt(src.offset); - throw new EOFException(MessageFormat.format( - JGitText.get().shortCompressedStreamAt, - Long.valueOf(src.offset))); - } - assert(crc1 != null); - expectedCRC = crc1.getValue(); + out.writeHeader(src, inflatedLength); + isHeaderWritten = true; + out.write(buf, 0, (int) dataLength); } else { - expectedCRC = -1; - } - } catch (DataFormatException dataFormat) { - setCorrupt(src.offset); - - CorruptObjectException corruptObject = new CorruptObjectException( - MessageFormat.format( - JGitText.get().objectAtHasBadZlibStream, - Long.valueOf(src.offset), getPackFile()), - dataFormat); - - throw new StoredObjectRepresentationNotAvailableException( - corruptObject); - - } catch (IOException ioError) { - throw new StoredObjectRepresentationNotAvailableException(ioError); - } - - if (quickCopy != null) { - // The entire object fits into a single byte array window slice, - // and we have it pinned. Write this out without copying. - // - out.writeHeader(src, inflatedLength); - quickCopy.write(out, dataOffset, (int) dataLength); - - } else if (dataLength <= buf.length) { - // Tiny optimization: Lots of objects are very small deltas or - // deflated commits that are likely to fit in the copy buffer. - // - if (!validate) { + // Now we are committed to sending the object. As we spool it out, + // check its CRC32 code to make sure there wasn't corruption between + // the verification we did above, and us actually outputting it. + // long pos = dataOffset; long cnt = dataLength; while (cnt > 0) { final int n = (int) Math.min(cnt, buf.length); readFully(pos, buf, 0, n, curs); + if (validate) { + assert(crc2 != null); + crc2.update(buf, 0, n); + } + if (!isHeaderWritten) { + out.writeHeader(src, inflatedLength); + isHeaderWritten = true; + } + out.write(buf, 0, n); pos += n; cnt -= n; } - } - out.writeHeader(src, inflatedLength); - out.write(buf, 0, (int) dataLength); - } else { - // Now we are committed to sending the object. As we spool it out, - // check its CRC32 code to make sure there wasn't corruption between - // the verification we did above, and us actually outputting it. - // - out.writeHeader(src, inflatedLength); - long pos = dataOffset; - long cnt = dataLength; - while (cnt > 0) { - final int n = (int) Math.min(cnt, buf.length); - readFully(pos, buf, 0, n, curs); if (validate) { assert(crc2 != null); - crc2.update(buf, 0, n); + if (crc2.getValue() != expectedCRC) { + throw new CorruptObjectException(MessageFormat.format( + JGitText.get().objectAtHasBadZlibStream, + Long.valueOf(src.offset), getPackFile())); + } } - out.write(buf, 0, n); - pos += n; - cnt -= n; } - if (validate) { - assert(crc2 != null); - if (crc2.getValue() != expectedCRC) { - throw new CorruptObjectException(MessageFormat.format( - JGitText.get().objectAtHasBadZlibStream, - Long.valueOf(src.offset), getPackFile())); - } + } catch (IOException ioError) { + if (!isHeaderWritten) { + throw new StoredObjectRepresentationNotAvailableException(ioError); } + throw ioError; } } -- cgit v1.2.3 From f7a4dd03560d10d504f63b9e2caeb32a6bf2cc0b Mon Sep 17 00:00:00 2001 From: Jacek Centkowski Date: Mon, 11 Nov 2024 12:36:03 +0100 Subject: Rename numberOfPackFilesAfterBitmap to numberOfPackFilesSinceBitmap As sugested in I608011462f1. Change-Id: If66226dd7b08ae768413fa614df5dcb6b44dc118 --- ...NumberOfPackFilesAfterBitmapStatisticsTest.java | 173 --------------------- ...NumberOfPackFilesSinceBitmapStatisticsTest.java | 173 +++++++++++++++++++++ .../org/eclipse/jgit/internal/storage/file/GC.java | 8 +- 3 files changed, 177 insertions(+), 177 deletions(-) delete mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcNumberOfPackFilesAfterBitmapStatisticsTest.java create mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcNumberOfPackFilesSinceBitmapStatisticsTest.java (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcNumberOfPackFilesAfterBitmapStatisticsTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcNumberOfPackFilesAfterBitmapStatisticsTest.java deleted file mode 100644 index e5a391f2e3..0000000000 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcNumberOfPackFilesAfterBitmapStatisticsTest.java +++ /dev/null @@ -1,173 +0,0 @@ -/* - * Copyright (c) 2024 Jacek Centkowski and others. - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.storage.file; - -import static org.junit.Assert.assertEquals; - -import java.io.BufferedOutputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.nio.file.Files; -import java.util.HashSet; -import java.util.Set; -import java.util.stream.StreamSupport; - -import org.eclipse.jgit.internal.storage.file.PackIndex.MutableEntry; -import org.eclipse.jgit.internal.storage.pack.PackExt; -import org.eclipse.jgit.internal.storage.pack.PackWriter; -import org.eclipse.jgit.junit.TestRepository; -import org.eclipse.jgit.lib.NullProgressMonitor; -import org.eclipse.jgit.lib.ObjectId; -import org.eclipse.jgit.lib.PersonIdent; -import org.eclipse.jgit.lib.Ref; -import org.eclipse.jgit.revwalk.RevCommit; -import org.eclipse.jgit.util.FileUtils; -import org.junit.Test; - -public class GcNumberOfPackFilesAfterBitmapStatisticsTest extends GcTestCase { - @Test - public void testShouldReportZeroObjectsForInitializedRepo() - throws IOException { - assertEquals(0L, gc.getStatistics().numberOfPackFilesAfterBitmap); - } - - @Test - public void testShouldReportAllPackFilesWhenNoGcWasPerformed() - throws Exception { - packAndPrune(); - long result = gc.getStatistics().numberOfPackFilesAfterBitmap; - - assertEquals(repo.getObjectDatabase().getPacks().size(), result); - } - - @Test - public void testShouldReportNoObjectsDirectlyAfterGc() throws Exception { - // given - addCommit(null); - gc.gc().get(); - assertEquals(1L, repositoryBitmapFiles()); - assertEquals(0L, gc.getStatistics().numberOfPackFilesAfterBitmap); - } - - @Test - public void testShouldReportNewObjectsAfterGcWhenRepositoryProgresses() - throws Exception { - // commit & gc - RevCommit parent = addCommit(null); - gc.gc().get(); - assertEquals(1L, repositoryBitmapFiles()); - - // progress & pack - addCommit(parent); - packAndPrune(); - - assertEquals(1L, gc.getStatistics().numberOfPackFilesAfterBitmap); - } - - @Test - public void testShouldReportNewObjectsFromTheLatestBitmapWhenRepositoryProgresses() - throws Exception { - // commit & gc - RevCommit parent = addCommit(null); - gc.gc().get(); - assertEquals(1L, repositoryBitmapFiles()); - - // progress & gc - parent = addCommit(parent); - gc.gc().get(); - assertEquals(2L, repositoryBitmapFiles()); - - // progress & pack - addCommit(parent); - packAndPrune(); - - assertEquals(1L, gc.getStatistics().numberOfPackFilesAfterBitmap); - } - - private void packAndPrune() throws Exception { - try (SkipNonExistingFilesTestRepository testRepo = new SkipNonExistingFilesTestRepository( - repo)) { - testRepo.packAndPrune(); - } - } - - private RevCommit addCommit(RevCommit parent) throws Exception { - return tr.branch("master").commit() - .author(new PersonIdent("repo-metrics", "repo@metrics.com")) - .parent(parent).create(); - } - - private long repositoryBitmapFiles() throws IOException { - return StreamSupport - .stream(Files - .newDirectoryStream(repo.getObjectDatabase() - .getPackDirectory().toPath(), "pack-*.bitmap") - .spliterator(), false) - .count(); - } - - /** - * The TestRepository has a {@link TestRepository#packAndPrune()} function - * but it fails in the last step after GC was performed as it doesn't - * SKIP_MISSING files. In order to circumvent it was copied and improved - * here. - */ - private static class SkipNonExistingFilesTestRepository - extends TestRepository { - private final FileRepository repo; - - private SkipNonExistingFilesTestRepository(FileRepository db) throws IOException { - super(db); - repo = db; - } - - @Override - public void packAndPrune() throws Exception { - ObjectDirectory odb = repo.getObjectDatabase(); - NullProgressMonitor m = NullProgressMonitor.INSTANCE; - - final PackFile pack, idx; - try (PackWriter pw = new PackWriter(repo)) { - Set all = new HashSet<>(); - for (Ref r : repo.getRefDatabase().getRefs()) - all.add(r.getObjectId()); - pw.preparePack(m, all, PackWriter.NONE); - - pack = new PackFile(odb.getPackDirectory(), pw.computeName(), - PackExt.PACK); - try (OutputStream out = new BufferedOutputStream( - new FileOutputStream(pack))) { - pw.writePack(m, m, out); - } - pack.setReadOnly(); - - idx = pack.create(PackExt.INDEX); - try (OutputStream out = new BufferedOutputStream( - new FileOutputStream(idx))) { - pw.writeIndex(out); - } - idx.setReadOnly(); - } - - odb.openPack(pack); - updateServerInfo(); - - // alternative packAndPrune implementation that skips missing files - // after GC. - for (Pack p : odb.getPacks()) { - for (MutableEntry e : p) - FileUtils.delete(odb.fileFor(e.toObjectId()), - FileUtils.SKIP_MISSING); - } - } - } -} diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcNumberOfPackFilesSinceBitmapStatisticsTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcNumberOfPackFilesSinceBitmapStatisticsTest.java new file mode 100644 index 0000000000..42cb3cd0c2 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcNumberOfPackFilesSinceBitmapStatisticsTest.java @@ -0,0 +1,173 @@ +/* + * Copyright (c) 2024 Jacek Centkowski and others. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.storage.file; + +import static org.junit.Assert.assertEquals; + +import java.io.BufferedOutputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.util.HashSet; +import java.util.Set; +import java.util.stream.StreamSupport; + +import org.eclipse.jgit.internal.storage.file.PackIndex.MutableEntry; +import org.eclipse.jgit.internal.storage.pack.PackExt; +import org.eclipse.jgit.internal.storage.pack.PackWriter; +import org.eclipse.jgit.junit.TestRepository; +import org.eclipse.jgit.lib.NullProgressMonitor; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.PersonIdent; +import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.util.FileUtils; +import org.junit.Test; + +public class GcNumberOfPackFilesSinceBitmapStatisticsTest extends GcTestCase { + @Test + public void testShouldReportZeroObjectsForInitializedRepo() + throws IOException { + assertEquals(0L, gc.getStatistics().numberOfPackFilesSinceBitmap); + } + + @Test + public void testShouldReportAllPackFilesWhenNoGcWasPerformed() + throws Exception { + packAndPrune(); + long result = gc.getStatistics().numberOfPackFilesSinceBitmap; + + assertEquals(repo.getObjectDatabase().getPacks().size(), result); + } + + @Test + public void testShouldReportNoObjectsDirectlyAfterGc() throws Exception { + // given + addCommit(null); + gc.gc().get(); + assertEquals(1L, repositoryBitmapFiles()); + assertEquals(0L, gc.getStatistics().numberOfPackFilesSinceBitmap); + } + + @Test + public void testShouldReportNewObjectsSinceGcWhenRepositoryProgresses() + throws Exception { + // commit & gc + RevCommit parent = addCommit(null); + gc.gc().get(); + assertEquals(1L, repositoryBitmapFiles()); + + // progress & pack + addCommit(parent); + packAndPrune(); + + assertEquals(1L, gc.getStatistics().numberOfPackFilesSinceBitmap); + } + + @Test + public void testShouldReportNewObjectsFromTheLatestBitmapWhenRepositoryProgresses() + throws Exception { + // commit & gc + RevCommit parent = addCommit(null); + gc.gc().get(); + assertEquals(1L, repositoryBitmapFiles()); + + // progress & gc + parent = addCommit(parent); + gc.gc().get(); + assertEquals(2L, repositoryBitmapFiles()); + + // progress & pack + addCommit(parent); + packAndPrune(); + + assertEquals(1L, gc.getStatistics().numberOfPackFilesSinceBitmap); + } + + private void packAndPrune() throws Exception { + try (SkipNonExistingFilesTestRepository testRepo = new SkipNonExistingFilesTestRepository( + repo)) { + testRepo.packAndPrune(); + } + } + + private RevCommit addCommit(RevCommit parent) throws Exception { + return tr.branch("master").commit() + .author(new PersonIdent("repo-metrics", "repo@metrics.com")) + .parent(parent).create(); + } + + private long repositoryBitmapFiles() throws IOException { + return StreamSupport + .stream(Files + .newDirectoryStream(repo.getObjectDatabase() + .getPackDirectory().toPath(), "pack-*.bitmap") + .spliterator(), false) + .count(); + } + + /** + * The TestRepository has a {@link TestRepository#packAndPrune()} function + * but it fails in the last step after GC was performed as it doesn't + * SKIP_MISSING files. In order to circumvent it was copied and improved + * here. + */ + private static class SkipNonExistingFilesTestRepository + extends TestRepository { + private final FileRepository repo; + + private SkipNonExistingFilesTestRepository(FileRepository db) throws IOException { + super(db); + repo = db; + } + + @Override + public void packAndPrune() throws Exception { + ObjectDirectory odb = repo.getObjectDatabase(); + NullProgressMonitor m = NullProgressMonitor.INSTANCE; + + final PackFile pack, idx; + try (PackWriter pw = new PackWriter(repo)) { + Set all = new HashSet<>(); + for (Ref r : repo.getRefDatabase().getRefs()) + all.add(r.getObjectId()); + pw.preparePack(m, all, PackWriter.NONE); + + pack = new PackFile(odb.getPackDirectory(), pw.computeName(), + PackExt.PACK); + try (OutputStream out = new BufferedOutputStream( + new FileOutputStream(pack))) { + pw.writePack(m, m, out); + } + pack.setReadOnly(); + + idx = pack.create(PackExt.INDEX); + try (OutputStream out = new BufferedOutputStream( + new FileOutputStream(idx))) { + pw.writeIndex(out); + } + idx.setReadOnly(); + } + + odb.openPack(pack); + updateServerInfo(); + + // alternative packAndPrune implementation that skips missing files + // after GC. + for (Pack p : odb.getPacks()) { + for (MutableEntry e : p) + FileUtils.delete(odb.fileFor(e.toObjectId()), + FileUtils.SKIP_MISSING); + } + } + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java index ee6abc372c..9494057a60 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java @@ -1512,7 +1512,7 @@ public class GC { * The number of pack files that were created after the last bitmap * generation. */ - public long numberOfPackFilesAfterBitmap; + public long numberOfPackFilesSinceBitmap; /** * The number of objects stored as loose objects. @@ -1549,8 +1549,8 @@ public class GC { final StringBuilder b = new StringBuilder(); b.append("numberOfPackedObjects=").append(numberOfPackedObjects); //$NON-NLS-1$ b.append(", numberOfPackFiles=").append(numberOfPackFiles); //$NON-NLS-1$ - b.append(", numberOfPackFilesAfterBitmap=") //$NON-NLS-1$ - .append(numberOfPackFilesAfterBitmap); + b.append(", numberOfPackFilesSinceBitmap=") //$NON-NLS-1$ + .append(numberOfPackFilesSinceBitmap); b.append(", numberOfLooseObjects=").append(numberOfLooseObjects); //$NON-NLS-1$ b.append(", numberOfLooseRefs=").append(numberOfLooseRefs); //$NON-NLS-1$ b.append(", numberOfPackedRefs=").append(numberOfPackedRefs); //$NON-NLS-1$ @@ -1578,7 +1578,7 @@ public class GC { if (p.getBitmapIndex() != null) { ret.numberOfBitmaps += p.getBitmapIndex().getBitmapCount(); } else { - ret.numberOfPackFilesAfterBitmap++; + ret.numberOfPackFilesSinceBitmap++; } } File objDir = repo.getObjectsDirectory(); -- cgit v1.2.3 From c64bc936938b1388065be1c89eb929278b505a87 Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Wed, 13 Nov 2024 11:58:06 -0800 Subject: SystemReader: Give a default implementation to #getTimezoneAt() This abstract method forces subclasses (e.g. DelegateSystemReader in gerrit) to update their code, but there is no strong reason to make it abstract (subclasses can override it if needed). Make the method concrete using the current default implementation (which is the same in the mock). Change-Id: Id1df0d71cab1d86879adf48e782f0050d3abcfa9 --- .../src/org/eclipse/jgit/junit/MockSystemReader.java | 5 ----- org.eclipse.jgit/src/org/eclipse/jgit/util/SystemReader.java | 9 +++------ 2 files changed, 3 insertions(+), 11 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/MockSystemReader.java b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/MockSystemReader.java index b0365aa7e1..b0a1827265 100644 --- a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/MockSystemReader.java +++ b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/MockSystemReader.java @@ -244,11 +244,6 @@ public class MockSystemReader extends SystemReader { return getTimeZone().getOffset(when) / (60 * 1000); } - @Override - public ZoneOffset getTimeZoneAt(Instant when) { - return getTimeZoneId().getRules().getOffset(when); - } - @Override public TimeZone getTimeZone() { return TimeZone.getTimeZone("GMT-03:30"); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/SystemReader.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/SystemReader.java index 7150e471bc..18b0e152c9 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/SystemReader.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/SystemReader.java @@ -178,11 +178,6 @@ public abstract class SystemReader { public int getTimezone(long when) { return getTimeZone().getOffset(when) / (60 * 1000); } - - @Override - public ZoneOffset getTimeZoneAt(Instant when) { - return getTimeZoneId().getRules().getOffset(when); - } } /** @@ -569,7 +564,9 @@ public abstract class SystemReader { * @return the local time zone * @since 7.1 */ - public abstract ZoneOffset getTimeZoneAt(Instant when); + public ZoneOffset getTimeZoneAt(Instant when) { + return getTimeZoneId().getRules().getOffset(when); + } /** * Get system time zone, possibly mocked for testing -- cgit v1.2.3 From 6cf5d194f528428f94bda3e2a1a8468244008c81 Mon Sep 17 00:00:00 2001 From: XenoAmess Date: Tue, 24 Sep 2024 04:18:06 +0800 Subject: RawText: improve performance of isCrLfText and isBinary Inline the function isBinary(byte, byte), and reduce several duplicated checks in it, for better performance. Change-Id: Ida855ed4fd7456d8fb7ed68f3af2dbfa0e25897c --- .../eclipse/jgit/benchmarks/RawTextBenchmark.java | 454 +++++++++++++++++++++ .../src/org/eclipse/jgit/diff/RawText.java | 49 ++- 2 files changed, 483 insertions(+), 20 deletions(-) create mode 100644 org.eclipse.jgit.benchmarks/src/org/eclipse/jgit/benchmarks/RawTextBenchmark.java (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.benchmarks/src/org/eclipse/jgit/benchmarks/RawTextBenchmark.java b/org.eclipse.jgit.benchmarks/src/org/eclipse/jgit/benchmarks/RawTextBenchmark.java new file mode 100644 index 0000000000..19297ebebb --- /dev/null +++ b/org.eclipse.jgit.benchmarks/src/org/eclipse/jgit/benchmarks/RawTextBenchmark.java @@ -0,0 +1,454 @@ +/* + * Copyright (C) 2022, Matthias Sohn and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.benchmarks; + +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +import java.util.concurrent.TimeUnit; + +import static org.eclipse.jgit.diff.RawText.getBufferSize; +import static org.eclipse.jgit.diff.RawText.isBinary; +import static org.eclipse.jgit.diff.RawText.isCrLfText; + +@State(Scope.Thread) +public class RawTextBenchmark { + + @State(Scope.Benchmark) + public static class BenchmarkState { + + @Param({"1", "2", "3", "4", "5", "6"}) + int testIndex; + + @Param({"false", "true"}) + boolean complete; + + byte[] bytes; + + @Setup + public void setupBenchmark() { + switch (testIndex) { + case 1: { + byte[] tmpBytes = "a".repeat(102400).getBytes(); + bytes = tmpBytes; + break; + } + case 2: { + byte[] tmpBytes = "a".repeat(102400).getBytes(); + byte[] tmpBytes2 = new byte[tmpBytes.length + 1]; + System.arraycopy(tmpBytes, 0, tmpBytes2, 0, tmpBytes.length); + tmpBytes2[500] = '\0'; + tmpBytes2[tmpBytes.length] = '\0'; + bytes = tmpBytes2; + break; + } + case 3: { + byte[] tmpBytes = "a".repeat(102400).getBytes(); + byte[] tmpBytes2 = new byte[tmpBytes.length + 1]; + System.arraycopy(tmpBytes, 0, tmpBytes2, 0, tmpBytes.length); + tmpBytes2[500] = '\r'; + tmpBytes2[tmpBytes.length] = '\r'; + bytes = tmpBytes2; + break; + } + case 4: { + byte[] tmpBytes = "a".repeat(102400).getBytes(); + byte[] tmpBytes2 = new byte[tmpBytes.length + 1]; + System.arraycopy(tmpBytes, 0, tmpBytes2, 0, tmpBytes.length); + tmpBytes2[499] = '\r'; + tmpBytes2[500] = '\n'; + tmpBytes2[tmpBytes.length - 1] = '\r'; + tmpBytes2[tmpBytes.length] = '\n'; + bytes = tmpBytes2; + break; + } + case 5: { + byte[] tmpBytes = "a".repeat(102400).getBytes(); + tmpBytes[0] = '\0'; + bytes = tmpBytes; + break; + } + case 6: { + byte[] tmpBytes = "a".repeat(102400).getBytes(); + tmpBytes[0] = '\r'; + bytes = tmpBytes; + break; + } + default: + } + } + + @TearDown + public void teardown() { + } + } + + @Benchmark + @BenchmarkMode({Mode.AverageTime}) + @OutputTimeUnit(TimeUnit.NANOSECONDS) + @Warmup(iterations = 2, time = 5, timeUnit = TimeUnit.SECONDS) + @Measurement(iterations = 2, time = 5, timeUnit = TimeUnit.SECONDS) + @Fork(1) + public void testIsCrLfTextOld(Blackhole blackhole, BenchmarkState state) { + blackhole.consume( + isCrLfTextOld( + state.bytes, + state.bytes.length, + state.complete + ) + ); + } + + @Benchmark + @BenchmarkMode({Mode.AverageTime}) + @OutputTimeUnit(TimeUnit.NANOSECONDS) + @Warmup(iterations = 2, time = 5, timeUnit = TimeUnit.SECONDS) + @Measurement(iterations = 2, time = 5, timeUnit = TimeUnit.SECONDS) + @Fork(1) + public void testIsCrLfTextNewCandidate1(Blackhole blackhole, BenchmarkState state) { + blackhole.consume( + isCrLfTextNewCandidate1( + state.bytes, + state.bytes.length, + state.complete + ) + ); + } + + @Benchmark + @BenchmarkMode({Mode.AverageTime}) + @OutputTimeUnit(TimeUnit.NANOSECONDS) + @Warmup(iterations = 2, time = 5, timeUnit = TimeUnit.SECONDS) + @Measurement(iterations = 2, time = 5, timeUnit = TimeUnit.SECONDS) + @Fork(1) + public void testIsCrLfTextNewCandidate2(Blackhole blackhole, BenchmarkState state) { + blackhole.consume( + isCrLfTextNewCandidate2( + state.bytes, + state.bytes.length, + state.complete + ) + ); + } + + @Benchmark + @BenchmarkMode({Mode.AverageTime}) + @OutputTimeUnit(TimeUnit.NANOSECONDS) + @Warmup(iterations = 2, time = 5, timeUnit = TimeUnit.SECONDS) + @Measurement(iterations = 2, time = 5, timeUnit = TimeUnit.SECONDS) + @Fork(1) + public void testIsCrLfTextNewCandidate3(Blackhole blackhole, BenchmarkState state) { + blackhole.consume( + isCrLfTextNewCandidate3( + state.bytes, + state.bytes.length, + state.complete + ) + ); + } + + @Benchmark + @BenchmarkMode({Mode.AverageTime}) + @OutputTimeUnit(TimeUnit.NANOSECONDS) + @Warmup(iterations = 2, time = 5, timeUnit = TimeUnit.SECONDS) + @Measurement(iterations = 2, time = 5, timeUnit = TimeUnit.SECONDS) + @Fork(1) + public void testIsCrLfTextNew(Blackhole blackhole, BenchmarkState state) { + blackhole.consume( + isCrLfText( + state.bytes, + state.bytes.length, + state.complete + ) + ); + } + + @Benchmark + @BenchmarkMode({Mode.AverageTime}) + @OutputTimeUnit(TimeUnit.NANOSECONDS) + @Warmup(iterations = 2, time = 5, timeUnit = TimeUnit.SECONDS) + @Measurement(iterations = 2, time = 5, timeUnit = TimeUnit.SECONDS) + @Fork(1) + public void testIsBinaryOld(Blackhole blackhole, BenchmarkState state) { + blackhole.consume( + isBinaryOld( + state.bytes, + state.bytes.length, + state.complete + ) + ); + } + + + @Benchmark + @BenchmarkMode({Mode.AverageTime}) + @OutputTimeUnit(TimeUnit.NANOSECONDS) + @Warmup(iterations = 2, time = 5, timeUnit = TimeUnit.SECONDS) + @Measurement(iterations = 2, time = 5, timeUnit = TimeUnit.SECONDS) + @Fork(1) + public void testIsBinaryNew(Blackhole blackhole, BenchmarkState state) { + blackhole.consume( + isBinary( + state.bytes, + state.bytes.length, + state.complete + ) + ); + } + + + /** + * Determine heuristically whether a byte array represents binary (as + * opposed to text) content. + * + * @param raw + * the raw file content. + * @param length + * number of bytes in {@code raw} to evaluate. This should be + * {@code raw.length} unless {@code raw} was over-allocated by + * the caller. + * @param complete + * whether {@code raw} contains the whole data + * @return true if raw is likely to be a binary file, false otherwise + * @since 6.0 + */ + public static boolean isBinaryOld(byte[] raw, int length, boolean complete) { + // Similar heuristic as C Git. Differences: + // - limited buffer size; may be only the beginning of a large blob + // - no counting of printable vs. non-printable bytes < 0x20 and 0x7F + int maxLength = getBufferSize(); + boolean isComplete = complete; + if (length > maxLength) { + // We restrict the length in all cases to getBufferSize() to get + // predictable behavior. Sometimes we load streams, and sometimes we + // have the full data in memory. With streams, we never look at more + // than the first getBufferSize() bytes. If we looked at more when + // we have the full data, different code paths in JGit might come to + // different conclusions. + length = maxLength; + isComplete = false; + } + byte last = 'x'; // Just something inconspicuous. + for (int ptr = 0; ptr < length; ptr++) { + byte curr = raw[ptr]; + if (isBinary(curr, last)) { + return true; + } + last = curr; + } + if (isComplete) { + // Buffer contains everything... + return last == '\r'; // ... so this must be a lone CR + } + return false; + } + + /** + * Determine heuristically whether a byte array represents text content + * using CR-LF as line separator. + * + * @param raw the raw file content. + * @param length number of bytes in {@code raw} to evaluate. + * @param complete whether {@code raw} contains the whole data + * @return {@code true} if raw is likely to be CR-LF delimited text, + * {@code false} otherwise + * @since 6.0 + */ + public static boolean isCrLfTextOld(byte[] raw, int length, boolean complete) { + boolean has_crlf = false; + byte last = 'x'; // Just something inconspicuous + for (int ptr = 0; ptr < length; ptr++) { + byte curr = raw[ptr]; + if (isBinary(curr, last)) { + return false; + } + if (curr == '\n' && last == '\r') { + has_crlf = true; + } + last = curr; + } + if (last == '\r') { + if (complete) { + // Lone CR: it's binary after all. + return false; + } + // Tough call. If the next byte, which we don't have, would be a + // '\n', it'd be a CR-LF text, otherwise it'd be binary. Just decide + // based on what we already scanned; it wasn't binary until now. + } + return has_crlf; + } + + /** + * Determine heuristically whether a byte array represents text content + * using CR-LF as line separator. + * + * @param raw + * the raw file content. + * @param length + * number of bytes in {@code raw} to evaluate. + * @return {@code true} if raw is likely to be CR-LF delimited text, + * {@code false} otherwise + * @param complete + * whether {@code raw} contains the whole data + * @since 6.0 + */ + public static boolean isCrLfTextNewCandidate1(byte[] raw, int length, boolean complete) { + boolean has_crlf = false; + + // first detect empty + if (length <= 0) { + return false; + } + + // next detect '\0' + for (int reversePtr = length - 1; reversePtr >= 0; --reversePtr) { + if (raw[reversePtr] == '\0') { + return false; + } + } + + // if '\r' be last, then if complete then return non-crlf + if (raw[length - 1] == '\r' && complete) { + return false; + } + + for (int ptr = 0; ptr < length - 1; ptr++) { + byte curr = raw[ptr]; + if (curr == '\r') { + byte next = raw[ptr + 1]; + if (next != '\n') { + return false; + } + // else + // we have crlf here + has_crlf = true; + // as next is '\n', it can never be '\r', just skip it from next check + ++ptr; + } + } + + return has_crlf; + } + + /** + * Determine heuristically whether a byte array represents text content + * using CR-LF as line separator. + * + * @param raw + * the raw file content. + * @param length + * number of bytes in {@code raw} to evaluate. + * @return {@code true} if raw is likely to be CR-LF delimited text, + * {@code false} otherwise + * @param complete + * whether {@code raw} contains the whole data + * @since 6.0 + */ + public static boolean isCrLfTextNewCandidate2(byte[] raw, int length, boolean complete) { + boolean has_crlf = false; + + // first detect empty + if (length <= 0) { + return false; + } + + // if '\r' be last, then if complete then return non-crlf + byte last = raw[length - 1]; + if (last == '\0' || last == '\r' && complete) { + return false; + } + + for (int ptr = 0; ptr < length - 1; ptr++) { + byte b = raw[ptr]; + switch (b) { + case '\0': + return false; + case '\r': { + ++ptr; + b = raw[ptr]; + if (b != '\n') { + return false; + } + // else + // we have crlf here + has_crlf = true; + // as next is '\n', it can never be '\r', just skip it from next check + break; + } + default: + // do nothing; + break; + } + } + + return has_crlf; + } + + /** + * Determine heuristically whether a byte array represents text content + * using CR-LF as line separator. + * + * @param raw + * the raw file content. + * @param length + * number of bytes in {@code raw} to evaluate. + * @return {@code true} if raw is likely to be CR-LF delimited text, + * {@code false} otherwise + * @param complete + * whether {@code raw} contains the whole data + * @since 6.0 + */ + public static boolean isCrLfTextNewCandidate3(byte[] raw, int length, boolean complete) { + boolean has_crlf = false; + + int ptr = -1; + byte current; + while (ptr < length - 2) { + current = raw[++ptr]; + if ('\0' == current || '\r' == current && (raw[++ptr] != '\n' || !(has_crlf = true))) { + return false; + } + } + + if (ptr == length - 2) { + // if '\r' be last, then if isComplete then return binary + current = raw[++ptr]; + if('\0' == current || '\r' == current && complete){ + return false; + } + } + + return has_crlf; + } + + + public static void main(String[] args) throws RunnerException { + Options opt = new OptionsBuilder() + .include(RawTextBenchmark.class.getSimpleName()) + .forks(1).jvmArgs("-ea").build(); + new Runner(opt).run(); + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/diff/RawText.java b/org.eclipse.jgit/src/org/eclipse/jgit/diff/RawText.java index 76dc87e72b..5a4d2aa45a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/diff/RawText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/diff/RawText.java @@ -360,18 +360,22 @@ public class RawText extends Sequence { length = maxLength; isComplete = false; } - byte last = 'x'; // Just something inconspicuous. - for (int ptr = 0; ptr < length; ptr++) { - byte curr = raw[ptr]; - if (isBinary(curr, last)) { + + int ptr = -1; + byte current; + while (ptr < length - 2) { + current = raw[++ptr]; + if ('\0' == current || '\r' == current && raw[++ptr] != '\n') { return true; } - last = curr; } - if (isComplete) { - // Buffer contains everything... - return last == '\r'; // ... so this must be a lone CR + + if (ptr == length - 2) { + // if '\r' be last, then if isComplete then return binary + current = raw[++ptr]; + return '\0' == current || '\r' == current && isComplete; } + return false; } @@ -467,26 +471,30 @@ public class RawText extends Sequence { */ public static boolean isCrLfText(byte[] raw, int length, boolean complete) { boolean has_crlf = false; - byte last = 'x'; // Just something inconspicuous - for (int ptr = 0; ptr < length; ptr++) { - byte curr = raw[ptr]; - if (isBinary(curr, last)) { + + int ptr = -1; + byte current; + while (ptr < length - 2) { + current = raw[++ptr]; + if (current == '\0') { return false; } - if (curr == '\n' && last == '\r') { + if (current == '\r') { + if (raw[++ptr] != '\n') { + return false; + } has_crlf = true; } - last = curr; } - if (last == '\r') { - if (complete) { - // Lone CR: it's binary after all. + + if (ptr == length - 2) { + // if '\r' be last, then if isComplete then return binary + current = raw[++ptr]; + if('\0' == current || '\r' == current && complete){ return false; } - // Tough call. If the next byte, which we don't have, would be a - // '\n', it'd be a CR-LF text, otherwise it'd be binary. Just decide - // based on what we already scanned; it wasn't binary until now. } + return has_crlf; } @@ -578,4 +586,5 @@ public class RawText extends Sequence { return new RawText(data, RawParseUtils.lineMapOrBinary(data, 0, (int) sz)); } } + } -- cgit v1.2.3 From 13a1ce7466eb620773930f1d485cb9b31c6bda2c Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Wed, 13 Nov 2024 16:11:17 -0800 Subject: [errorprone] RawText: Add parenthesis for explicit op precedence errorprone reports: [OperatorPrecedence] Use grouping parenthesis to make the operator precedence explicit Take the chance to fix also https://errorprone.info/bugpattern/YodaCondition in the same lines. Change-Id: I6d8f00842ef2bb24cd00fc413121b8a4e20c186b --- org.eclipse.jgit/src/org/eclipse/jgit/diff/RawText.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/diff/RawText.java b/org.eclipse.jgit/src/org/eclipse/jgit/diff/RawText.java index 5a4d2aa45a..fdfe533618 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/diff/RawText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/diff/RawText.java @@ -365,7 +365,7 @@ public class RawText extends Sequence { byte current; while (ptr < length - 2) { current = raw[++ptr]; - if ('\0' == current || '\r' == current && raw[++ptr] != '\n') { + if (current == '\0' || (current == '\r' && raw[++ptr] != '\n')) { return true; } } @@ -373,7 +373,7 @@ public class RawText extends Sequence { if (ptr == length - 2) { // if '\r' be last, then if isComplete then return binary current = raw[++ptr]; - return '\0' == current || '\r' == current && isComplete; + return current == '\0' || (current == '\r' && isComplete); } return false; @@ -490,7 +490,7 @@ public class RawText extends Sequence { if (ptr == length - 2) { // if '\r' be last, then if isComplete then return binary current = raw[++ptr]; - if('\0' == current || '\r' == current && complete){ + if (current == '\0' || (current == '\r' && complete)) { return false; } } -- cgit v1.2.3 From c67393562be7e3db0e39cb2d3ff5f8192e2f2cd5 Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Wed, 13 Nov 2024 16:25:42 -0800 Subject: SystemReader#now: make it a concrete method Abstract methods break subclasses (e.g. DelegateSystemReader in gerrit). Updating jgit and gerrit is simpler if we do not add them. I am not sure why some methods are abstract and others dont, but now() can be a concrete method. Make now() concrete. Implement it by default based on getCurrentTime(), so subclasses overriding that method get the same value. Change-Id: I697749f8cba698c5388ed13ebdc2b238d6259358 --- .../src/org/eclipse/jgit/junit/MockSystemReader.java | 6 ------ org.eclipse.jgit/src/org/eclipse/jgit/util/SystemReader.java | 6 +++++- 2 files changed, 5 insertions(+), 7 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/MockSystemReader.java b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/MockSystemReader.java index bf236ab17d..38f0d0b2cb 100644 --- a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/MockSystemReader.java +++ b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/MockSystemReader.java @@ -18,7 +18,6 @@ import java.lang.reflect.Field; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.time.Duration; -import java.time.Instant; import java.time.ZoneId; import java.time.ZoneOffset; import java.util.HashMap; @@ -204,11 +203,6 @@ public class MockSystemReader extends SystemReader { return now; } - @Override - public Instant now() { - return Instant.ofEpochMilli(now); - } - @Override public MonotonicClock getClock() { return () -> { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/SystemReader.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/SystemReader.java index 18b0e152c9..7cdf0ee1a2 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/SystemReader.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/SystemReader.java @@ -532,7 +532,11 @@ public abstract class SystemReader { * * @since 7.1 */ - public abstract Instant now(); + public Instant now() { + // Subclasses overriding getCurrentTime should keep working + // TODO(ifrade): Once we remove getCurrentTime, use Instant.now() + return Instant.ofEpochMilli(getCurrentTime()); + } /** * Get clock instance preferred by this system. -- cgit v1.2.3 From b7856f885bf02af20b07359823396a92ffa6e76e Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Tue, 12 Nov 2024 12:29:13 -0800 Subject: SystemReader: add method to get LocalDateTime Using #civilNow() because in the documentation, this calendar-based representation is called "civil time". Change-Id: Iaa363e66683cb548419666068a4ffef44a776e12 --- org.eclipse.jgit/src/org/eclipse/jgit/util/SystemReader.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/SystemReader.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/SystemReader.java index 7cdf0ee1a2..55cc878e02 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/SystemReader.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/SystemReader.java @@ -26,6 +26,7 @@ import java.nio.file.Paths; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.time.Instant; +import java.time.LocalDateTime; import java.time.ZoneId; import java.time.ZoneOffset; import java.util.Locale; @@ -538,6 +539,17 @@ public abstract class SystemReader { return Instant.ofEpochMilli(getCurrentTime()); } + /** + * Get "now" as civil time, in the System timezone + * + * @return the current system time + * + * @since 7.1 + */ + public LocalDateTime civilNow() { + return LocalDateTime.ofInstant(now(), getTimeZoneId()); + } + /** * Get clock instance preferred by this system. * -- cgit v1.2.3 From ba905906c1a0b56232ef987db4c6c77d44f25f25 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Thu, 14 Nov 2024 22:12:02 +0100 Subject: Change default similarity score to 50(%) to match git's default Git uses a default similarity score of 50(%) for rename detection. Fix RenameDetector to use the same default score. See https://git-scm.com/docs/git-diff#Documentation/git-diff.txt--Mltngt Bug: jgit-110 Change-Id: I4b75910a02bca1afc108ad9e5609fda1e49a29da --- .../src/org/eclipse/jgit/diff/SimilarityRenameDetector.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/diff/SimilarityRenameDetector.java b/org.eclipse.jgit/src/org/eclipse/jgit/diff/SimilarityRenameDetector.java index 5de7bac112..fb98df7c9e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/diff/SimilarityRenameDetector.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/diff/SimilarityRenameDetector.java @@ -80,7 +80,7 @@ class SimilarityRenameDetector { private long[] matrix; /** Score a pair must exceed to be considered a rename. */ - private int renameScore = 60; + private int renameScore = 50; /** * File size threshold (in bytes) for detecting renames. Files larger -- cgit v1.2.3 From 307ef6b4b51f5f9266f6212015a9b5a1f1e75df4 Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Mon, 11 Nov 2024 13:13:59 -0800 Subject: GitTimeParser: A date parser using the java.time API Replacement of GitDateParser that uses java.time classes instead of the obsolete Date. Updating GitDateParser would have been a mess of deprecation and methods with confusing names, so I opted for writing a parallel class with the new types. Some differences: * The new DateTimeFormatter is thread-safe, so we don't need the LocalThread cache * No code seems to use other locale than the default, we don't need to cache per locale either Change-Id: If24610a055a47702fb5b7be2fc35a7c722480ee3 --- .../jgit/util/GitTimeParserBadlyFormattedTest.java | 62 ++++++ .../org/eclipse/jgit/util/GitTimeParserTest.java | 247 +++++++++++++++++++++ .../src/org/eclipse/jgit/util/GitDateParser.java | 3 + .../src/org/eclipse/jgit/util/GitTimeParser.java | 202 +++++++++++++++++ 4 files changed, 514 insertions(+) create mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/util/GitTimeParserBadlyFormattedTest.java create mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/util/GitTimeParserTest.java create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/util/GitTimeParser.java (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/GitTimeParserBadlyFormattedTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/GitTimeParserBadlyFormattedTest.java new file mode 100644 index 0000000000..e5f162d11a --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/GitTimeParserBadlyFormattedTest.java @@ -0,0 +1,62 @@ +/* + * Copyright (C) 2012, Christian Halstrick and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.util; + +import static org.junit.Assert.assertThrows; + +import java.text.ParseException; + +import org.eclipse.jgit.junit.MockSystemReader; +import org.junit.After; +import org.junit.Before; +import org.junit.experimental.theories.DataPoints; +import org.junit.experimental.theories.Theories; +import org.junit.experimental.theories.Theory; +import org.junit.runner.RunWith; + +/** + * Tests which assert that unparseable Strings lead to ParseExceptions + */ +@RunWith(Theories.class) +public class GitTimeParserBadlyFormattedTest { + private String dateStr; + + @Before + public void setUp() { + MockSystemReader mockSystemReader = new MockSystemReader(); + SystemReader.setInstance(mockSystemReader); + } + + @After + public void tearDown() { + SystemReader.setInstance(null); + } + + public GitTimeParserBadlyFormattedTest(String dateStr) { + this.dateStr = dateStr; + } + + @DataPoints + public static String[] getDataPoints() { + return new String[] { "", "1970", "3000.3000.3000", "3 yesterday ago", + "now yesterday ago", "yesterdays", "3.day. 2.week.ago", + "day ago", "Gra Feb 21 15:35:00 2007 +0100", + "Sun Feb 21 15:35:00 2007 +0100", + "Wed Feb 21 15:35:00 Grand +0100" }; + } + + @Theory + public void badlyFormattedWithoutRef() { + assertThrows( + "The expected ParseException while parsing '" + dateStr + + "' did not occur.", + ParseException.class, () -> GitTimeParser.parse(dateStr)); + } +} diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/GitTimeParserTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/GitTimeParserTest.java new file mode 100644 index 0000000000..0e5eb283a4 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/GitTimeParserTest.java @@ -0,0 +1,247 @@ +/* + * Copyright (C) 2024, Christian Halstrick and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.util; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.text.ParseException; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.Period; +import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoField; +import java.time.temporal.TemporalAccessor; + +import org.eclipse.jgit.junit.MockSystemReader; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class GitTimeParserTest { + MockSystemReader mockSystemReader; + + @Before + public void setUp() { + mockSystemReader = new MockSystemReader(); + SystemReader.setInstance(mockSystemReader); + } + + @After + public void tearDown() { + SystemReader.setInstance(null); + } + + @Test + public void yesterday() throws ParseException { + LocalDateTime parse = GitTimeParser.parse("yesterday"); + + LocalDateTime now = SystemReader.getInstance().civilNow(); + assertEquals(Period.between(parse.toLocalDate(), now.toLocalDate()), + Period.ofDays(1)); + } + + @Test + public void never() throws ParseException { + LocalDateTime parse = GitTimeParser.parse("never"); + assertEquals(LocalDateTime.MAX, parse); + } + + @Test + public void now_pointInTime() throws ParseException { + LocalDateTime aTime = asLocalDateTime("2007-02-21 15:35:00 +0100"); + + LocalDateTime parsedNow = GitTimeParser.parse("now", aTime); + + assertEquals(aTime, parsedNow); + } + + @Test + public void now_systemTime() throws ParseException { + LocalDateTime firstNow = GitTimeParser.parse("now"); + assertEquals(SystemReader.getInstance().civilNow(), firstNow); + mockSystemReader.tick(10); + LocalDateTime secondNow = GitTimeParser.parse("now"); + assertTrue(secondNow.isAfter(firstNow)); + } + + @Test + public void weeksAgo() throws ParseException { + LocalDateTime aTime = asLocalDateTime("2007-02-21 15:35:00 +0100"); + + LocalDateTime parse = GitTimeParser.parse("2 weeks ago", aTime); + assertEquals(asLocalDateTime("2007-02-07 15:35:00 +0100"), parse); + } + + @Test + public void daysAndWeeksAgo() throws ParseException { + LocalDateTime aTime = asLocalDateTime("2007-02-21 15:35:00 +0100"); + + LocalDateTime twoWeeksAgoActual = GitTimeParser.parse("2 weeks ago", + aTime); + + LocalDateTime twoWeeksAgoExpected = asLocalDateTime( + "2007-02-07 15:35:00 +0100"); + assertEquals(twoWeeksAgoExpected, twoWeeksAgoActual); + + LocalDateTime combinedWhitespace = GitTimeParser + .parse("3 days 2 weeks ago", aTime); + LocalDateTime combinedWhitespaceExpected = asLocalDateTime( + "2007-02-04 15:35:00 +0100"); + assertEquals(combinedWhitespaceExpected, combinedWhitespace); + + LocalDateTime combinedDots = GitTimeParser.parse("3.day.2.week.ago", + aTime); + LocalDateTime combinedDotsExpected = asLocalDateTime( + "2007-02-04 15:35:00 +0100"); + assertEquals(combinedDotsExpected, combinedDots); + } + + @Test + public void hoursAgo() throws ParseException { + LocalDateTime aTime = asLocalDateTime("2007-02-21 17:35:00 +0100"); + + LocalDateTime twoHoursAgoActual = GitTimeParser.parse("2 hours ago", + aTime); + + LocalDateTime twoHoursAgoExpected = asLocalDateTime( + "2007-02-21 15:35:00 +0100"); + assertEquals(twoHoursAgoExpected, twoHoursAgoActual); + } + + @Test + public void hoursAgo_acrossDay() throws ParseException { + LocalDateTime aTime = asLocalDateTime("2007-02-21 00:35:00 +0100"); + + LocalDateTime twoHoursAgoActual = GitTimeParser.parse("2 hours ago", + aTime); + + LocalDateTime twoHoursAgoExpected = asLocalDateTime( + "2007-02-20 22:35:00 +0100"); + assertEquals(twoHoursAgoExpected, twoHoursAgoActual); + } + + @Test + public void minutesHoursAgoCombined() throws ParseException { + LocalDateTime aTime = asLocalDateTime("2007-02-04 15:35:00 +0100"); + + LocalDateTime combinedWhitespace = GitTimeParser + .parse("3 hours 2 minutes ago", aTime); + LocalDateTime combinedWhitespaceExpected = asLocalDateTime( + "2007-02-04 12:33:00 +0100"); + assertEquals(combinedWhitespaceExpected, combinedWhitespace); + + LocalDateTime combinedDots = GitTimeParser + .parse("3.hours.2.minutes.ago", aTime); + LocalDateTime combinedDotsExpected = asLocalDateTime( + "2007-02-04 12:33:00 +0100"); + assertEquals(combinedDotsExpected, combinedDots); + } + + @Test + public void minutesAgo() throws ParseException { + LocalDateTime aTime = asLocalDateTime("2007-02-21 17:35:10 +0100"); + + LocalDateTime twoMinutesAgo = GitTimeParser.parse("2 minutes ago", + aTime); + + LocalDateTime twoMinutesAgoExpected = asLocalDateTime( + "2007-02-21 17:33:10 +0100"); + assertEquals(twoMinutesAgoExpected, twoMinutesAgo); + } + + @Test + public void minutesAgo_acrossDay() throws ParseException { + LocalDateTime aTime = asLocalDateTime("2007-02-21 00:35:10 +0100"); + + LocalDateTime minutesAgoActual = GitTimeParser.parse("40 minutes ago", + aTime); + + LocalDateTime minutesAgoExpected = asLocalDateTime( + "2007-02-20 23:55:10 +0100"); + assertEquals(minutesAgoExpected, minutesAgoActual); + } + + @Test + public void iso() throws ParseException { + String dateStr = "2007-02-21 15:35:00 +0100"; + + LocalDateTime actual = GitTimeParser.parse(dateStr); + + LocalDateTime expected = asLocalDateTime(dateStr); + assertEquals(expected, actual); + } + + @Test + public void rfc() throws ParseException { + String dateStr = "Wed, 21 Feb 2007 15:35:00 +0100"; + + LocalDateTime actual = GitTimeParser.parse(dateStr); + + LocalDateTime expected = asLocalDateTime(dateStr, + "EEE, dd MMM yyyy HH:mm:ss Z"); + assertEquals(expected, actual); + } + + @Test + public void shortFmt() throws ParseException { + assertParsing("2007-02-21", "yyyy-MM-dd"); + } + + @Test + public void shortWithDots() throws ParseException { + assertParsing("2007.02.21", "yyyy.MM.dd"); + } + + @Test + public void shortWithSlash() throws ParseException { + assertParsing("02/21/2007", "MM/dd/yyyy"); + } + + @Test + public void shortWithDotsReverse() throws ParseException { + assertParsing("21.02.2007", "dd.MM.yyyy"); + } + + @Test + public void defaultFmt() throws ParseException { + assertParsing("Wed Feb 21 15:35:00 2007 +0100", + "EEE MMM dd HH:mm:ss yyyy Z"); + } + + @Test + public void local() throws ParseException { + assertParsing("Wed Feb 21 15:35:00 2007", "EEE MMM dd HH:mm:ss yyyy"); + } + + private static void assertParsing(String dateStr, String format) + throws ParseException { + LocalDateTime actual = GitTimeParser.parse(dateStr); + + LocalDateTime expected = asLocalDateTime(dateStr, format); + assertEquals(expected, actual); + } + + private static LocalDateTime asLocalDateTime(String dateStr) { + return asLocalDateTime(dateStr, "yyyy-MM-dd HH:mm:ss Z"); + } + + private static LocalDateTime asLocalDateTime(String dateStr, + String pattern) { + DateTimeFormatter fmt = DateTimeFormatter.ofPattern(pattern); + TemporalAccessor ta = fmt + .withZone(SystemReader.getInstance().getTimeZoneId()) + .withLocale(SystemReader.getInstance().getLocale()) + .parse(dateStr); + return ta.isSupported(ChronoField.HOUR_OF_DAY) ? LocalDateTime.from(ta) + : LocalDate.from(ta).atStartOfDay(); + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/GitDateParser.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/GitDateParser.java index 6a4b39652a..f080056546 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/GitDateParser.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/GitDateParser.java @@ -28,7 +28,10 @@ import org.eclipse.jgit.internal.JGitText; * used. One example is the parsing of the config parameter gc.pruneexpire. The * parser can handle only subset of what native gits approxidate parser * understands. + * + * @deprecated Use {@link GitTimeParser} instead. */ +@Deprecated(since = "7.1") public class GitDateParser { /** * The Date representing never. Though this is a concrete value, most diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/GitTimeParser.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/GitTimeParser.java new file mode 100644 index 0000000000..e238e3e92e --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/GitTimeParser.java @@ -0,0 +1,202 @@ +/* + * Copyright (C) 2024 Christian Halstrick and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.util; + +import java.text.MessageFormat; +import java.text.ParseException; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.time.temporal.ChronoField; +import java.time.temporal.TemporalAccessor; +import java.util.HashMap; +import java.util.Map; + +import org.eclipse.jgit.internal.JGitText; + +/** + * Parses strings with time and date specifications into + * {@link java.time.Instant}. + * + * When git needs to parse strings specified by the user this parser can be + * used. One example is the parsing of the config parameter gc.pruneexpire. The + * parser can handle only subset of what native gits approxidate parser + * understands. + * + * @since 7.1 + */ +public class GitTimeParser { + + private static final Map formatCache = new HashMap<>(); + + // An enum of all those formats which this parser can parse with the help of + // a DateTimeFormatter. There are other formats (e.g. the relative formats + // like "yesterday" or "1 week ago") which this parser can parse but which + // are not listed here because they are parsed without the help of a + // DateTimeFormatter. + enum ParseableSimpleDateFormat { + ISO("yyyy-MM-dd HH:mm:ss Z"), // //$NON-NLS-1$ + RFC("EEE, dd MMM yyyy HH:mm:ss Z"), // //$NON-NLS-1$ + SHORT("yyyy-MM-dd"), // //$NON-NLS-1$ + SHORT_WITH_DOTS_REVERSE("dd.MM.yyyy"), // //$NON-NLS-1$ + SHORT_WITH_DOTS("yyyy.MM.dd"), // //$NON-NLS-1$ + SHORT_WITH_SLASH("MM/dd/yyyy"), // //$NON-NLS-1$ + DEFAULT("EEE MMM dd HH:mm:ss yyyy Z"), // //$NON-NLS-1$ + LOCAL("EEE MMM dd HH:mm:ss yyyy"); //$NON-NLS-1$ + + private final String formatStr; + + ParseableSimpleDateFormat(String formatStr) { + this.formatStr = formatStr; + } + } + + /** + * Parses a string into a {@link java.time.LocalDateTime} using the default + * locale. Since this parser also supports relative formats (e.g. + * "yesterday") the caller can specify the reference date. These types of + * strings can be parsed: + *
    + *
  • "never"
    • + *
  • "now"
    • + *
  • "yesterday"
    • + *
  • "(x) years|months|weeks|days|hours|minutes|seconds ago"
    + * Multiple specs can be combined like in "2 weeks 3 days ago". Instead of ' + * ' one can use '.' to separate the words
    • + *
  • "yyyy-MM-dd HH:mm:ss Z" (ISO)
    • + *
  • "EEE, dd MMM yyyy HH:mm:ss Z" (RFC)
    • + *
  • "yyyy-MM-dd"
    • + *
  • "yyyy.MM.dd"
    • + *
  • "MM/dd/yyyy",
    • + *
  • "dd.MM.yyyy"
    • + *
  • "EEE MMM dd HH:mm:ss yyyy Z" (DEFAULT)
    • + *
  • "EEE MMM dd HH:mm:ss yyyy" (LOCAL)
    • + *
+ * + * @param dateStr + * the string to be parsed + * @return the parsed {@link java.time.LocalDateTime} + * @throws java.text.ParseException + * if the given dateStr was not recognized + */ + public static LocalDateTime parse(String dateStr) throws ParseException { + return parse(dateStr, SystemReader.getInstance().civilNow()); + } + + // Only tests seem to use this method + static LocalDateTime parse(String dateStr, LocalDateTime now) + throws ParseException { + dateStr = dateStr.trim(); + LocalDateTime ret; + + if ("never".equalsIgnoreCase(dateStr)) //$NON-NLS-1$ + return LocalDateTime.MAX; + ret = parse_relative(dateStr, now); + if (ret != null) + return ret; + for (ParseableSimpleDateFormat f : ParseableSimpleDateFormat.values()) { + try { + return parse_simple(dateStr, f); + } catch (DateTimeParseException e) { + // simply proceed with the next parser + } + } + ParseableSimpleDateFormat[] values = ParseableSimpleDateFormat.values(); + StringBuilder allFormats = new StringBuilder("\"") //$NON-NLS-1$ + .append(values[0].formatStr); + for (int i = 1; i < values.length; i++) + allFormats.append("\", \"").append(values[i].formatStr); //$NON-NLS-1$ + allFormats.append("\""); //$NON-NLS-1$ + throw new ParseException( + MessageFormat.format(JGitText.get().cannotParseDate, dateStr, + allFormats.toString()), + 0); + } + + // tries to parse a string with the formats supported by DateTimeFormatter + private static LocalDateTime parse_simple(String dateStr, + ParseableSimpleDateFormat f) throws DateTimeParseException { + DateTimeFormatter dateFormat = formatCache.computeIfAbsent(f, + format -> DateTimeFormatter.ofPattern(f.formatStr) + .withLocale(SystemReader.getInstance().getLocale())); + TemporalAccessor parsed = dateFormat.parse(dateStr); + return parsed.isSupported(ChronoField.HOUR_OF_DAY) + ? LocalDateTime.from(parsed) + : LocalDate.from(parsed).atStartOfDay(); + } + + // tries to parse a string with a relative time specification + @SuppressWarnings("nls") + private static LocalDateTime parse_relative(String dateStr, + LocalDateTime now) { + // check for the static words "yesterday" or "now" + if ("now".equals(dateStr)) { + return now; + } + + if ("yesterday".equals(dateStr)) { + return now.minusDays(1); + } + + // parse constructs like "3 days ago", "5.week.2.day.ago" + String[] parts = dateStr.split("\\.| "); + int partsLength = parts.length; + // check we have an odd number of parts (at least 3) and that the last + // part is "ago" + if (partsLength < 3 || (partsLength & 1) == 0 + || !"ago".equals(parts[parts.length - 1])) + return null; + int number; + for (int i = 0; i < parts.length - 2; i += 2) { + try { + number = Integer.parseInt(parts[i]); + } catch (NumberFormatException e) { + return null; + } + if (parts[i + 1] == null) { + return null; + } + switch (parts[i + 1]) { + case "year": + case "years": + now = now.minusYears(number); + break; + case "month": + case "months": + now = now.minusMonths(number); + break; + case "week": + case "weeks": + now = now.minusWeeks(number); + break; + case "day": + case "days": + now = now.minusDays(number); + break; + case "hour": + case "hours": + now = now.minusHours(number); + break; + case "minute": + case "minutes": + now = now.minusMinutes(number); + break; + case "second": + case "seconds": + now = now.minusSeconds(number); + break; + default: + return null; + } + } + return now; + } +} -- cgit v1.2.3 From 92b35db291b5c661037b2724ac40488dbb44f52d Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Fri, 15 Nov 2024 10:26:21 -0800 Subject: PersonIdent: Use java.time instead of older Date and milliseconds From errorprone: Date has a bad API that leads to bugs; prefer java.time.Instant or LocalDate. Replace the long with milliseconds and int with minutes offset with an Instant and a ZoneOffset. Create new constructors and deprecate variants with Date, milliseconds and minute offsets. When comparing instances of PersonIdent truncate the timestamp precision to 1 second since git commit timestamps are persisted with 1 second precision [1]. [1] https://git-scm.com/docs/git-commit#Documentation/git-commit.txt-Gitinternalformat Change-Id: Id4ba1f108e1ba0bfcdd87ba37c67e2d3cc7d254f --- .../tst/org/eclipse/jgit/lib/PersonIdentTest.java | 6 +- .../src/org/eclipse/jgit/lib/PersonIdent.java | 141 +++++++++++++++------ 2 files changed, 103 insertions(+), 44 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/PersonIdentTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/PersonIdentTest.java index 97da1757e0..943a68b82c 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/PersonIdentTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/PersonIdentTest.java @@ -55,7 +55,8 @@ public class PersonIdentTest { p.getWhenAsInstant()); assertEquals("A U Thor 1142878501 -0500", p.toExternalString()); - assertEquals(ZoneId.of("GMT-05:00"), p.getZoneId()); + assertEquals(ZoneId.of("GMT-05:00").getRules().getOffset( + Instant.ofEpochMilli(1142878501000L)), p.getZoneOffset()); } @Test @@ -69,7 +70,8 @@ public class PersonIdentTest { p.getWhenAsInstant()); assertEquals("A U Thor 1142878501 +0530", p.toExternalString()); - assertEquals(ZoneId.of("GMT+05:30"), p.getZoneId()); + assertEquals(ZoneId.of("GMT+05:30").getRules().getOffset( + Instant.ofEpochMilli(1142878501000L)), p.getZoneOffset()); } @SuppressWarnings("unused") diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/PersonIdent.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/PersonIdent.java index 3ba055aae8..a59dde7083 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/PersonIdent.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/PersonIdent.java @@ -13,9 +13,11 @@ package org.eclipse.jgit.lib; import java.io.Serializable; -import java.text.SimpleDateFormat; import java.time.Instant; import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoUnit; import java.util.Date; import java.util.Locale; import java.util.TimeZone; @@ -48,6 +50,17 @@ public class PersonIdent implements Serializable { return TimeZone.getTimeZone(tzId.toString()); } + /** + * Translate a minutes offset into a ZoneId + * + * @param tzOffset as minutes east of UTC + * @return a ZoneId for this offset + * @since 7.1 + */ + public static ZoneId getZoneId(int tzOffset) { + return ZoneOffset.ofHoursMinutes(tzOffset / 60, tzOffset % 60); + } + /** * Format a timezone offset. * @@ -121,13 +134,17 @@ public class PersonIdent implements Serializable { } } + // Write offsets as [+-]HHMM + private static final DateTimeFormatter OFFSET_FORMATTER = DateTimeFormatter + .ofPattern("Z", Locale.US); //$NON-NLS-1$ + private final String name; private final String emailAddress; - private final long when; + private final Instant when; - private final int tzOffset; + private final ZoneId tzOffset; /** * Creates new PersonIdent from config info in repository, with current time. @@ -160,7 +177,7 @@ public class PersonIdent implements Serializable { * a {@link java.lang.String} object. */ public PersonIdent(String aName, String aEmailAddress) { - this(aName, aEmailAddress, SystemReader.getInstance().getCurrentTime()); + this(aName, aEmailAddress, SystemReader.getInstance().now()); } /** @@ -177,7 +194,7 @@ public class PersonIdent implements Serializable { */ public PersonIdent(String aName, String aEmailAddress, ProposedTimestamp when) { - this(aName, aEmailAddress, when.millis()); + this(aName, aEmailAddress, when.instant()); } /** @@ -189,8 +206,25 @@ public class PersonIdent implements Serializable { * local time * @param tz * time zone + * @deprecated Use {@link #PersonIdent(PersonIdent, Instant, ZoneId)} instead. */ + @Deprecated(since = "7.1") public PersonIdent(PersonIdent pi, Date when, TimeZone tz) { + this(pi.getName(), pi.getEmailAddress(), when.toInstant(), tz.toZoneId()); + } + + /** + * Copy a PersonIdent, but alter the clone's time stamp + * + * @param pi + * original {@link org.eclipse.jgit.lib.PersonIdent} + * @param when + * local time + * @param tz + * time zone offset + * @since 7.1 + */ + public PersonIdent(PersonIdent pi, Instant when, ZoneId tz) { this(pi.getName(), pi.getEmailAddress(), when, tz); } @@ -202,9 +236,11 @@ public class PersonIdent implements Serializable { * original {@link org.eclipse.jgit.lib.PersonIdent} * @param aWhen * local time + * @deprecated Use the variant with an Instant instead */ + @Deprecated(since = "7.1") public PersonIdent(PersonIdent pi, Date aWhen) { - this(pi.getName(), pi.getEmailAddress(), aWhen.getTime(), pi.tzOffset); + this(pi.getName(), pi.getEmailAddress(), aWhen.toInstant()); } /** @@ -218,7 +254,7 @@ public class PersonIdent implements Serializable { * @since 6.1 */ public PersonIdent(PersonIdent pi, Instant aWhen) { - this(pi.getName(), pi.getEmailAddress(), aWhen.toEpochMilli(), pi.tzOffset); + this(pi.getName(), pi.getEmailAddress(), aWhen, pi.tzOffset); } /** @@ -230,11 +266,12 @@ public class PersonIdent implements Serializable { * local time stamp * @param aTZ * time zone + * @deprecated Use the variant with Instant and ZoneId instead */ + @Deprecated(since = "7.1") public PersonIdent(final String aName, final String aEmailAddress, final Date aWhen, final TimeZone aTZ) { - this(aName, aEmailAddress, aWhen.getTime(), aTZ.getOffset(aWhen - .getTime()) / (60 * 1000)); + this(aName, aEmailAddress, aWhen.toInstant(), aTZ.toZoneId()); } /** @@ -252,10 +289,16 @@ public class PersonIdent implements Serializable { */ public PersonIdent(final String aName, String aEmailAddress, Instant aWhen, ZoneId zoneId) { - this(aName, aEmailAddress, aWhen.toEpochMilli(), - TimeZone.getTimeZone(zoneId) - .getOffset(aWhen - .toEpochMilli()) / (60 * 1000)); + if (aName == null) + throw new IllegalArgumentException( + JGitText.get().personIdentNameNonNull); + if (aEmailAddress == null) + throw new IllegalArgumentException( + JGitText.get().personIdentEmailNonNull); + name = aName; + emailAddress = aEmailAddress; + when = aWhen; + tzOffset = zoneId; } /** @@ -267,15 +310,18 @@ public class PersonIdent implements Serializable { * local time stamp * @param aTZ * time zone + * @deprecated Use the variant with Instant and ZoneId instead */ + @Deprecated(since = "7.1") public PersonIdent(PersonIdent pi, long aWhen, int aTZ) { - this(pi.getName(), pi.getEmailAddress(), aWhen, aTZ); + this(pi.getName(), pi.getEmailAddress(), Instant.ofEpochMilli(aWhen), + getZoneId(aTZ)); } private PersonIdent(final String aName, final String aEmailAddress, - long when) { + Instant when) { this(aName, aEmailAddress, when, SystemReader.getInstance() - .getTimezone(when)); + .getTimeZoneAt(when)); } private PersonIdent(UserConfig config) { @@ -298,19 +344,12 @@ public class PersonIdent implements Serializable { * local time stamp * @param aTZ * time zone + * @deprecated Use the variant with Instant and ZoneId instead */ + @Deprecated(since = "7.1") public PersonIdent(final String aName, final String aEmailAddress, final long aWhen, final int aTZ) { - if (aName == null) - throw new IllegalArgumentException( - JGitText.get().personIdentNameNonNull); - if (aEmailAddress == null) - throw new IllegalArgumentException( - JGitText.get().personIdentEmailNonNull); - name = aName; - emailAddress = aEmailAddress; - when = aWhen; - tzOffset = aTZ; + this(aName, aEmailAddress, Instant.ofEpochMilli(aWhen), getZoneId(aTZ)); } /** @@ -335,9 +374,12 @@ public class PersonIdent implements Serializable { * Get timestamp * * @return timestamp + * + * @deprecated Use getWhenAsInstant instead */ + @Deprecated(since = "7.1") public Date getWhen() { - return new Date(when); + return Date.from(when); } /** @@ -347,26 +389,39 @@ public class PersonIdent implements Serializable { * @since 6.1 */ public Instant getWhenAsInstant() { - return Instant.ofEpochMilli(when); + return when; } /** * Get this person's declared time zone * * @return this person's declared time zone; null if time zone is unknown. + * + * @deprecated Use getZoneId instead */ + @Deprecated(since = "7.1") public TimeZone getTimeZone() { - return getTimeZone(tzOffset); + return TimeZone.getTimeZone(tzOffset); } /** * Get the time zone id * * @return the time zone id - * @since 6.1 + * @since 7.1 */ public ZoneId getZoneId() { - return getTimeZone().toZoneId(); + return tzOffset; + } + + /** + * Return the offset in this timezone at the specific time + * + * @return the offset + * @since 7.1 + */ + public ZoneOffset getZoneOffset() { + return tzOffset.getRules().getOffset(when); } /** @@ -374,9 +429,11 @@ public class PersonIdent implements Serializable { * * @return this person's declared time zone as minutes east of UTC. If the * timezone is to the west of UTC it is negative. + * @deprecated Use {@link #getZoneOffset()} and read minutes from there */ + @Deprecated(since = "7.1") public int getTimeZoneOffset() { - return tzOffset; + return getZoneOffset().getTotalSeconds() / 60; } /** @@ -388,7 +445,7 @@ public class PersonIdent implements Serializable { public int hashCode() { int hc = getEmailAddress().hashCode(); hc *= 31; - hc += (int) (when / 1000L); + hc += when.hashCode(); return hc; } @@ -398,7 +455,9 @@ public class PersonIdent implements Serializable { final PersonIdent p = (PersonIdent) o; return getName().equals(p.getName()) && getEmailAddress().equals(p.getEmailAddress()) - && when / 1000L == p.when / 1000L; + // commmit timestamps are stored with 1 second precision + && when.truncatedTo(ChronoUnit.SECONDS) + .equals(p.when.truncatedTo(ChronoUnit.SECONDS)); } return false; } @@ -414,9 +473,9 @@ public class PersonIdent implements Serializable { r.append(" <"); //$NON-NLS-1$ appendSanitized(r, getEmailAddress()); r.append("> "); //$NON-NLS-1$ - r.append(when / 1000); + r.append(when.toEpochMilli() / 1000); r.append(' '); - appendTimezone(r, tzOffset); + r.append(OFFSET_FORMATTER.format(getZoneOffset())); return r.toString(); } @@ -424,18 +483,16 @@ public class PersonIdent implements Serializable { @SuppressWarnings("nls") public String toString() { final StringBuilder r = new StringBuilder(); - final SimpleDateFormat dtfmt; - dtfmt = new SimpleDateFormat("EEE MMM d HH:mm:ss yyyy Z", Locale.US); - dtfmt.setTimeZone(getTimeZone()); - + DateTimeFormatter dtfmt = DateTimeFormatter + .ofPattern("EEE MMM d HH:mm:ss yyyy Z", Locale.US) //$NON-NLS-1$ + .withZone(tzOffset); r.append("PersonIdent["); r.append(getName()); r.append(", "); r.append(getEmailAddress()); r.append(", "); - r.append(dtfmt.format(Long.valueOf(when))); + r.append(dtfmt.format(when)); r.append("]"); - return r.toString(); } } -- cgit v1.2.3 From 70c63a7e4e5f0373fd0abec0e5bf029747237b53 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Tue, 19 Nov 2024 13:12:53 +0100 Subject: Suppress non-externalized string warnings Change-Id: Ib0737f7ec6b9872f6d4514d140e7d32a4a40809d --- .../jgit/internal/storage/dfs/DfsBlockCacheConfig.java | 14 ++++++-------- .../jgit/internal/storage/dfs/DfsBlockCacheStats.java | 2 +- .../jgit/internal/storage/dfs/PackExtBlockCacheTable.java | 4 ++-- .../src/org/eclipse/jgit/merge/RecursiveMerger.java | 6 +++--- 4 files changed, 12 insertions(+), 14 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java index d63c208bbb..17bf51876d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheConfig.java @@ -51,7 +51,7 @@ public class DfsBlockCacheConfig { /** Default number of max cache hits. */ public static final int DEFAULT_CACHE_HOT_MAX = 1; - static final String DEFAULT_NAME = ""; + static final String DEFAULT_NAME = ""; //$NON-NLS-1$ private String name; @@ -91,7 +91,7 @@ public class DfsBlockCacheConfig { * {@link PrintWriter} to write the cache's configuration to. */ public void print(PrintWriter writer) { - print(/* linePrefix= */ "", /* pad= */ " ", writer); + print(/* linePrefix= */ "", /* pad= */ " ", writer); //$NON-NLS-1$//$NON-NLS-2$ } /** @@ -105,14 +105,12 @@ public class DfsBlockCacheConfig { * @param writer * {@link PrintWriter} to write the cache's configuration to. */ + @SuppressWarnings("nls") private void print(String linePrefix, String pad, PrintWriter writer) { String currentPrefixLevel = linePrefix; if (!name.isEmpty() || !packExtCacheConfigurations.isEmpty()) { - String name = this.name; - if (name.isEmpty()) { - name = ""; - } - writer.println(linePrefix + "Name: " + name); + writer.println(linePrefix + "Name: " + + (name.isEmpty() ? DEFAULT_NAME : this.name)); currentPrefixLevel += pad; } writer.println(currentPrefixLevel + "BlockLimit: " + blockLimit); @@ -602,7 +600,7 @@ public class DfsBlockCacheConfig { void print(String linePrefix, String pad, PrintWriter writer) { packExtCacheConfiguration.print(linePrefix, pad, writer); - writer.println(linePrefix + pad + "PackExts: " + writer.println(linePrefix + pad + "PackExts: " //$NON-NLS-1$ + packExts.stream().sorted().collect(Collectors.toList())); } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheStats.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheStats.java index 518db8c82d..436f57437e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheStats.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsBlockCacheStats.java @@ -46,7 +46,7 @@ class DfsBlockCacheStats implements BlockCacheStats { private final AtomicReference liveBytes; DfsBlockCacheStats() { - this(""); + this(""); //$NON-NLS-1$ } DfsBlockCacheStats(String name) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/PackExtBlockCacheTable.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/PackExtBlockCacheTable.java index 28b021c68f..bb44f9397a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/PackExtBlockCacheTable.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/PackExtBlockCacheTable.java @@ -125,10 +125,10 @@ class PackExtBlockCacheTable implements DfsBlockCacheTable { Set blockCacheTables = new HashSet<>(); blockCacheTables.add(defaultBlockCacheTable); blockCacheTables.addAll(packExtBlockCacheTables.values()); - String name = defaultBlockCacheTable.getName() + "," + String name = defaultBlockCacheTable.getName() + "," //$NON-NLS-1$ + packExtBlockCacheTables.values().stream() .map(DfsBlockCacheTable::getName).sorted() - .collect(Collectors.joining(",")); + .collect(Collectors.joining(",")); //$NON-NLS-1$ return new PackExtBlockCacheTable(name, defaultBlockCacheTable, List.copyOf(blockCacheTables), packExtBlockCacheTables); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/merge/RecursiveMerger.java b/org.eclipse.jgit/src/org/eclipse/jgit/merge/RecursiveMerger.java index 24614f7d18..ec1b599b77 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/merge/RecursiveMerger.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/merge/RecursiveMerger.java @@ -244,11 +244,11 @@ public class RecursiveMerger extends ResolveMerger { private String failingPathsMessage() { int max = 25; String failedPaths = failingPaths.entrySet().stream().limit(max) - .map(entry -> entry.getKey() + ":" + entry.getValue()) - .collect(Collectors.joining("\n")); + .map(entry -> entry.getKey() + ":" + entry.getValue()) //$NON-NLS-1$ + .collect(Collectors.joining("\n")); //$NON-NLS-1$ if (failingPaths.size() > max) { - failedPaths = String.format("%s\n... (%s failing paths omitted)", + failedPaths = String.format("%s\n... (%s failing paths omitted)", //$NON-NLS-1$ failedPaths, failingPaths.size() - max); } return failedPaths; -- cgit v1.2.3 From 76efd28bc91caa914eb4c16656c2928f92400547 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Tue, 19 Nov 2024 13:19:03 +0100 Subject: RecursiveMerger: fix boxing warning Change-Id: I6f6ae540b128ff6b965931e829da1368a8b88ddb --- org.eclipse.jgit/src/org/eclipse/jgit/merge/RecursiveMerger.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/merge/RecursiveMerger.java b/org.eclipse.jgit/src/org/eclipse/jgit/merge/RecursiveMerger.java index ec1b599b77..fc5ab62898 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/merge/RecursiveMerger.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/merge/RecursiveMerger.java @@ -249,7 +249,7 @@ public class RecursiveMerger extends ResolveMerger { if (failingPaths.size() > max) { failedPaths = String.format("%s\n... (%s failing paths omitted)", //$NON-NLS-1$ - failedPaths, failingPaths.size() - max); + failedPaths, Integer.valueOf(failingPaths.size() - max)); } return failedPaths; } -- cgit v1.2.3 From 4b3c5194acd8a9b18bf269888cab2ea29c376508 Mon Sep 17 00:00:00 2001 From: Kaushik Lingarkar Date: Tue, 19 Nov 2024 09:37:33 -0800 Subject: Support built-in diff drivers for hunk header function names The regexes defined for each built-in driver will be used to determine the function name for a hunk header. Each driver can specify a list of regexes to negate and match. They can also define pattern compilation flags if needed. These drivers only apply to text files with unified patch type. Following built-in drivers have been added: - cpp - dts - java - python - rust Support for more languages can be added as needed to match the cgit implementation. Change-Id: Ice5430bfed7e4aaf9f00e52e44402479984953c5 --- .../org/eclipse/jgit/test/resources/greeting.c | 43 +++++ .../jgit/test/resources/greeting.javasource | 37 +++++ .../org/eclipse/jgit/test/resources/greeting.py | 26 ++++ .../org/eclipse/jgit/test/resources/greeting.rs | 27 ++++ .../org/eclipse/jgit/test/resources/sample.dtsi | 25 +++ .../jgit/diff/DiffFormatterBuiltInDriverTest.java | 173 +++++++++++++++++++++ org.eclipse.jgit/.settings/.api_filters | 28 ++++ .../src/org/eclipse/jgit/diff/DiffDriver.java | 116 ++++++++++++++ .../src/org/eclipse/jgit/diff/DiffFormatter.java | 140 +++++++++++++++-- .../eclipse/jgit/diff/PatchIdDiffFormatter.java | 4 +- 10 files changed, 606 insertions(+), 13 deletions(-) create mode 100644 org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/test/resources/greeting.c create mode 100644 org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/test/resources/greeting.javasource create mode 100644 org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/test/resources/greeting.py create mode 100644 org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/test/resources/greeting.rs create mode 100644 org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/test/resources/sample.dtsi create mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/diff/DiffFormatterBuiltInDriverTest.java create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffDriver.java (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/test/resources/greeting.c b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/test/resources/greeting.c new file mode 100644 index 0000000000..3661160921 --- /dev/null +++ b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/test/resources/greeting.c @@ -0,0 +1,43 @@ +#include +#include +#include + +void getGreeting(char *result, const char *name) { + sprintf(result, "Hello, %s!", name); +} + +void getFarewell(char *result, const char *name) { + sprintf(result, "Goodbye, %s. Have a great day!", name); +} + +void toLower(char *str) { + for (int i = 0; str[i]; i++) { + str[i] = tolower(str[i]); + } +} + +void getPersonalizedGreeting(char *result, const char *name, const char *timeOfDay) { + char timeOfDayLower[50]; + strcpy(timeOfDayLower, timeOfDay); + toLower(timeOfDayLower); + if (strcmp(timeOfDayLower, "morning") == 0) { + sprintf(result, "Good morning, %s", name); + } else if (strcmp(timeOfDayLower, "afternoon") == 0) { + sprintf(result, "Good afternoon, %s", name); + } else if (strcmp(timeOfDayLower, "evening") == 0) { + sprintf(result, "Good evening, %s", name); + } else { + sprintf(result, "Good day, %s", name); + } +} + +int main() { + char result[100]; + getGreeting(result, "foo"); + printf("%s\\n", result); + getFarewell(result, "bar"); + printf("%s\\n", result); + getPersonalizedGreeting(result, "baz", "morning"); + printf("%s\\n", result); + return 0; +} diff --git a/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/test/resources/greeting.javasource b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/test/resources/greeting.javasource new file mode 100644 index 0000000000..9659685c63 --- /dev/null +++ b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/test/resources/greeting.javasource @@ -0,0 +1,37 @@ +public class Greeting { + public String getGreeting(String name) { + String msg = "Hello, " + name + "!"; + return msg; + } + + public String getFarewell(String name) { + String msg = "Goodbye, " + name + ". Have a great day!"; + return msg; + } + + public String getPersonalizedGreeting(String name, String timeOfDay) { + String msg; + switch (timeOfDay.toLowerCase()) { + case "morning": + msg = "Good morning, " + name; + break; + case "afternoon": + msg = "Good afternoon, " + name; + break; + case "evening": + msg = "Good evening, " + name; + break; + default: + msg = "Good day, " + name; + break; + } + return msg; + } + + public static void main(String[] args) { + Greeting greeting = new Greeting(); + System.out.println(greeting.getGreeting("foo")); + System.out.println(greeting.getFarewell("bar")); + System.out.println(greeting.getPersonalizedGreeting("baz", "morning")); + } +} diff --git a/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/test/resources/greeting.py b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/test/resources/greeting.py new file mode 100644 index 0000000000..9eda6cd8fe --- /dev/null +++ b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/test/resources/greeting.py @@ -0,0 +1,26 @@ +class Greeting: + def get_greeting(self, name): + greeting_message = f"Hello, {name}!" + return greeting_message + + def get_farewell(self, name): + farewell_message = f"Goodbye, {name}. Have a great day!" + return farewell_message + + def get_personalized_greeting(self, name, time_of_day): + time_of_day = time_of_day.lower() + if time_of_day == "morning": + personalized_message = f"Good morning, {name}" + elif time_of_day == "afternoon": + personalized_message = f"Good afternoon, {name}" + elif time_of_day == "evening": + personalized_message = f"Good evening, {name}" + else: + personalized_message = f"Good day, {name}" + return personalized_message + +if __name__ == "__main__": + greeting = Greeting() + print(greeting.get_greeting("foo")) + print(greeting.get_farewell("bar")) + print(greeting.get_personalized_greeting("baz", "morning")) diff --git a/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/test/resources/greeting.rs b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/test/resources/greeting.rs new file mode 100644 index 0000000000..a3aa5cbe7c --- /dev/null +++ b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/test/resources/greeting.rs @@ -0,0 +1,27 @@ +struct Greeting; + +impl Greeting { + fn get_greeting(&self, name: &str) -> String { + format!("Hello, {}!", name) + } + + fn get_farewell(&self, name: &str) -> String { + format!("Goodbye, {}. Have a great day!", name) + } + + fn get_personalized_greeting(&self, name: &str, time_of_day: &str) -> String { + match time_of_day.to_lowercase().as_str() { + "morning" => format!("Good morning, {}", name), + "afternoon" => format!("Good afternoon, {}", name), + "evening" => format!("Good evening, {}", name), + _ => format!("Good day, {}", name), + } + } +} + +fn main() { + let greeting = Greeting; + println!("{}", greeting.get_greeting("foo")); + println!("{}", greeting.get_farewell("bar")); + println!("{}", greeting.get_personalized_greeting("baz", "morning")); +} diff --git a/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/test/resources/sample.dtsi b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/test/resources/sample.dtsi new file mode 100644 index 0000000000..6aa4ecdd4c --- /dev/null +++ b/org.eclipse.jgit.test/tst-rsrc/org/eclipse/jgit/test/resources/sample.dtsi @@ -0,0 +1,25 @@ +/dts-v1/; + +/ { + model = "Example Board"; + compatible = "example,board"; + cpus { + cpu@0 { + device_type = "cpu"; + compatible = "arm,cortex-a9"; + reg = <0>; + }; + }; + + memory { + device_type = "memory"; + reg = <0x80000000 0x20000000>; + }; + + uart0: uart@101f1000 { + compatible = "ns16550a"; + reg = <0x101f1000 0x1000>; + interrupts = <5>; + clock-frequency = <24000000>; + }; +}; diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/diff/DiffFormatterBuiltInDriverTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/diff/DiffFormatterBuiltInDriverTest.java new file mode 100644 index 0000000000..1352871983 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/diff/DiffFormatterBuiltInDriverTest.java @@ -0,0 +1,173 @@ +/* + * Copyright (c) 2024 Qualcomm Innovation Center, Inc. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.diff; + +import static org.junit.Assert.assertEquals; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.stream.Collectors; +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.junit.JGitTestUtil; +import org.eclipse.jgit.junit.RepositoryTestCase; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.treewalk.CanonicalTreeParser; +import org.junit.Test; + +public class DiffFormatterBuiltInDriverTest extends RepositoryTestCase { + @Test + public void testCppDriver() throws Exception { + String fileName = "greeting.c"; + String body = Files.readString( + Path.of(JGitTestUtil.getTestResourceFile(fileName) + .getAbsolutePath())); + RevCommit c1; + RevCommit c2; + try (Git git = new Git(db)) { + createCommit(git, ".gitattributes", "*.c diff=cpp"); + c1 = createCommit(git, fileName, body); + c2 = createCommit(git, fileName, + body.replace("Good day", "Greetings") + .replace("baz", "qux")); + } + try (ByteArrayOutputStream os = new ByteArrayOutputStream(); + DiffFormatter diffFormatter = new DiffFormatter(os)) { + String actual = getHunkHeaders(c1, c2, os, diffFormatter); + String expected = + "@@ -27,7 +27,7 @@ void getPersonalizedGreeting(char *result, const char *name, const char *timeOfD\n" + + "@@ -37,7 +37,7 @@ int main() {"; + assertEquals(expected, actual); + } + } + + @Test + public void testDtsDriver() throws Exception { + String fileName = "sample.dtsi"; + String body = Files.readString( + Path.of(JGitTestUtil.getTestResourceFile(fileName) + .getAbsolutePath())); + RevCommit c1; + RevCommit c2; + try (Git git = new Git(db)) { + createCommit(git, ".gitattributes", "*.dtsi diff=dts"); + c1 = createCommit(git, fileName, body); + c2 = createCommit(git, fileName, + body.replace("clock-frequency = <24000000>", + "clock-frequency = <48000000>")); + } + try (ByteArrayOutputStream os = new ByteArrayOutputStream(); + DiffFormatter diffFormatter = new DiffFormatter(os)) { + String actual = getHunkHeaders(c1, c2, os, diffFormatter); + String expected = "@@ -20,6 +20,6 @@ uart0: uart@101f1000 {"; + assertEquals(expected, actual); + } + } + + @Test + public void testJavaDriver() throws Exception { + String resourceName = "greeting.javasource"; + String body = Files.readString( + Path.of(JGitTestUtil.getTestResourceFile(resourceName) + .getAbsolutePath())); + RevCommit c1; + RevCommit c2; + try (Git git = new Git(db)) { + createCommit(git, ".gitattributes", "*.java diff=java"); + String fileName = "Greeting.java"; + c1 = createCommit(git, fileName, body); + c2 = createCommit(git, fileName, + body.replace("Good day", "Greetings") + .replace("baz", "qux")); + } + try (ByteArrayOutputStream os = new ByteArrayOutputStream(); + DiffFormatter diffFormatter = new DiffFormatter(os)) { + String actual = getHunkHeaders(c1, c2, os, diffFormatter); + String expected = + "@@ -22,7 +22,7 @@ public String getPersonalizedGreeting(String name, String timeOfDay) {\n" + + "@@ -32,6 +32,6 @@ public static void main(String[] args) {"; + assertEquals(expected, actual); + } + } + + @Test + public void testPythonDriver() throws Exception { + String fileName = "greeting.py"; + String body = Files.readString( + Path.of(JGitTestUtil.getTestResourceFile(fileName) + .getAbsolutePath())); + RevCommit c1; + RevCommit c2; + try (Git git = new Git(db)) { + createCommit(git, ".gitattributes", "*.py diff=python"); + c1 = createCommit(git, fileName, body); + c2 = createCommit(git, fileName, + body.replace("Good day", "Greetings")); + } + try (ByteArrayOutputStream os = new ByteArrayOutputStream(); + DiffFormatter diffFormatter = new DiffFormatter(os)) { + String actual = getHunkHeaders(c1, c2, os, diffFormatter); + String expected = "@@ -16,7 +16,7 @@ def get_personalized_greeting(self, name, time_of_day):"; + assertEquals(expected, actual); + } + } + + @Test + public void testRustDriver() throws Exception { + String fileName = "greeting.rs"; + String body = Files.readString( + Path.of(JGitTestUtil.getTestResourceFile(fileName) + .getAbsolutePath())); + RevCommit c1; + RevCommit c2; + try (Git git = new Git(db)) { + createCommit(git, ".gitattributes", "*.rs diff=rust"); + c1 = createCommit(git, fileName, body); + c2 = createCommit(git, fileName, + body.replace("Good day", "Greetings") + .replace("baz", "qux")); + } + try (ByteArrayOutputStream os = new ByteArrayOutputStream(); + DiffFormatter diffFormatter = new DiffFormatter(os)) { + String actual = getHunkHeaders(c1, c2, os, diffFormatter); + String expected = + "@@ -14,7 +14,7 @@ fn get_personalized_greeting(&self, name: &str, time_of_day: &str) -> String {\n" + + "@@ -23,5 +23,5 @@ fn main() {"; + assertEquals(expected, actual); + } + } + + private String getHunkHeaders(RevCommit c1, RevCommit c2, + ByteArrayOutputStream os, DiffFormatter diffFormatter) + throws IOException { + diffFormatter.setRepository(db); + diffFormatter.format(new CanonicalTreeParser(null, db.newObjectReader(), + c1.getTree()), + new CanonicalTreeParser(null, db.newObjectReader(), + c2.getTree())); + diffFormatter.flush(); + return Arrays.stream(os.toString(StandardCharsets.UTF_8).split("\n")) + .filter(line -> line.startsWith("@@")) + .collect(Collectors.joining("\n")); + } + + private RevCommit createCommit(Git git, String fileName, String body) + throws IOException, GitAPIException { + writeTrashFile(fileName, body); + git.add().addFilepattern(".").call(); + return git.commit().setMessage("message").call(); + } +} diff --git a/org.eclipse.jgit/.settings/.api_filters b/org.eclipse.jgit/.settings/.api_filters index 2c22f0231c..d3ae4cf376 100644 --- a/org.eclipse.jgit/.settings/.api_filters +++ b/org.eclipse.jgit/.settings/.api_filters @@ -1,5 +1,33 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffDriver.java b/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffDriver.java new file mode 100644 index 0000000000..9ae1aaa298 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffDriver.java @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2024 Qualcomm Innovation Center, Inc. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.diff; + +import java.util.List; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +/** + * Built-in drivers for various languages, sorted by name. These drivers will be + * used to determine function names for a hunk. + *

+ * When writing or updating patterns, assume the contents are syntactically + * correct. Patterns can be simple and need not cover all syntactical corner + * cases, as long as they are sufficiently permissive. + * + * @since 6.10.1 + */ +@SuppressWarnings({"ImmutableEnumChecker", "nls"}) +public enum DiffDriver { + /** + * Built-in diff driver for c++ + */ + cpp(List.of( + /* Jump targets or access declarations */ + "^[ \\t]*[A-Za-z_][A-Za-z_0-9]*:\\s*($|/[/*])"), List.of( + /* functions/methods, variables, and compounds at top level */ + "^((::\\s*)?[A-Za-z_].*)$")), + /** + * Built-in diff driver for device + * tree files + */ + dts(List.of(";", "="), List.of( + /* lines beginning with a word optionally preceded by '&' or the root */ + "^[ \\t]*((/[ \\t]*\\{|&?[a-zA-Z_]).*)")), + /** + * Built-in diff driver for java + */ + java(List.of( + "^[ \\t]*(catch|do|for|if|instanceof|new|return|switch|throw|while)"), + List.of( + /* Class, enum, interface, and record declarations */ + "^[ \\t]*(([a-z-]+[ \\t]+)*(class|enum|interface|record)[ \\t]+.*)$", + /* Method definitions; note that constructor signatures are not */ + /* matched because they are indistinguishable from method calls. */ + "^[ \\t]*(([A-Za-z_<>&\\]\\[][?&<>.,A-Za-z_0-9]*[ \\t]+)+[A-Za-z_]" + + "[A-Za-z_0-9]*[ \\t]*\\([^;]*)$")), + /** + * Built-in diff driver for python + */ + python(List.of("^[ \\t]*((class|(async[ \\t]+)?def)[ \\t].*)$")), + /** + * Built-in diff driver for java + */ + rust(List.of("^[\\t ]*((pub(\\([^\\)]+\\))?[\\t ]+)?" + + "((async|const|unsafe|extern([\\t ]+\"[^\"]+\"))[\\t ]+)?" + + "(struct|enum|union|mod|trait|fn|impl|macro_rules!)[< \\t]+[^;]*)$")); + + private final List negatePatterns; + + private final List matchPatterns; + + DiffDriver(List negate, List match, int flags) { + if (negate != null) { + this.negatePatterns = negate.stream() + .map(r -> Pattern.compile(r, flags)) + .collect(Collectors.toList()); + } else { + this.negatePatterns = null; + } + this.matchPatterns = match.stream().map(r -> Pattern.compile(r, flags)) + .collect(Collectors.toList()); + } + + DiffDriver(List match) { + this(null, match, 0); + } + + DiffDriver(List negate, List match) { + this(negate, match, 0); + } + + /** + * Returns the list of patterns used to exclude certain lines from being + * considered as function names. + * + * @return the list of negate patterns + */ + public List getNegatePatterns() { + return negatePatterns; + } + + /** + * Returns the list of patterns used to match lines for potential function + * names. + * + * @return the list of match patterns + */ + public List getMatchPatterns() { + return matchPatterns; + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffFormatter.java b/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffFormatter.java index 2f472b5c0a..fa446e18cd 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffFormatter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffFormatter.java @@ -30,7 +30,9 @@ import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.regex.Pattern; import org.eclipse.jgit.api.errors.CanceledException; +import org.eclipse.jgit.attributes.Attribute; import org.eclipse.jgit.diff.DiffAlgorithm.SupportedAlgorithm; import org.eclipse.jgit.diff.DiffEntry.ChangeType; import org.eclipse.jgit.dircache.DirCacheIterator; @@ -703,7 +705,7 @@ public class DiffFormatter implements AutoCloseable { */ public void format(DiffEntry ent) throws IOException { FormatResult res = createFormatResult(ent); - format(res.header, res.a, res.b); + format(res.header, res.a, res.b, getDiffDriver(ent)); } private static byte[] writeGitLinkText(AbbreviatedObjectId id) { @@ -749,11 +751,14 @@ public class DiffFormatter implements AutoCloseable { * text source for the post-image version of the content. This * must match the content of * {@link org.eclipse.jgit.patch.FileHeader#getNewId()}. + * @param diffDriver + * the diff driver used to obtain function names in hunk headers * @throws java.io.IOException - * writing to the supplied stream failed. + * writing to the supplied stream failed. + * @since 6.10.1 */ - public void format(FileHeader head, RawText a, RawText b) - throws IOException { + public void format(FileHeader head, RawText a, RawText b, + DiffDriver diffDriver) throws IOException { // Reuse the existing FileHeader as-is by blindly copying its // header lines, but avoiding its hunks. Instead we recreate // the hunks from the text instances we have been supplied. @@ -763,8 +768,49 @@ public class DiffFormatter implements AutoCloseable { if (!head.getHunks().isEmpty()) end = head.getHunks().get(0).getStartOffset(); out.write(head.getBuffer(), start, end - start); - if (head.getPatchType() == PatchType.UNIFIED) - format(head.toEditList(), a, b); + if (head.getPatchType() == PatchType.UNIFIED) { + format(head.toEditList(), a, b, diffDriver); + } + } + + /** + * Format a patch script, reusing a previously parsed FileHeader. + *

+ * This formatter is primarily useful for editing an existing patch script + * to increase or reduce the number of lines of context within the script. + * All header lines are reused as-is from the supplied FileHeader. + * + * @param head + * existing file header containing the header lines to copy. + * @param a + * text source for the pre-image version of the content. This must match + * the content of {@link org.eclipse.jgit.patch.FileHeader#getOldId()}. + * @param b + * text source for the post-image version of the content. This must match + * the content of {@link org.eclipse.jgit.patch.FileHeader#getNewId()}. + * @throws java.io.IOException + * writing to the supplied stream failed. + */ + public void format(FileHeader head, RawText a, RawText b) + throws IOException { + format(head, a, b, null); + } + + /** + * Formats a list of edits in unified diff format + * + * @param edits + * some differences which have been calculated between A and B + * @param a + * the text A which was compared + * @param b + * the text B which was compared + * @throws java.io.IOException + * if an IO error occurred + */ + public void format(EditList edits, RawText a, RawText b) + throws IOException { + format(edits, a, b, null); } /** @@ -776,11 +822,14 @@ public class DiffFormatter implements AutoCloseable { * the text A which was compared * @param b * the text B which was compared + * @param diffDriver + * the diff driver used to obtain function names in hunk headers * @throws java.io.IOException * if an IO error occurred + * @since 6.10.1 */ - public void format(EditList edits, RawText a, RawText b) - throws IOException { + public void format(EditList edits, RawText a, RawText b, + DiffDriver diffDriver) throws IOException { for (int curIdx = 0; curIdx < edits.size();) { Edit curEdit = edits.get(curIdx); final int endIdx = findCombinedEnd(edits, curIdx); @@ -791,7 +840,8 @@ public class DiffFormatter implements AutoCloseable { final int aEnd = (int) Math.min(a.size(), (long) endEdit.getEndA() + context); final int bEnd = (int) Math.min(b.size(), (long) endEdit.getEndB() + context); - writeHunkHeader(aCur, aEnd, bCur, bEnd); + writeHunkHeader(aCur, aEnd, bCur, bEnd, + getFuncName(a, aCur - 1, diffDriver)); while (aCur < aEnd || bCur < bEnd) { if (aCur < curEdit.getBeginA() || endIdx + 1 < curIdx) { @@ -881,8 +931,30 @@ public class DiffFormatter implements AutoCloseable { * @throws java.io.IOException * if an IO error occurred */ - protected void writeHunkHeader(int aStartLine, int aEndLine, - int bStartLine, int bEndLine) throws IOException { + protected void writeHunkHeader(int aStartLine, int aEndLine, int bStartLine, + int bEndLine) throws IOException { + writeHunkHeader(aStartLine, aEndLine, bStartLine, bEndLine, null); + } + + /** + * Output a hunk header + * + * @param aStartLine + * within first source + * @param aEndLine + * within first source + * @param bStartLine + * within second source + * @param bEndLine + * within second source + * @param funcName + * function name of this hunk + * @throws java.io.IOException + * if an IO error occurred + * @since 6.10.1 + */ + protected void writeHunkHeader(int aStartLine, int aEndLine, int bStartLine, + int bEndLine, String funcName) throws IOException { out.write('@'); out.write('@'); writeRange('-', aStartLine + 1, aEndLine - aStartLine); @@ -890,6 +962,10 @@ public class DiffFormatter implements AutoCloseable { out.write(' '); out.write('@'); out.write('@'); + if (funcName != null) { + out.write(' '); + out.write(funcName.getBytes()); + } out.write('\n'); } @@ -1247,4 +1323,46 @@ public class DiffFormatter implements AutoCloseable { private static boolean end(Edit edit, int a, int b) { return edit.getEndA() <= a && edit.getEndB() <= b; } + + private String getFuncName(RawText text, int startAt, + DiffDriver diffDriver) { + if (diffDriver != null) { + while (startAt > 0) { + String line = text.getString(startAt); + startAt--; + if (matchesAny(diffDriver.getNegatePatterns(), line)) { + continue; + } + if (matchesAny(diffDriver.getMatchPatterns(), line)) { + String funcName = line.replaceAll("^[ \\t]+", ""); + return funcName.substring(0, + Math.min(funcName.length(), 80)).trim(); + } + } + } + return null; + } + + private boolean matchesAny(List patterns, String text) { + if (patterns != null) { + for (Pattern p : patterns) { + if (p.matcher(text).find()) { + return true; + } + } + } + return false; + } + + private DiffDriver getDiffDriver(DiffEntry entry) { + Attribute diffAttr = entry.getDiffAttribute(); + if (diffAttr != null) { + try { + return DiffDriver.valueOf(diffAttr.getValue()); + } catch (IllegalArgumentException e) { + return null; + } + } + return null; + } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/diff/PatchIdDiffFormatter.java b/org.eclipse.jgit/src/org/eclipse/jgit/diff/PatchIdDiffFormatter.java index 4343642f9a..b401bbe73d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/diff/PatchIdDiffFormatter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/diff/PatchIdDiffFormatter.java @@ -44,8 +44,8 @@ public class PatchIdDiffFormatter extends DiffFormatter { } @Override - protected void writeHunkHeader(int aStartLine, int aEndLine, - int bStartLine, int bEndLine) throws IOException { + protected void writeHunkHeader(int aStartLine, int aEndLine, int bStartLine, + int bEndLine, String funcName) throws IOException { // The hunk header is not taken into account for patch id calculation } -- cgit v1.2.3 From 469928898db10d4b19bd34658d3b2f93f12952b3 Mon Sep 17 00:00:00 2001 From: Jacek Centkowski Date: Thu, 31 Oct 2024 18:30:02 +0100 Subject: Add numberOfObjectsSinceBitmap to RepoStatistics Introduce a numberOfObjectsSinceBitmap that contains the number of objects stored in pack files and as loose objects created since the latest bitmap generation. Note that the existing GcNumberOfPackFilesAfterBitmapStatisticsTest.java was renamed to GcSinceBitmapStatisticsTest.java and extended to cover also this statistic. Change-Id: I8ae1db142ddfcd42a5a1d6da01bc67f695562e0e --- ...NumberOfPackFilesSinceBitmapStatisticsTest.java | 97 ------------ .../storage/file/GcSinceBitmapStatisticsTest.java | 167 +++++++++++++++++++++ .../org/eclipse/jgit/internal/storage/file/GC.java | 18 ++- 3 files changed, 184 insertions(+), 98 deletions(-) delete mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcNumberOfPackFilesSinceBitmapStatisticsTest.java create mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcSinceBitmapStatisticsTest.java (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcNumberOfPackFilesSinceBitmapStatisticsTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcNumberOfPackFilesSinceBitmapStatisticsTest.java deleted file mode 100644 index fa13bb57c8..0000000000 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcNumberOfPackFilesSinceBitmapStatisticsTest.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright (c) 2024 Jacek Centkowski and others. - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Distribution License v. 1.0 which is available at - * https://www.eclipse.org/org/documents/edl-v10.php. - * - * SPDX-License-Identifier: BSD-3-Clause - */ - -package org.eclipse.jgit.internal.storage.file; - -import static org.junit.Assert.assertEquals; - -import java.io.IOException; -import java.nio.file.Files; -import java.util.stream.StreamSupport; - -import org.eclipse.jgit.lib.PersonIdent; -import org.eclipse.jgit.revwalk.RevCommit; -import org.junit.Test; - -public class GcNumberOfPackFilesSinceBitmapStatisticsTest extends GcTestCase { - @Test - public void testShouldReportZeroObjectsForInitializedRepo() - throws IOException { - assertEquals(0L, gc.getStatistics().numberOfPackFilesSinceBitmap); - } - - @Test - public void testShouldReportAllPackFilesWhenNoGcWasPerformed() - throws Exception { - tr.packAndPrune(); - long result = gc.getStatistics().numberOfPackFilesSinceBitmap; - - assertEquals(repo.getObjectDatabase().getPacks().size(), result); - } - - @Test - public void testShouldReportNoObjectsDirectlyAfterGc() throws Exception { - // given - addCommit(null); - gc.gc().get(); - assertEquals(1L, repositoryBitmapFiles()); - assertEquals(0L, gc.getStatistics().numberOfPackFilesSinceBitmap); - } - - @Test - public void testShouldReportNewObjectsSinceGcWhenRepositoryProgresses() - throws Exception { - // commit & gc - RevCommit parent = addCommit(null); - gc.gc().get(); - assertEquals(1L, repositoryBitmapFiles()); - - // progress & pack - addCommit(parent); - tr.packAndPrune(); - - assertEquals(1L, gc.getStatistics().numberOfPackFilesSinceBitmap); - } - - @Test - public void testShouldReportNewObjectsFromTheLatestBitmapWhenRepositoryProgresses() - throws Exception { - // commit & gc - RevCommit parent = addCommit(null); - gc.gc().get(); - assertEquals(1L, repositoryBitmapFiles()); - - // progress & gc - parent = addCommit(parent); - gc.gc().get(); - assertEquals(2L, repositoryBitmapFiles()); - - // progress & pack - addCommit(parent); - tr.packAndPrune(); - - assertEquals(1L, gc.getStatistics().numberOfPackFilesSinceBitmap); - } - - private RevCommit addCommit(RevCommit parent) throws Exception { - return tr.branch("master").commit() - .author(new PersonIdent("repo-metrics", "repo@metrics.com")) - .parent(parent).create(); - } - - private long repositoryBitmapFiles() throws IOException { - return StreamSupport - .stream(Files - .newDirectoryStream(repo.getObjectDatabase() - .getPackDirectory().toPath(), "pack-*.bitmap") - .spliterator(), false) - .count(); - } -} diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcSinceBitmapStatisticsTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcSinceBitmapStatisticsTest.java new file mode 100644 index 0000000000..3cd766c4e9 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcSinceBitmapStatisticsTest.java @@ -0,0 +1,167 @@ +/* + * Copyright (c) 2024 Jacek Centkowski and others. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.storage.file; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.nio.file.Files; +import java.util.Collection; +import java.util.stream.StreamSupport; + +import org.eclipse.jgit.internal.storage.file.GC.RepoStatistics; +import org.eclipse.jgit.lib.PersonIdent; +import org.eclipse.jgit.revwalk.RevCommit; +import org.junit.Test; + +public class GcSinceBitmapStatisticsTest extends GcTestCase { + @Test + public void testShouldReportZeroPacksAndObjectsForInitializedRepo() + throws IOException { + RepoStatistics s = gc.getStatistics(); + assertEquals(0L, s.numberOfPackFilesSinceBitmap); + assertEquals(0L, s.numberOfObjectsSinceBitmap); + } + + @Test + public void testShouldReportAllPackFilesWhenNoGcWasPerformed() + throws Exception { + tr.packAndPrune(); + long result = gc.getStatistics().numberOfPackFilesSinceBitmap; + + assertEquals(repo.getObjectDatabase().getPacks().size(), result); + } + + @Test + public void testShouldReportAllObjectsWhenNoGcWasPerformed() + throws Exception { + tr.packAndPrune(); + + assertEquals( + getNumberOfObjectsInPacks(repo.getObjectDatabase().getPacks()), + gc.getStatistics().numberOfObjectsSinceBitmap); + } + + @Test + public void testShouldReportNoPacksDirectlyAfterGc() throws Exception { + // given + addCommit(null); + gc.gc().get(); + assertEquals(1L, repositoryBitmapFiles()); + assertEquals(0L, gc.getStatistics().numberOfPackFilesSinceBitmap); + } + + @Test + public void testShouldReportNoObjectsDirectlyAfterGc() throws Exception { + // given + addCommit(null); + assertEquals(2L, gc.getStatistics().numberOfObjectsSinceBitmap); + + gc.gc().get(); + assertEquals(0L, gc.getStatistics().numberOfObjectsSinceBitmap); + } + + @Test + public void testShouldReportNewPacksSinceGcWhenRepositoryProgresses() + throws Exception { + // commit & gc + RevCommit parent = addCommit(null); + gc.gc().get(); + assertEquals(1L, repositoryBitmapFiles()); + + // progress & pack + addCommit(parent); + tr.packAndPrune(); + + assertEquals(1L, gc.getStatistics().numberOfPackFilesSinceBitmap); + } + + @Test + public void testShouldReportNewObjectsSinceGcWhenRepositoryProgresses() + throws Exception { + // commit & gc + RevCommit parent = addCommit(null); + gc.gc().get(); + assertEquals(0L, gc.getStatistics().numberOfObjectsSinceBitmap); + + // progress & pack + addCommit(parent); + assertEquals(1L, gc.getStatistics().numberOfObjectsSinceBitmap); + + tr.packAndPrune(); + assertEquals(3L, gc.getStatistics().numberOfObjectsSinceBitmap); + } + + @Test + public void testShouldReportNewPacksFromTheLatestBitmapWhenRepositoryProgresses() + throws Exception { + // commit & gc + RevCommit parent = addCommit(null); + gc.gc().get(); + assertEquals(1L, repositoryBitmapFiles()); + + // progress & gc + parent = addCommit(parent); + gc.gc().get(); + assertEquals(2L, repositoryBitmapFiles()); + + // progress & pack + addCommit(parent); + tr.packAndPrune(); + + assertEquals(1L, gc.getStatistics().numberOfPackFilesSinceBitmap); + } + + @Test + public void testShouldReportNewObjectsFromTheLatestBitmapWhenRepositoryProgresses() + throws Exception { + // commit & gc + RevCommit parent = addCommit(null); + gc.gc().get(); + + // progress & gc + parent = addCommit(parent); + gc.gc().get(); + assertEquals(0L, gc.getStatistics().numberOfObjectsSinceBitmap); + + // progress & pack + addCommit(parent); + assertEquals(1L, gc.getStatistics().numberOfObjectsSinceBitmap); + + tr.packAndPrune(); + assertEquals(4L, gc.getStatistics().numberOfObjectsSinceBitmap); + } + + private RevCommit addCommit(RevCommit parent) throws Exception { + return tr.branch("master").commit() + .author(new PersonIdent("repo-metrics", "repo@metrics.com")) + .parent(parent).create(); + } + + private long repositoryBitmapFiles() throws IOException { + return StreamSupport + .stream(Files + .newDirectoryStream(repo.getObjectDatabase() + .getPackDirectory().toPath(), "pack-*.bitmap") + .spliterator(), false) + .count(); + } + + private long getNumberOfObjectsInPacks(Collection packs) { + return packs.stream().mapToLong(pack -> { + try { + return pack.getObjectCount(); + } catch (IOException e) { + throw new RuntimeException(e); + } + }).sum(); + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java index 9494057a60..b33afed131 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java @@ -1514,6 +1514,12 @@ public class GC { */ public long numberOfPackFilesSinceBitmap; + /** + * The number of objects stored in pack files and as loose object + * created after the last bitmap generation. + */ + public long numberOfObjectsSinceBitmap; + /** * The number of objects stored as loose objects. */ @@ -1551,6 +1557,8 @@ public class GC { b.append(", numberOfPackFiles=").append(numberOfPackFiles); //$NON-NLS-1$ b.append(", numberOfPackFilesSinceBitmap=") //$NON-NLS-1$ .append(numberOfPackFilesSinceBitmap); + b.append(", numberOfObjectsSinceBitmap=") //$NON-NLS-1$ + .append(numberOfObjectsSinceBitmap); b.append(", numberOfLooseObjects=").append(numberOfLooseObjects); //$NON-NLS-1$ b.append(", numberOfLooseRefs=").append(numberOfLooseRefs); //$NON-NLS-1$ b.append(", numberOfPackedRefs=").append(numberOfPackedRefs); //$NON-NLS-1$ @@ -1571,14 +1579,19 @@ public class GC { public RepoStatistics getStatistics() throws IOException { RepoStatistics ret = new RepoStatistics(); Collection packs = repo.getObjectDatabase().getPacks(); + long latestBitmapTime = Long.MIN_VALUE; for (Pack p : packs) { - ret.numberOfPackedObjects += p.getIndex().getObjectCount(); + long packedObjects = p.getIndex().getObjectCount(); + ret.numberOfPackedObjects += packedObjects; ret.numberOfPackFiles++; ret.sizeOfPackedObjects += p.getPackFile().length(); if (p.getBitmapIndex() != null) { ret.numberOfBitmaps += p.getBitmapIndex().getBitmapCount(); + latestBitmapTime = p.getFileSnapshot().lastModifiedInstant() + .toEpochMilli(); } else { ret.numberOfPackFilesSinceBitmap++; + ret.numberOfObjectsSinceBitmap += packedObjects; } } File objDir = repo.getObjectsDirectory(); @@ -1595,6 +1608,9 @@ public class GC { continue; ret.numberOfLooseObjects++; ret.sizeOfLooseObjects += f.length(); + if (f.lastModified() > latestBitmapTime) { + ret.numberOfObjectsSinceBitmap ++; + } } } } -- cgit v1.2.3 From f072d2ec7b1fe21d403c93f8c3dac7fc9e9e526a Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Wed, 20 Nov 2024 11:18:46 -0800 Subject: PersonIdent: Revert @since of #getZoneId In [1], the @since tag of #getZoneId was updated to 7.1 by mistake. The implementation of the method is different but the API hasn't changed. Revert the tag to 6.1, when the method was introduced. [1] https://gerrithub.io/c/eclipse-jgit/jgit/+/1204142/9/org.eclipse.jgit/src/org/eclipse/jgit/lib/PersonIdent.java Change-Id: If71d763ac28d4ec02bfebb1e65f56227f44e027d --- org.eclipse.jgit/src/org/eclipse/jgit/lib/PersonIdent.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/PersonIdent.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/PersonIdent.java index a59dde7083..600fec42ca 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/PersonIdent.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/PersonIdent.java @@ -408,7 +408,7 @@ public class PersonIdent implements Serializable { * Get the time zone id * * @return the time zone id - * @since 7.1 + * @since 6.1 */ public ZoneId getZoneId() { return tzOffset; @@ -496,4 +496,3 @@ public class PersonIdent implements Serializable { return r.toString(); } } - -- cgit v1.2.3 From ee8a4149135284e2319cb5c64de8387a3334bcdd Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Wed, 20 Nov 2024 13:09:24 -0800 Subject: PersonIdent: Preserve the timezone when copying with new time The PersonIdent(PersonIdent,Date) constructor must create a copy with the same author/email/timezone but different time. When we changed the implementation to the new Instant/ZoneId version, we forgot to pass the timezone. This made fail some tests downstream. Pass the timezone when constructing the copy. Change-Id: Iaa979e6dbaa3c55d4c4d2040068ab8b03163cd4e --- org.eclipse.jgit/src/org/eclipse/jgit/lib/PersonIdent.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/PersonIdent.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/PersonIdent.java index 600fec42ca..5d3db9e6ee 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/PersonIdent.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/PersonIdent.java @@ -240,7 +240,8 @@ public class PersonIdent implements Serializable { */ @Deprecated(since = "7.1") public PersonIdent(PersonIdent pi, Date aWhen) { - this(pi.getName(), pi.getEmailAddress(), aWhen.toInstant()); + this(pi.getName(), pi.getEmailAddress(), aWhen.toInstant(), + pi.tzOffset); } /** -- cgit v1.2.3 From 4618d19629f2097f7a5c706c109284d6466596be Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Wed, 20 Nov 2024 14:06:46 -0800 Subject: GitTimeParser: Fix multiple errorprone and style comments This code came from GitDateParser and inherited these issues. It is a good time to fix them: * Use always {} in if and fors (eclipse guide) * Use the more efficient EnumMap for the formatters * variable.equals(CONSTANT) instead of CONSTANT.equals(var) https://errorprone.info/bugpattern/YodaCondition * private constructor to prevent instantiations of a utility classes https://errorprone.info/bugpattern/PrivateConstructorForUtilityClass * methods named in camelCase * Use string.split(..., int) https://errorprone.info/bugpattern/StringSplitter * Initialize variable in first use Change-Id: I0ae70ad9befc66fa9c057af135f2b0b2dab3869a --- .../jgit/util/GitTimeParserBadlyFormattedTest.java | 2 +- .../src/org/eclipse/jgit/util/GitTimeParser.java | 41 ++++++++++++++-------- 2 files changed, 27 insertions(+), 16 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/GitTimeParserBadlyFormattedTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/GitTimeParserBadlyFormattedTest.java index e5f162d11a..a59d7bc7bb 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/GitTimeParserBadlyFormattedTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/GitTimeParserBadlyFormattedTest.java @@ -45,7 +45,7 @@ public class GitTimeParserBadlyFormattedTest { @DataPoints public static String[] getDataPoints() { - return new String[] { "", "1970", "3000.3000.3000", "3 yesterday ago", + return new String[] { "", ".", "...", "1970", "3000.3000.3000", "3 yesterday ago", "now yesterday ago", "yesterdays", "3.day. 2.week.ago", "day ago", "Gra Feb 21 15:35:00 2007 +0100", "Sun Feb 21 15:35:00 2007 +0100", diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/GitTimeParser.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/GitTimeParser.java index e238e3e92e..7d00fcd5ed 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/GitTimeParser.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/GitTimeParser.java @@ -17,9 +17,10 @@ import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; import java.time.temporal.ChronoField; import java.time.temporal.TemporalAccessor; -import java.util.HashMap; +import java.util.EnumMap; import java.util.Map; +import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.internal.JGitText; /** @@ -35,7 +36,8 @@ import org.eclipse.jgit.internal.JGitText; */ public class GitTimeParser { - private static final Map formatCache = new HashMap<>(); + private static final Map formatCache = new EnumMap<>( + ParseableSimpleDateFormat.class); // An enum of all those formats which this parser can parse with the help of // a DateTimeFormatter. There are other formats (e.g. the relative formats @@ -59,6 +61,10 @@ public class GitTimeParser { } } + private GitTimeParser() { + // This class is not supposed to be instantiated + } + /** * Parses a string into a {@link java.time.LocalDateTime} using the default * locale. Since this parser also supports relative formats (e.g. @@ -95,16 +101,17 @@ public class GitTimeParser { static LocalDateTime parse(String dateStr, LocalDateTime now) throws ParseException { dateStr = dateStr.trim(); - LocalDateTime ret; - if ("never".equalsIgnoreCase(dateStr)) //$NON-NLS-1$ + if (dateStr.equalsIgnoreCase("never")) { //$NON-NLS-1$ return LocalDateTime.MAX; - ret = parse_relative(dateStr, now); - if (ret != null) + } + LocalDateTime ret = parseRelative(dateStr, now); + if (ret != null) { return ret; + } for (ParseableSimpleDateFormat f : ParseableSimpleDateFormat.values()) { try { - return parse_simple(dateStr, f); + return parseSimple(dateStr, f); } catch (DateTimeParseException e) { // simply proceed with the next parser } @@ -112,8 +119,9 @@ public class GitTimeParser { ParseableSimpleDateFormat[] values = ParseableSimpleDateFormat.values(); StringBuilder allFormats = new StringBuilder("\"") //$NON-NLS-1$ .append(values[0].formatStr); - for (int i = 1; i < values.length; i++) + for (int i = 1; i < values.length; i++) { allFormats.append("\", \"").append(values[i].formatStr); //$NON-NLS-1$ + } allFormats.append("\""); //$NON-NLS-1$ throw new ParseException( MessageFormat.format(JGitText.get().cannotParseDate, dateStr, @@ -122,10 +130,11 @@ public class GitTimeParser { } // tries to parse a string with the formats supported by DateTimeFormatter - private static LocalDateTime parse_simple(String dateStr, + private static LocalDateTime parseSimple(String dateStr, ParseableSimpleDateFormat f) throws DateTimeParseException { DateTimeFormatter dateFormat = formatCache.computeIfAbsent(f, - format -> DateTimeFormatter.ofPattern(f.formatStr) + format -> DateTimeFormatter + .ofPattern(f.formatStr) .withLocale(SystemReader.getInstance().getLocale())); TemporalAccessor parsed = dateFormat.parse(dateStr); return parsed.isSupported(ChronoField.HOUR_OF_DAY) @@ -135,25 +144,27 @@ public class GitTimeParser { // tries to parse a string with a relative time specification @SuppressWarnings("nls") - private static LocalDateTime parse_relative(String dateStr, + @Nullable + private static LocalDateTime parseRelative(String dateStr, LocalDateTime now) { // check for the static words "yesterday" or "now" - if ("now".equals(dateStr)) { + if (dateStr.equals("now")) { return now; } - if ("yesterday".equals(dateStr)) { + if (dateStr.equals("yesterday")) { return now.minusDays(1); } // parse constructs like "3 days ago", "5.week.2.day.ago" - String[] parts = dateStr.split("\\.| "); + String[] parts = dateStr.split("\\.| ", -1); int partsLength = parts.length; // check we have an odd number of parts (at least 3) and that the last // part is "ago" if (partsLength < 3 || (partsLength & 1) == 0 - || !"ago".equals(parts[parts.length - 1])) + || !parts[parts.length - 1].equals("ago")) { return null; + } int number; for (int i = 0; i < parts.length - 2; i += 2) { try { -- cgit v1.2.3 From 48f36f969542930722a9041b24bbb3c00dc919a1 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Wed, 20 Nov 2024 23:42:39 +0100 Subject: DiffDriver: fix formatting of javadoc Change-Id: I01da1862719b6623727deae10a706f30ee6cabbf --- .../src/org/eclipse/jgit/diff/DiffDriver.java | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffDriver.java b/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffDriver.java index 9ae1aaa298..9873980ff1 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffDriver.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffDriver.java @@ -28,8 +28,8 @@ import java.util.stream.Collectors; @SuppressWarnings({"ImmutableEnumChecker", "nls"}) public enum DiffDriver { /** - * Built-in diff driver for c++ + * Built-in diff driver for c++ */ cpp(List.of( /* Jump targets or access declarations */ @@ -37,16 +37,16 @@ public enum DiffDriver { /* functions/methods, variables, and compounds at top level */ "^((::\\s*)?[A-Za-z_].*)$")), /** - * Built-in diff driver for device + * Built-in diff driver for device * tree files */ dts(List.of(";", "="), List.of( /* lines beginning with a word optionally preceded by '&' or the root */ "^[ \\t]*((/[ \\t]*\\{|&?[a-zA-Z_]).*)")), /** - * Built-in diff driver for java + * Built-in diff driver for java */ java(List.of( "^[ \\t]*(catch|do|for|if|instanceof|new|return|switch|throw|while)"), @@ -58,13 +58,13 @@ public enum DiffDriver { "^[ \\t]*(([A-Za-z_<>&\\]\\[][?&<>.,A-Za-z_0-9]*[ \\t]+)+[A-Za-z_]" + "[A-Za-z_0-9]*[ \\t]*\\([^;]*)$")), /** - * Built-in diff driver for python + * Built-in diff driver for + * python */ python(List.of("^[ \\t]*((class|(async[ \\t]+)?def)[ \\t].*)$")), /** - * Built-in diff driver for java + * Built-in diff driver for + * java */ rust(List.of("^[\\t ]*((pub(\\([^\\)]+\\))?[\\t ]+)?" + "((async|const|unsafe|extern([\\t ]+\"[^\"]+\"))[\\t ]+)?" -- cgit v1.2.3 From c824610abba794a1f8f13d6ff2ec1c09590ce697 Mon Sep 17 00:00:00 2001 From: Kaushik Lingarkar Date: Wed, 20 Nov 2024 15:22:11 -0800 Subject: DiffDriver: fix doc for rust built-in Change-Id: Ic430686dc41ecbd8d8d19068416bc18ba2d94f3f --- org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffDriver.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffDriver.java b/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffDriver.java index 9873980ff1..b74444400e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffDriver.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffDriver.java @@ -64,7 +64,7 @@ public enum DiffDriver { python(List.of("^[ \\t]*((class|(async[ \\t]+)?def)[ \\t].*)$")), /** * Built-in diff driver for - * java + * rust */ rust(List.of("^[\\t ]*((pub(\\([^\\)]+\\))?[\\t ]+)?" + "((async|const|unsafe|extern([\\t ]+\"[^\"]+\"))[\\t ]+)?" -- cgit v1.2.3 From 5b1513a28d337e7e3453e557ee9dde292678eb81 Mon Sep 17 00:00:00 2001 From: pszlazak Date: Sun, 17 Nov 2024 23:11:02 +0100 Subject: Align request policies with CGit CGit defines the SHA request policies using a bitmask that represents which policy is implied by another policy. For example, in CGit the ALLOW_TIP_SHA1 is 0x01 and ALLOW_REACHABLE_SHA1 is 0x02, which are associated to two different bit in a 3-bit value. The ALLOW_ANY_SHA1 value is 0x07 which denotes a different policy that implies the previous two ones, because is represented with a 3-bit bitmask having all ones. Associate the JGit RequestPolicy enum to the same CGit bitmask values and use the same logic for the purpose of advertising the server capabilities. The JGit code becomes easier to read and associate with its counterpart in CGit, especially during the capabilities advertising phase. Also add a new utility method RequestPolicy.implies() which is more readable than a direct bitmask and operator. Bug: jgit-68 Change-Id: I932150dca1211ba9c8c34a523f13e84d7390063b (cherry picked from commit 1519c147948eb1108bdf45f2aeed84746dacff9c) --- .../src/org/eclipse/jgit/transport/UploadPack.java | 38 ++++++++++++++-------- 1 file changed, 25 insertions(+), 13 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java index 1dcfe5691f..e9ea9b25a6 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java @@ -118,13 +118,13 @@ public class UploadPack implements Closeable { /** Policy the server uses to validate client requests */ public enum RequestPolicy { /** Client may only ask for objects the server advertised a reference for. */ - ADVERTISED, + ADVERTISED(0x08), /** * Client may ask for any commit reachable from a reference advertised by * the server. */ - REACHABLE_COMMIT, + REACHABLE_COMMIT(0x02), /** * Client may ask for objects that are the tip of any reference, even if not @@ -134,18 +134,34 @@ public class UploadPack implements Closeable { * * @since 3.1 */ - TIP, + TIP(0x01), /** * Client may ask for any commit reachable from any reference, even if that - * reference wasn't advertised. + * reference wasn't advertised, implies REACHABLE_COMMIT and TIP. * * @since 3.1 */ - REACHABLE_COMMIT_TIP, + REACHABLE_COMMIT_TIP(0x03), - /** Client may ask for any SHA-1 in the repository. */ - ANY; + /** Client may ask for any SHA-1 in the repository, implies REACHABLE_COMMIT_TIP. */ + ANY(0x07); + + private final int bitmask; + + RequestPolicy(int bitmask) { + this.bitmask = bitmask; + } + + /** + * Check if the current policy implies another, based on its bitmask. + * + * @param implied the implied policy based on its bitmask. + * @return true if the policy is implied. + */ + public boolean implies(RequestPolicy implied) { + return (bitmask & implied.bitmask) != 0; + } } /** @@ -1629,13 +1645,9 @@ public class UploadPack implements Closeable { if (!biDirectionalPipe) adv.advertiseCapability(OPTION_NO_DONE); RequestPolicy policy = getRequestPolicy(); - if (policy == RequestPolicy.TIP - || policy == RequestPolicy.REACHABLE_COMMIT_TIP - || policy == null) + if (policy == null || policy.implies(RequestPolicy.TIP)) adv.advertiseCapability(OPTION_ALLOW_TIP_SHA1_IN_WANT); - if (policy == RequestPolicy.REACHABLE_COMMIT - || policy == RequestPolicy.REACHABLE_COMMIT_TIP - || policy == null) + if (policy == null || policy.implies(RequestPolicy.REACHABLE_COMMIT)) adv.advertiseCapability(OPTION_ALLOW_REACHABLE_SHA1_IN_WANT); adv.advertiseCapability(OPTION_AGENT, UserAgent.get()); if (transferConfig.isAllowFilter()) { -- cgit v1.2.3 From 6fa28d7677f8242d73dc32294fa7db6e86c23b25 Mon Sep 17 00:00:00 2001 From: Yash Chaturvedi Date: Thu, 14 Nov 2024 20:03:08 +0530 Subject: Add pack-refs command to the CLI This command can be used to optimize storage of references. For a RefDirectory database, it packs non-symbolic, loose refs into packed-refs. By default, only the references under '$GIT_DIR/refs/tags' are packed. The '--all' option can be used to pack all the references under '$GIT_DIR/refs'. For Reftable, all refs are compacted into a single table. Change-Id: I92e786403f8638d35ae3037844a7ad89e8959e02 --- .../tst/org/eclipse/jgit/pgm/PackRefsTest.java | 81 ++++++++++++++++++++ .../services/org.eclipse.jgit.pgm.TextBuiltin | 1 + .../eclipse/jgit/pgm/internal/CLIText.properties | 2 + .../src/org/eclipse/jgit/pgm/PackRefs.java | 34 +++++++++ .../jgit/internal/storage/file/GcPackRefsTest.java | 31 +++++--- .../org/eclipse/jgit/internal/JGitText.properties | 2 + org.eclipse.jgit/src/org/eclipse/jgit/api/Git.java | 10 +++ .../src/org/eclipse/jgit/api/PackRefsCommand.java | 87 ++++++++++++++++++++++ .../src/org/eclipse/jgit/internal/JGitText.java | 2 + .../storage/file/FileReftableDatabase.java | 19 +++++ .../jgit/internal/storage/file/FileRepository.java | 3 +- .../org/eclipse/jgit/internal/storage/file/GC.java | 51 +++---------- .../jgit/internal/storage/file/RefDirectory.java | 29 ++++++++ .../src/org/eclipse/jgit/lib/RefDatabase.java | 19 +++++ 14 files changed, 317 insertions(+), 54 deletions(-) create mode 100644 org.eclipse.jgit.pgm.test/tst/org/eclipse/jgit/pgm/PackRefsTest.java create mode 100644 org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/PackRefs.java create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/api/PackRefsCommand.java (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.pgm.test/tst/org/eclipse/jgit/pgm/PackRefsTest.java b/org.eclipse.jgit.pgm.test/tst/org/eclipse/jgit/pgm/PackRefsTest.java new file mode 100644 index 0000000000..b4d4ea9e56 --- /dev/null +++ b/org.eclipse.jgit.pgm.test/tst/org/eclipse/jgit/pgm/PackRefsTest.java @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2024 Qualcomm Innovation Center, Inc. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.pgm; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.io.File; + +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.internal.storage.file.FileRepository; +import org.eclipse.jgit.lib.CLIRepositoryTestCase; +import org.eclipse.jgit.lib.ConfigConstants; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.Ref; +import org.junit.Before; +import org.junit.Test; + +public class PackRefsTest extends CLIRepositoryTestCase { + private Git git; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + git = new Git(db); + git.commit().setMessage("initial commit").call(); + } + + @Test + public void tagPacked() throws Exception { + git.tag().setName("test").call(); + git.packRefs().call(); + assertEquals(Ref.Storage.PACKED, + git.getRepository().exactRef("refs/tags/test").getStorage()); + } + + @Test + public void nonTagRefNotPackedWithoutAll() throws Exception { + git.branchCreate().setName("test").call(); + git.packRefs().call(); + assertEquals(Ref.Storage.LOOSE, + git.getRepository().exactRef("refs/heads/test").getStorage()); + } + + @Test + public void nonTagRefPackedWithAll() throws Exception { + git.branchCreate().setName("test").call(); + git.packRefs().setAll(true).call(); + assertEquals(Ref.Storage.PACKED, + git.getRepository().exactRef("refs/heads/test").getStorage()); + } + + @Test + public void refTableCompacted() throws Exception { + ((FileRepository) git.getRepository()).convertRefStorage( + ConfigConstants.CONFIG_REF_STORAGE_REFTABLE, false, false); + + git.commit().setMessage("test commit").call(); + File tableDir = new File(db.getDirectory(), Constants.REFTABLE); + File[] reftables = tableDir.listFiles(); + assertNotNull(reftables); + assertTrue(reftables.length > 2); + + git.packRefs().call(); + + reftables = tableDir.listFiles(); + assertNotNull(reftables); + assertEquals(2, reftables.length); + } +} diff --git a/org.eclipse.jgit.pgm/META-INF/services/org.eclipse.jgit.pgm.TextBuiltin b/org.eclipse.jgit.pgm/META-INF/services/org.eclipse.jgit.pgm.TextBuiltin index 08d37278de..41b0091b77 100644 --- a/org.eclipse.jgit.pgm/META-INF/services/org.eclipse.jgit.pgm.TextBuiltin +++ b/org.eclipse.jgit.pgm/META-INF/services/org.eclipse.jgit.pgm.TextBuiltin @@ -26,6 +26,7 @@ org.eclipse.jgit.pgm.LsTree org.eclipse.jgit.pgm.Merge org.eclipse.jgit.pgm.MergeBase org.eclipse.jgit.pgm.MergeTool +org.eclipse.jgit.pgm.PackRefs org.eclipse.jgit.pgm.Push org.eclipse.jgit.pgm.ReceivePack org.eclipse.jgit.pgm.Reflog diff --git a/org.eclipse.jgit.pgm/resources/org/eclipse/jgit/pgm/internal/CLIText.properties b/org.eclipse.jgit.pgm/resources/org/eclipse/jgit/pgm/internal/CLIText.properties index 50ee809b98..d24b639a31 100644 --- a/org.eclipse.jgit.pgm/resources/org/eclipse/jgit/pgm/internal/CLIText.properties +++ b/org.eclipse.jgit.pgm/resources/org/eclipse/jgit/pgm/internal/CLIText.properties @@ -257,6 +257,7 @@ updating=Updating {0}..{1} usage_Abbrev=Instead of using the default number of hexadecimal digits (which will vary according to the number of objects in the repository with a default of 7) of the abbreviated object name, use digits, or as many digits as needed to form a unique object name. An of 0 will suppress long format, only showing the closest tag. usage_addRenormalize=Apply the "clean" process freshly to tracked files to forcibly add them again to the index. This implies -u. usage_Aggressive=This option will cause gc to more aggressively optimize the repository at the expense of taking much more time +usage_All=Pack all refs, except hidden refs, broken refs, and symbolic refs. usage_AlwaysFallback=Show uniquely abbreviated commit object as fallback usage_bareClone=Make a bare Git repository. That is, instead of creating [DIRECTORY] and placing the administrative files in [DIRECTORY]/.git, make the [DIRECTORY] itself the $GIT_DIR. usage_extraArgument=Pass an extra argument to a merge driver. Currently supported are "-X ours" and "-X theirs". @@ -300,6 +301,7 @@ usage_Match=Only consider tags matching the given glob(7) pattern or patterns, e usage_MergeBase=Find as good common ancestors as possible for a merge usage_MergesTwoDevelopmentHistories=Merges two development histories usage_PackKeptObjects=Include objects in packs locked by a ".keep" file when repacking +usage_PackRefs=Pack heads and tags for efficient repository access usage_PreserveOldPacks=Preserve old pack files by moving them into the preserved subdirectory instead of deleting them after repacking usage_PrunePreserved=Remove the preserved subdirectory containing previously preserved old pack files before repacking, and before preserving more old pack files usage_ReadDirCache= Read the DirCache 100 times diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/PackRefs.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/PackRefs.java new file mode 100644 index 0000000000..ee05f5ca0b --- /dev/null +++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/PackRefs.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2024 Qualcomm Innovation Center, Inc. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.pgm; + +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.lib.TextProgressMonitor; +import org.kohsuke.args4j.Option; + +@Command(common = true, usage = "usage_PackRefs") +class PackRefs extends TextBuiltin { + @Option(name = "--all", usage = "usage_All") + private boolean all; + + @Override + protected void run() { + Git git = Git.wrap(db); + try { + git.packRefs().setProgressMonitor(new TextProgressMonitor(errw)) + .setAll(all).call(); + } catch (GitAPIException e) { + throw die(e.getMessage(), e); + } + } +} diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcPackRefsTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcPackRefsTest.java index c57295518d..f84be21e82 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcPackRefsTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcPackRefsTest.java @@ -19,7 +19,6 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertSame; import java.io.File; -import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.concurrent.BrokenBarrierException; @@ -31,6 +30,8 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.PackRefsCommand; +import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.junit.TestRepository.BranchBuilder; import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; @@ -49,7 +50,7 @@ public class GcPackRefsTest extends GcTestCase { RevBlob a = tr.blob("a"); tr.lightweightTag("t", a); - gc.packRefs(); + packRefs(false); assertSame(repo.exactRef("refs/tags/t").getStorage(), Storage.PACKED); } @@ -60,7 +61,7 @@ public class GcPackRefsTest extends GcTestCase { String name = repo.findRef(ref).getName(); Path dir = repo.getCommonDirectory().toPath().resolve(name).getParent(); assertNotNull(dir); - gc.packRefs(); + packRefs(true); assertFalse(Files.exists(dir)); } @@ -75,9 +76,9 @@ public class GcPackRefsTest extends GcTestCase { Callable packRefs = () -> { syncPoint.await(); try { - gc.packRefs(); + packRefs(false); return 0; - } catch (IOException e) { + } catch (GitAPIException e) { return 1; } }; @@ -102,7 +103,7 @@ public class GcPackRefsTest extends GcTestCase { "refs/tags/t1")); try { refLock.lock(); - gc.packRefs(); + packRefs(false); } finally { refLock.unlock(); } @@ -145,7 +146,7 @@ public class GcPackRefsTest extends GcTestCase { Future result2 = pool.submit(() -> { refUpdateLockedRef.await(); - gc.packRefs(); + packRefs(false); packRefsDone.await(); return null; }); @@ -173,19 +174,20 @@ public class GcPackRefsTest extends GcTestCase { assertEquals(repo.exactRef("HEAD").getTarget().getName(), "refs/heads/master"); assertNull(repo.exactRef("HEAD").getTarget().getObjectId()); - gc.packRefs(); + PackRefsCommand packRefsCommand = git.packRefs().setAll(true); + packRefsCommand.call(); assertSame(repo.exactRef("HEAD").getStorage(), Storage.LOOSE); assertEquals(repo.exactRef("HEAD").getTarget().getName(), "refs/heads/master"); assertNull(repo.exactRef("HEAD").getTarget().getObjectId()); git.checkout().setName("refs/heads/side").call(); - gc.packRefs(); + packRefsCommand.call(); assertSame(repo.exactRef("HEAD").getStorage(), Storage.LOOSE); // check for detached HEAD git.checkout().setName(first.getName()).call(); - gc.packRefs(); + packRefsCommand.call(); assertSame(repo.exactRef("HEAD").getStorage(), Storage.LOOSE); } @@ -208,7 +210,7 @@ public class GcPackRefsTest extends GcTestCase { assertEquals(repo.exactRef("HEAD").getTarget().getName(), "refs/heads/master"); assertNull(repo.exactRef("HEAD").getTarget().getObjectId()); - gc.packRefs(); + packRefs(true); assertSame(repo.exactRef("HEAD").getStorage(), Storage.LOOSE); assertEquals(repo.exactRef("HEAD").getTarget().getName(), "refs/heads/master"); @@ -216,9 +218,14 @@ public class GcPackRefsTest extends GcTestCase { // check for non-detached HEAD repo.updateRef(Constants.HEAD).link("refs/heads/side"); - gc.packRefs(); + packRefs(true); assertSame(repo.exactRef("HEAD").getStorage(), Storage.LOOSE); assertEquals(repo.exactRef("HEAD").getTarget().getObjectId(), second.getId()); } + + private void packRefs(boolean all) throws GitAPIException { + new PackRefsCommand(repo).setAll(all).call(); + } + } diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties index 024ca77f21..acfe812a20 100644 --- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties +++ b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties @@ -597,6 +597,8 @@ packInaccessible=Failed to access pack file {0}, caught {1} consecutive errors w packingCancelledDuringObjectsWriting=Packing cancelled during objects writing packObjectCountMismatch=Pack object count mismatch: pack {0} index {1}: {2} packRefs=Pack refs +packRefsFailed=Packing refs failed +packRefsSuccessful=Packed refs successfully packSizeNotSetYet=Pack size not yet set since it has not yet been received packTooLargeForIndexVersion1=Pack too large for index version 1 packWasDeleted=Pack file {0} was deleted, removing it from pack list diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/Git.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/Git.java index 3dc53ec248..5bc035a46a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/Git.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/Git.java @@ -713,6 +713,16 @@ public class Git implements AutoCloseable { return new GarbageCollectCommand(repo); } + /** + * Return a command object to execute a {@code PackRefs} command + * + * @return a {@link org.eclipse.jgit.api.PackRefsCommand} + * @since 7.1 + */ + public PackRefsCommand packRefs() { + return new PackRefsCommand(repo); + } + /** * Return a command object to find human-readable names of revisions. * diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/PackRefsCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/PackRefsCommand.java new file mode 100644 index 0000000000..29a69c5ac4 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/PackRefsCommand.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2024 Qualcomm Innovation Center, Inc. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.api; + +import java.io.IOException; + +import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.api.errors.JGitInternalException; +import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.lib.NullProgressMonitor; +import org.eclipse.jgit.lib.ProgressMonitor; +import org.eclipse.jgit.lib.Repository; + +/** + * Optimize storage of references. + * + * @since 7.1 + */ +public class PackRefsCommand extends GitCommand { + private ProgressMonitor monitor; + + private boolean all; + + /** + * Creates a new {@link PackRefsCommand} instance with default values. + * + * @param repo + * the repository this command will be used on + */ + public PackRefsCommand(Repository repo) { + super(repo); + this.monitor = NullProgressMonitor.INSTANCE; + } + + /** + * Set progress monitor + * + * @param monitor + * a progress monitor + * @return this instance + */ + public PackRefsCommand setProgressMonitor(ProgressMonitor monitor) { + this.monitor = monitor; + return this; + } + + /** + * Specify whether to pack all the references. + * + * @param all + * if true all the loose refs will be packed + * @return this instance + */ + public PackRefsCommand setAll(boolean all) { + this.all = all; + return this; + } + + /** + * Whether to pack all the references + * + * @return whether to pack all the references + */ + public boolean isAll() { + return all; + } + + @Override + public String call() throws GitAPIException { + checkCallable(); + try { + repo.getRefDatabase().packRefs(monitor, this); + return JGitText.get().packRefsSuccessful; + } catch (IOException e) { + throw new JGitInternalException(JGitText.get().packRefsFailed, e); + } + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java index 0980219e25..2d9d2c527c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java @@ -627,6 +627,8 @@ public class JGitText extends TranslationBundle { /***/ public String packingCancelledDuringObjectsWriting; /***/ public String packObjectCountMismatch; /***/ public String packRefs; + /***/ public String packRefsFailed; + /***/ public String packRefsSuccessful; /***/ public String packSizeNotSetYet; /***/ public String packTooLargeForIndexVersion1; /***/ public String packWasDeleted; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java index 80240e5062..25b7583b95 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java @@ -28,8 +28,10 @@ import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; import org.eclipse.jgit.annotations.NonNull; +import org.eclipse.jgit.api.PackRefsCommand; import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.events.RefsChangedEvent; +import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.reftable.MergedReftable; import org.eclipse.jgit.internal.storage.reftable.ReftableBatchRefUpdate; import org.eclipse.jgit.internal.storage.reftable.ReftableDatabase; @@ -39,6 +41,7 @@ import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectIdRef; import org.eclipse.jgit.lib.PersonIdent; +import org.eclipse.jgit.lib.ProgressMonitor; import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.RefDatabase; import org.eclipse.jgit.lib.RefRename; @@ -107,6 +110,22 @@ public class FileReftableDatabase extends RefDatabase { return reftableDatabase.hasFastTipsWithSha1(); } + /** + * {@inheritDoc} + * + * For Reftable, all the data is compacted into a single table. + */ + @Override + public void packRefs(ProgressMonitor pm, PackRefsCommand packRefs) + throws IOException { + pm.beginTask(JGitText.get().packRefs, 1); + try { + compactFully(); + } finally { + pm.endTask(); + } + } + /** * Runs a full compaction for GC purposes. * @throws IOException on I/O errors diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java index 230dc6f45a..84c85659ff 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java @@ -33,6 +33,7 @@ import java.util.Set; import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.annotations.Nullable; +import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.api.errors.JGitInternalException; import org.eclipse.jgit.attributes.AttributesNode; import org.eclipse.jgit.attributes.AttributesNodeProvider; @@ -599,7 +600,7 @@ public class FileRepository extends Repository { gc.setBackground(shouldAutoDetach()); try { gc.gc(); - } catch (ParseException | IOException e) { + } catch (ParseException | IOException | GitAPIException e) { throw new JGitInternalException(JGitText.get().gcFailed, e); } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java index c9da4d8d67..7f3369364b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java @@ -63,6 +63,8 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.eclipse.jgit.annotations.NonNull; +import org.eclipse.jgit.api.PackRefsCommand; +import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.dircache.DirCacheIterator; import org.eclipse.jgit.errors.CancelledException; import org.eclipse.jgit.errors.CorruptObjectException; @@ -233,9 +235,11 @@ public class GC { * @throws java.text.ParseException * If the configuration parameter "gc.pruneexpire" couldn't be * parsed + * @throws GitAPIException + * If packing refs failed */ public CompletableFuture> gc() - throws IOException, ParseException { + throws IOException, ParseException, GitAPIException { if (!background) { return CompletableFuture.completedFuture(doGc()); } @@ -254,7 +258,7 @@ public class GC { gcLog.commit(); } return newPacks; - } catch (IOException | ParseException e) { + } catch (IOException | ParseException | GitAPIException e) { try { gcLog.write(e.getMessage()); StringWriter sw = new StringWriter(); @@ -277,7 +281,8 @@ public class GC { return (executor != null) ? executor : WorkQueue.getExecutor(); } - private Collection doGc() throws IOException, ParseException { + private Collection doGc() + throws IOException, ParseException, GitAPIException { if (automatic && !needGc()) { return Collections.emptyList(); } @@ -286,7 +291,8 @@ public class GC { return Collections.emptyList(); } pm.start(6 /* tasks */); - packRefs(); + new PackRefsCommand(repo).setProgressMonitor(pm).setAll(true) + .call(); // TODO: implement reflog_expire(pm, repo); Collection newPacks = repack(); prune(Collections.emptySet()); @@ -779,43 +785,6 @@ public class GC { && Objects.equals(r1.getObjectId(), r2.getObjectId()); } - /** - * Pack ref storage. For a RefDirectory database, this packs all - * non-symbolic, loose refs into packed-refs. For Reftable, all of the data - * is compacted into a single table. - * - * @throws java.io.IOException - * if an IO error occurred - */ - public void packRefs() throws IOException { - RefDatabase refDb = repo.getRefDatabase(); - if (refDb instanceof FileReftableDatabase) { - // TODO: abstract this more cleanly. - pm.beginTask(JGitText.get().packRefs, 1); - try { - ((FileReftableDatabase) refDb).compactFully(); - } finally { - pm.endTask(); - } - return; - } - - Collection refs = refDb.getRefsByPrefix(Constants.R_REFS); - List refsToBePacked = new ArrayList<>(refs.size()); - pm.beginTask(JGitText.get().packRefs, refs.size()); - try { - for (Ref ref : refs) { - checkCancelled(); - if (!ref.isSymbolic() && ref.getStorage().isLoose()) - refsToBePacked.add(ref.getName()); - pm.update(1); - } - ((RefDirectory) repo.getRefDatabase()).pack(refsToBePacked); - } finally { - pm.endTask(); - } - } - /** * Packs all objects which reachable from any of the heads into one pack * file. Additionally all objects which are not reachable from any head but diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java index 604868133e..6aa1157e37 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java @@ -57,6 +57,7 @@ import java.util.stream.Stream; import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.annotations.Nullable; +import org.eclipse.jgit.api.PackRefsCommand; import org.eclipse.jgit.errors.InvalidObjectIdException; import org.eclipse.jgit.errors.LockFailedException; import org.eclipse.jgit.errors.MissingObjectException; @@ -69,6 +70,7 @@ import org.eclipse.jgit.lib.CoreConfig.TrustLooseRefStat; import org.eclipse.jgit.lib.CoreConfig.TrustPackedRefsStat; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectIdRef; +import org.eclipse.jgit.lib.ProgressMonitor; import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.RefComparator; import org.eclipse.jgit.lib.RefDatabase; @@ -287,6 +289,33 @@ public class RefDirectory extends RefDatabase { clearReferences(); } + /** + * {@inheritDoc} + * + * For a RefDirectory database, by default this packs non-symbolic, loose + * tag refs into packed-refs. If {@code all} flag is set, this packs all the + * non-symbolic, loose refs. + */ + @Override + public void packRefs(ProgressMonitor pm, PackRefsCommand packRefs) + throws IOException { + String prefix = packRefs.isAll() ? R_REFS : R_TAGS; + Collection refs = getRefsByPrefix(prefix); + List refsToBePacked = new ArrayList<>(refs.size()); + pm.beginTask(JGitText.get().packRefs, refs.size()); + try { + for (Ref ref : refs) { + if (!ref.isSymbolic() && ref.getStorage().isLoose()) { + refsToBePacked.add(ref.getName()); + } + pm.update(1); + } + pack(refsToBePacked); + } finally { + pm.endTask(); + } + } + @Override public boolean isNameConflicting(String name) throws IOException { // Cannot be nested within an existing reference. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefDatabase.java index 114246beb2..09cb5a83dd 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefDatabase.java @@ -26,6 +26,7 @@ import java.util.stream.Stream; import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.annotations.Nullable; +import org.eclipse.jgit.api.PackRefsCommand; /** * Abstraction of name to {@link org.eclipse.jgit.lib.ObjectId} mapping. @@ -593,4 +594,22 @@ public abstract class RefDatabase { } return null; } + + /** + * Optimize pack ref storage. + * + * @param pm + * a progress monitor + * + * @param packRefs + * {@link PackRefsCommand} to control ref packing behavior + * + * @throws java.io.IOException + * if an IO error occurred + * @since 7.1 + */ + public void packRefs(ProgressMonitor pm, PackRefsCommand packRefs) + throws IOException { + // nothing + } } -- cgit v1.2.3 From f026c19a054a5247ddcdf747479be87b7e01152e Mon Sep 17 00:00:00 2001 From: Martin Fick Date: Fri, 22 Nov 2024 17:08:57 -0800 Subject: PackDirectory: Filter out tmp GC pack files git repack passes a ".tmp-XXXX-" prefix to git pack-objects when repacking. git pack-objects then adds a "pack-XXXXX.pack" to this to create the name of new packfiles it writes to. PackDirectory was previously very lenient and would allow these files to be added to its list of known packfiles. Fix PackDirectory to filter these out since they are not meant to be consumed yet, and doing so can cause user facing errors. Change-Id: I072e57d9522e02049db17d3f4977df7eda14bba7 Signed-off-by: Martin Fick --- .../src/org/eclipse/jgit/internal/storage/file/PackDirectory.java | 2 +- .../src/org/eclipse/jgit/internal/storage/file/PackFile.java | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java index 8221cff442..51a8148838 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java @@ -545,7 +545,7 @@ class PackDirectory { for (String name : nameList) { try { PackFile pack = new PackFile(directory, name); - if (pack.getPackExt() != null) { + if (pack.getPackExt() != null && !pack.isTmpGCFile()) { Map packByExt = packFilesByExtById .get(pack.getId()); if (packByExt == null) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java index 19979d0ed5..c9b05ad025 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java @@ -27,6 +27,7 @@ public class PackFile extends File { private static final long serialVersionUID = 1L; private static final String PREFIX = "pack-"; //$NON-NLS-1$ + private static final String TMP_GC_PREFIX = ".tmp-"; //$NON-NLS-1$ private final String base; // PREFIX + id i.e. // pack-0123456789012345678901234567890123456789 @@ -125,6 +126,13 @@ public class PackFile extends File { return packExt; } + /** + * @return whether the file is a temporary GC file + */ + public boolean isTmpGCFile() { + return id.startsWith(TMP_GC_PREFIX); + } + /** * Create a new similar PackFile with the given extension instead. * -- cgit v1.2.3 From 90da22c118c19c1c42029ae30c5a2a424ca62e7e Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Wed, 27 Nov 2024 13:10:21 +0100 Subject: DiffFormatter: silence non-externalized string warnings Change-Id: I08488598d1cc58f17e6994cf00e74b20307ada06 --- org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffFormatter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffFormatter.java b/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffFormatter.java index fa446e18cd..bc07cedf4f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffFormatter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffFormatter.java @@ -1334,7 +1334,7 @@ public class DiffFormatter implements AutoCloseable { continue; } if (matchesAny(diffDriver.getMatchPatterns(), line)) { - String funcName = line.replaceAll("^[ \\t]+", ""); + String funcName = line.replaceAll("^[ \\t]+", ""); //$NON-NLS-1$//$NON-NLS-2$ return funcName.substring(0, Math.min(funcName.length(), 80)).trim(); } -- cgit v1.2.3 From 702db13af2b96c0f699fa6be0a63ba281535f337 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Sun, 1 Dec 2024 21:59:03 +0100 Subject: UploadPack#implies: add missing @since tag The method was originally introduced in 7.1 and then back ported to the stable-6.10 branch. Change-Id: I7250013227a666de44e439802d51110bba5af2b9 --- org.eclipse.jgit/.settings/.api_filters | 8 ++++++++ org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java | 4 +++- 2 files changed, 11 insertions(+), 1 deletion(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/.settings/.api_filters b/org.eclipse.jgit/.settings/.api_filters index d3ae4cf376..94adb25f78 100644 --- a/org.eclipse.jgit/.settings/.api_filters +++ b/org.eclipse.jgit/.settings/.api_filters @@ -80,4 +80,12 @@ + + + + + + + + diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java index e9ea9b25a6..968728de8d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java @@ -156,8 +156,10 @@ public class UploadPack implements Closeable { /** * Check if the current policy implies another, based on its bitmask. * - * @param implied the implied policy based on its bitmask. + * @param implied + * the implied policy based on its bitmask. * @return true if the policy is implied. + * @since 6.10.1 */ public boolean implies(RequestPolicy implied) { return (bitmask & implied.bitmask) != 0; -- cgit v1.2.3 From 44d61a3d727eb68db9e57acec0704bfc6758f878 Mon Sep 17 00:00:00 2001 From: Martin Fick Date: Wed, 27 Nov 2024 10:15:30 -0800 Subject: WindowCache: add bulk purge(), call from bulk sites Purging a Pack from the WindowCache requires a linear search over all the entries in the cache, and thus is rather expensive. Since git gc often deletes more than one packfile at a time, avoid multiplying this expensive operation when possible by purging a Set of Packs when closing deleted pack files during a directory scan. Purge the Set of Packs with a single linear search of the cache. Closing the PackDirectory also cccbngljkihltghcnbiftcubdvgugdcvujkejehbjr Change-Id: Ic9b45cab57e1ef610c2e20ad392d8b45f8091f41 Signed-off-by: Martin Fick --- .../eclipse/jgit/internal/storage/file/Pack.java | 25 ++++++++++++++++------ .../jgit/internal/storage/file/PackDirectory.java | 9 +++----- .../jgit/internal/storage/file/WindowCache.java | 24 +++++++++++++-------- 3 files changed, 37 insertions(+), 21 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java index 3518342f97..9073c1675a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java @@ -296,15 +296,28 @@ public class Pack implements Iterable { } /** - * Close the resources utilized by this repository + * Close the resources utilized by these pack files + * + * @param packs + * packs to close + */ + public static void close(Set packs) { + WindowCache.purge(packs); + packs.forEach(p -> p.closeIndices()); + } + + /** + * Close the resources utilized by this pack file */ public void close() { WindowCache.purge(this); - synchronized (this) { - loadedIdx.clear(); - reverseIdx.clear(); - bitmapIdx.clear(); - } + closeIndices(); + } + + private synchronized void closeIndices() { + loadedIdx.clear(); + reverseIdx.clear(); + bitmapIdx.clear(); } /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java index 51a8148838..e31126f027 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java @@ -24,6 +24,7 @@ import java.util.Collection; import java.util.Collections; import java.util.EnumMap; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -111,9 +112,7 @@ class PackDirectory { void close() { PackList packs = packList.get(); if (packs != NO_PACKS && packList.compareAndSet(packs, NO_PACKS)) { - for (Pack p : packs.packs) { - p.close(); - } + Pack.close(Set.of(packs.packs)); } } @@ -484,9 +483,7 @@ class PackDirectory { return old; } - for (Pack p : forReuse.values()) { - p.close(); - } + Pack.close(new HashSet<>(forReuse.values())); if (list.isEmpty()) { return new PackList(snapshot, NO_PACKS.packs); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCache.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCache.java index 30f8240aa9..3ec7e6e666 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCache.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCache.java @@ -17,6 +17,7 @@ import java.lang.ref.SoftReference; import java.util.Collections; import java.util.Map; import java.util.Random; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicBoolean; @@ -397,7 +398,11 @@ public class WindowCache { } static final void purge(Pack pack) { - cache.removeAll(pack); + purge(Collections.singleton(pack)); + } + + static final void purge(Set packs) { + cache.removeAll(packs); } /** cleanup released and/or garbage collected windows. */ @@ -710,20 +715,21 @@ public class WindowCache { /** * Clear all entries related to a single file. *

- * Typically this method is invoked during {@link Pack#close()}, when we - * know the pack is never going to be useful to us again (for example, it no - * longer exists on disk). A concurrent reader loading an entry from this - * same pack may cause the pack to become stuck in the cache anyway. + * Typically this method is invoked during {@link Pack#close()}, or + * {@link Pack#close(Set)}, when we know the packs are never going to be + * useful to us again (for example, they no longer exist on disk after gc). + * A concurrent reader loading an entry from these same packs may cause a + * pack to become stuck in the cache anyway. * - * @param pack - * the file to purge all entries of. + * @param packs + * the files to purge all entries of. */ - private void removeAll(Pack pack) { + private void removeAll(Set packs) { for (int s = 0; s < tableSize; s++) { final Entry e1 = table.get(s); boolean hasDead = false; for (Entry e = e1; e != null; e = e.next) { - if (e.ref.getPack() == pack) { + if (packs.contains(e.ref.getPack())) { e.kill(); hasDead = true; } else if (e.dead) -- cgit v1.2.3 From fb6adb0366bc3e15f30952fe49456b1f96bcf1af Mon Sep 17 00:00:00 2001 From: Martin Fick Date: Fri, 22 Nov 2024 17:57:17 -0800 Subject: Pack: ensure packfile is still valid while still recoverable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When copyAsIs2() needs to send a large object requiring multiple read iterations — any of which could fail if the object isn't already in the WindowCache — verify first that the packfile is still valid at the latest possible moment, just before sending the header that would make recovery impossible. Change-Id: I234fd4b315b579a0506c8fbdea0c6787bdc09fcd Signed-off-by: Martin Fick --- .../src/org/eclipse/jgit/internal/storage/file/Pack.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java index 9073c1675a..61577a956c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java @@ -597,13 +597,19 @@ public class Pack implements Iterable { assert(crc2 != null); crc2.update(buf, 0, n); } + cnt -= n; if (!isHeaderWritten) { + if (invalid && cnt > 0) { + // Since this is not the last iteration and the packfile is invalid, + // better to assume the iterations will not all complete here while + // it is still likely recoverable. + throw new StoredObjectRepresentationNotAvailableException(invalidatingCause); + } out.writeHeader(src, inflatedLength); isHeaderWritten = true; } out.write(buf, 0, n); pos += n; - cnt -= n; } if (validate) { assert(crc2 != null); -- cgit v1.2.3 From bd57a19fa35b6de493926e2996b28235b96b5690 Mon Sep 17 00:00:00 2001 From: "jackdt@google.com" Date: Tue, 3 Dec 2024 16:21:55 -0800 Subject: TreeWalk: Make a null check before dereferencing the config variable. Gerrit finds a null pointer exception accessing the conf in some merge operations. It is not clear why, but could be related to [1] and [2] where the attributes node provider is set (instead of being null). Check if the config is there before reading the value and return null if not (as if the config didn't have the value set). The config is @Nullable so this check makes sense in any case. [1] https://review.gerrithub.io/c/eclipse-jgit/jgit/+/1203278 [2] https://gerrit-review.git.corp.google.com/c/gerrit/+/441721 Change-Id: I2b786c0a23fa2e09fb9fe041c025d42823defb04 --- org.eclipse.jgit/src/org/eclipse/jgit/treewalk/TreeWalk.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/TreeWalk.java b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/TreeWalk.java index aaac2a72e0..f66b3f65de 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/TreeWalk.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/TreeWalk.java @@ -38,12 +38,12 @@ import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.Config; import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; -import org.eclipse.jgit.lib.CoreConfig.EolStreamType; import org.eclipse.jgit.lib.FileMode; import org.eclipse.jgit.lib.MutableObjectId; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectReader; import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.lib.CoreConfig.EolStreamType; import org.eclipse.jgit.revwalk.RevTree; import org.eclipse.jgit.treewalk.filter.PathFilter; import org.eclipse.jgit.treewalk.filter.TreeFilter; @@ -1591,6 +1591,9 @@ public class TreeWalk implements AutoCloseable, AttributesProvider { String filterCommand = filterCommandsByNameDotType.get(key); if (filterCommand != null) return filterCommand; + if (config == null) { + return null; + } filterCommand = config.getString(ConfigConstants.CONFIG_FILTER_SECTION, filterDriverName, filterCommandType); boolean useBuiltin = config.getBoolean( -- cgit v1.2.3 From f2741cacec9f46724a0e48cc55cbb8892361e3d0 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Wed, 4 Dec 2024 10:34:10 +0100 Subject: Fix NPE in DiffFormatter#getDiffDriver Guard against diff attribute with a null value. Change-Id: Icc4b8d2f360ef105c6d64716cb42f2979967075b --- .../src/org/eclipse/jgit/diff/DiffFormatter.java | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffFormatter.java b/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffFormatter.java index fa446e18cd..6375a60383 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffFormatter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffFormatter.java @@ -1356,13 +1356,17 @@ public class DiffFormatter implements AutoCloseable { private DiffDriver getDiffDriver(DiffEntry entry) { Attribute diffAttr = entry.getDiffAttribute(); - if (diffAttr != null) { - try { - return DiffDriver.valueOf(diffAttr.getValue()); - } catch (IllegalArgumentException e) { - return null; - } + if (diffAttr == null) { + return null; + } + String diffAttrValue = diffAttr.getValue(); + if (diffAttrValue == null) { + return null; + } + try { + return DiffDriver.valueOf(diffAttrValue); + } catch (IllegalArgumentException e) { + return null; } - return null; } } -- cgit v1.2.3 From 7f246a05dcbf11a6c9dcd21215f38f62ad801866 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Wed, 4 Dec 2024 11:19:39 +0100 Subject: Fix potential NPE in ResolveMerger#getAttributesContentMergeStrategy Change-Id: I2a4c57438c16a0c5bc1af4c7772eaf65049911e2 --- org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java b/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java index a6a287bcf4..a50a6440b9 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/merge/ResolveMerger.java @@ -1507,9 +1507,12 @@ public class ResolveMerger extends ThreeWayMerger { private ContentMergeStrategy getAttributesContentMergeStrategy( Attributes attributes, ContentMergeStrategy strategy) { Attribute attr = attributes.get(Constants.ATTR_MERGE); - if (attr != null && attr.getValue() - .equals(Constants.ATTR_BUILTIN_UNION_MERGE_DRIVER)) { - return ContentMergeStrategy.UNION; + if (attr != null) { + String attrValue = attr.getValue(); + if (attrValue != null && attrValue + .equals(Constants.ATTR_BUILTIN_UNION_MERGE_DRIVER)) { + return ContentMergeStrategy.UNION; + } } return strategy; } -- cgit v1.2.3 From 2f597a662e39a53cb2baac02cf0881204cb3555b Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Tue, 19 Nov 2024 12:01:31 -0800 Subject: RawParseUtils: Use java.time API to build PersonIdents PersonIdent has deprecated the long/int constructors for the time, in favor of Instant/ZoneId. Update RawParseUtils to create PersonIdents with the new constructors. Change-Id: Ic62ff0c4219b4bd228c694e73e00a7c794e3553a --- .../src/org/eclipse/jgit/util/RawParseUtils.java | 53 ++++++++++++++++------ 1 file changed, 39 insertions(+), 14 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/RawParseUtils.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/RawParseUtils.java index 2ce86904c5..1fc695959a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/RawParseUtils.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/RawParseUtils.java @@ -13,6 +13,8 @@ package org.eclipse.jgit.util; import static java.nio.charset.StandardCharsets.ISO_8859_1; import static java.nio.charset.StandardCharsets.UTF_8; +import static java.time.Instant.EPOCH; +import static java.time.ZoneOffset.UTC; import static org.eclipse.jgit.lib.ObjectChecker.author; import static org.eclipse.jgit.lib.ObjectChecker.committer; import static org.eclipse.jgit.lib.ObjectChecker.encoding; @@ -30,6 +32,9 @@ import java.nio.charset.CharsetDecoder; import java.nio.charset.CodingErrorAction; import java.nio.charset.IllegalCharsetNameException; import java.nio.charset.UnsupportedCharsetException; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZoneOffset; import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -458,6 +463,24 @@ public final class RawParseUtils { return tzHours * 60 + tzMins; } + /** + * Parse a Git style timezone string in [+-]hhmm format + * + * @param b + * buffer to scan. + * @param ptr + * position within buffer to start parsing digits at. + * @param ptrResult + * optional location to return the new ptr value through. If null + * the ptr value will be discarded. + * @return the ZoneOffset represention of the timezone offset string + */ + private static ZoneId parseZoneOffset(final byte[] b, int ptr, + MutableInteger ptrResult) { + int hhmm = parseBase10(b, ptr, ptrResult); + return ZoneOffset.ofHoursMinutes(hhmm / 100, hhmm % 100); + } + /** * Locate the first position after a given character. * @@ -1027,17 +1050,19 @@ public final class RawParseUtils { // character if there is no trailing LF. final int tzBegin = lastIndexOfTrim(raw, ' ', nextLF(raw, emailE - 1) - 2) + 1; - if (tzBegin <= emailE) // No time/zone, still valid - return new PersonIdent(name, email, 0, 0); + if (tzBegin <= emailE) { // No time/zone, still valid + return new PersonIdent(name, email, EPOCH, UTC); + } final int whenBegin = Math.max(emailE, lastIndexOfTrim(raw, ' ', tzBegin - 1) + 1); - if (whenBegin >= tzBegin - 1) // No time/zone, still valid - return new PersonIdent(name, email, 0, 0); + if (whenBegin >= tzBegin - 1) { // No time/zone, still valid + return new PersonIdent(name, email, EPOCH, UTC); + } - final long when = parseLongBase10(raw, whenBegin, null); - final int tz = parseTimeZoneOffset(raw, tzBegin); - return new PersonIdent(name, email, when * 1000L, tz); + long when = parseLongBase10(raw, whenBegin, null); + return new PersonIdent(name, email, Instant.ofEpochSecond(when), + parseZoneOffset(raw, tzBegin, null)); } /** @@ -1075,16 +1100,16 @@ public final class RawParseUtils { name = decode(raw, nameB, stop); final MutableInteger ptrout = new MutableInteger(); - long when; - int tz; + Instant when; + ZoneId tz; if (emailE < stop) { - when = parseLongBase10(raw, emailE + 1, ptrout); - tz = parseTimeZoneOffset(raw, ptrout.value); + when = Instant.ofEpochSecond(parseLongBase10(raw, emailE + 1, ptrout)); + tz = parseZoneOffset(raw, ptrout.value, null); } else { - when = 0; - tz = 0; + when = EPOCH; + tz = UTC; } - return new PersonIdent(name, email, when * 1000L, tz); + return new PersonIdent(name, email, when, tz); } /** -- cgit v1.2.3 From 4156bdfe8ef8af2cc1d48a185e62fdf83e14c8ed Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Wed, 4 Dec 2024 11:20:19 +0100 Subject: Mark Attribute#getValue as @Nullable Change-Id: I172c43ff2e3e682f38a91ce288190245fa5d5ebe --- org.eclipse.jgit/src/org/eclipse/jgit/attributes/Attribute.java | 3 +++ 1 file changed, 3 insertions(+) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/attributes/Attribute.java b/org.eclipse.jgit/src/org/eclipse/jgit/attributes/Attribute.java index fe3e22a21f..9c4d8700a2 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/attributes/Attribute.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/attributes/Attribute.java @@ -9,6 +9,8 @@ */ package org.eclipse.jgit.attributes; +import org.eclipse.jgit.annotations.Nullable; + /** * Represents an attribute. *

@@ -139,6 +141,7 @@ public final class Attribute { * * @return the attribute value (may be null) */ + @Nullable public String getValue() { return value; } -- cgit v1.2.3 From dd8b13acc0d60b53f399ce7e0c6718b81ad2ab16 Mon Sep 17 00:00:00 2001 From: Martin Fick Date: Thu, 5 Dec 2024 13:37:37 -0800 Subject: FileSnapshot: refactor to share error handling It is important to keep the exception handling for getting file attributes the same in all places in this class; put that code into a common method. Change-Id: I1fcce5efd10aa562a4e0e34d3ce94bcc83850237 Signed-off-by: Martin Fick --- .../jgit/internal/storage/file/FileSnapshot.java | 30 +++++++++++----------- 1 file changed, 15 insertions(+), 15 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java index c88ac984ec..c5e9e99673 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java @@ -22,6 +22,7 @@ import java.time.Instant; import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.util.Locale; +import java.util.NoSuchElementException; import java.util.Objects; import java.util.concurrent.TimeUnit; @@ -231,14 +232,8 @@ public class FileSnapshot { this.useConfig = useConfig; BasicFileAttributes fileAttributes = null; try { - fileAttributes = FS.DETECTED.fileAttributes(file); - } catch (NoSuchFileException e) { - this.lastModified = Instant.EPOCH; - this.size = 0L; - this.fileKey = MISSING_FILEKEY; - return; - } catch (IOException e) { - LOG.error(e.getMessage(), e); + fileAttributes = getFileAttributes(file); + } catch (NoSuchElementException e) { this.lastModified = Instant.EPOCH; this.size = 0L; this.fileKey = MISSING_FILEKEY; @@ -319,16 +314,11 @@ public class FileSnapshot { long currSize; Object currFileKey; try { - BasicFileAttributes fileAttributes = FS.DETECTED.fileAttributes(path); + BasicFileAttributes fileAttributes = getFileAttributes(path); currLastModified = fileAttributes.lastModifiedTime().toInstant(); currSize = fileAttributes.size(); currFileKey = getFileKey(fileAttributes); - } catch (NoSuchFileException e) { - currLastModified = Instant.EPOCH; - currSize = 0L; - currFileKey = MISSING_FILEKEY; - } catch (IOException e) { - LOG.error(e.getMessage(), e); + } catch (NoSuchElementException e) { currLastModified = Instant.EPOCH; currSize = 0L; currFileKey = MISSING_FILEKEY; @@ -586,4 +576,14 @@ public class FileSnapshot { } return fileStoreAttributeCache; } + + private static BasicFileAttributes getFileAttributes(File path) throws NoSuchElementException { + try { + return FS.DETECTED.fileAttributes(path); + } catch (NoSuchFileException e) { + } catch (IOException e) { + LOG.error(e.getMessage(), e); + } + throw new NoSuchElementException(path.toString()); + } } -- cgit v1.2.3 From eb0ef9d16ea780c431837f904ad4d1ad0aaa5d7d Mon Sep 17 00:00:00 2001 From: Martin Fick Date: Thu, 5 Dec 2024 14:33:34 -0800 Subject: FileSnapshot: silence "Not a Directory" exceptions Sometimes a FileSystemException with "Not a Directory" can be thrown while creating a FileSnapshot, likely because the file or directory was deleted. Since NoSuchFileExceptions are already silenced, and the FileSnapshot already handles all IOExceptions, there is likely no value in seeing this info in the logs, treat these situation the same and silence them also. Change-Id: I022639c42154a0a4c71065741f023f5eb95f9b55 Signed-off-by: Martin Fick --- .../src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java index c5e9e99673..d34baf358c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java @@ -15,6 +15,7 @@ import static org.eclipse.jgit.util.FS.FileStoreAttributes.FALLBACK_TIMESTAMP_RE import java.io.File; import java.io.IOException; +import java.nio.file.FileSystemException; import java.nio.file.NoSuchFileException; import java.nio.file.attribute.BasicFileAttributes; import java.time.Duration; @@ -581,6 +582,10 @@ public class FileSnapshot { try { return FS.DETECTED.fileAttributes(path); } catch (NoSuchFileException e) { + } catch (FileSystemException e) { + if (!e.getMessage().endsWith("Not a directory")) { + LOG.error(e.getMessage(), e); + } } catch (IOException e) { LOG.error(e.getMessage(), e); } -- cgit v1.2.3 From 7fedd15c82eda8c8ff694b6e809e83bd64a5ab33 Mon Sep 17 00:00:00 2001 From: Martin Fick Date: Thu, 5 Dec 2024 15:53:01 -0800 Subject: FileSnapshot: silence "Stale file handle" exceptions Sometimes a FileSystemException with "Stale file handle" can be thrown while creating a FileSnapshot, likely because the file or directory was deleted. Since NoSuchFileExceptions are already silenced, and the FileSnapshot already handles all IOExceptions, there is likely no value in seeing this info in the logs, treat these situation the same and silence them also. Change-Id: I922f4edf2d332cd704e60276f41a76df242f281c Signed-off-by: Martin Fick --- .../eclipse/jgit/internal/storage/file/FileSnapshot.java | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java index d34baf358c..d752e315db 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java @@ -28,6 +28,7 @@ import java.util.Objects; import java.util.concurrent.TimeUnit; import org.eclipse.jgit.annotations.NonNull; +import org.eclipse.jgit.util.FileUtils; import org.eclipse.jgit.util.FS; import org.eclipse.jgit.util.FS.FileStoreAttributes; import org.slf4j.Logger; @@ -580,11 +581,18 @@ public class FileSnapshot { private static BasicFileAttributes getFileAttributes(File path) throws NoSuchElementException { try { - return FS.DETECTED.fileAttributes(path); + try { + return FS.DETECTED.fileAttributes(path); + } catch (IOException e) { + if (!FileUtils.isStaleFileHandle(e)) { + throw e; + } + } } catch (NoSuchFileException e) { } catch (FileSystemException e) { - if (!e.getMessage().endsWith("Not a directory")) { - LOG.error(e.getMessage(), e); + String msg = e.getMessage(); + if (!msg.endsWith("Not a directory")) { + LOG.error(msg, e); } } catch (IOException e) { LOG.error(e.getMessage(), e); -- cgit v1.2.3 From e081e9b9049ff3ffb085cae40d50ebdfb563ef17 Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Mon, 9 Dec 2024 10:54:51 -0800 Subject: RawParseUtils: Default to UTC in invalid timezones PersonIdent used to translate invalid timezones to UTC [1], but the new java.time code just throws an exception. Also the parsing used happen on demand, but now is done in the constructor, so the exception is thrown even if the timezone is not used at all. Check the parsed timezone and default to UTC if it is out of range. [1] https://docs.oracle.com/javase/8/docs/api/java/util/TimeZone.html#getTimeZone-java.lang.String- Change-Id: I90dd7d842ac8f44caef3b76d57375dead76bebde --- .../jgit/util/RawParseUtils_ParsePersonIdentTest.java | 15 +++++++++++++++ .../src/org/eclipse/jgit/util/RawParseUtils.java | 10 ++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/RawParseUtils_ParsePersonIdentTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/RawParseUtils_ParsePersonIdentTest.java index e517889c83..6d23db81d8 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/RawParseUtils_ParsePersonIdentTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/RawParseUtils_ParsePersonIdentTest.java @@ -107,6 +107,21 @@ public class RawParseUtils_ParsePersonIdentTest { assertPersonIdent("Me 1234567890 +8315"); + assertEquals(tooBig.getZoneOffset().getTotalSeconds(), 0); + + PersonIdent tooSmall = RawParseUtils + .parsePersonIdent("Me 1234567890 -8315"); + assertEquals(tooSmall.getZoneOffset().getTotalSeconds(), 0); + + PersonIdent notATime = RawParseUtils + .parsePersonIdent("Me 1234567890 -0370"); + assertEquals(notATime.getZoneOffset().getTotalSeconds(), 0); + } + private static void assertPersonIdent(String line, PersonIdent expected) { PersonIdent actual = RawParseUtils.parsePersonIdent(line); assertEquals(expected, actual); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/RawParseUtils.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/RawParseUtils.java index 1fc695959a..3ed72516c7 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/RawParseUtils.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/RawParseUtils.java @@ -32,6 +32,7 @@ import java.nio.charset.CharsetDecoder; import java.nio.charset.CodingErrorAction; import java.nio.charset.IllegalCharsetNameException; import java.nio.charset.UnsupportedCharsetException; +import java.time.DateTimeException; import java.time.Instant; import java.time.ZoneId; import java.time.ZoneOffset; @@ -473,12 +474,17 @@ public final class RawParseUtils { * @param ptrResult * optional location to return the new ptr value through. If null * the ptr value will be discarded. - * @return the ZoneOffset represention of the timezone offset string + * @return the ZoneOffset represention of the timezone offset string. + * Invalid offsets default to UTC. */ private static ZoneId parseZoneOffset(final byte[] b, int ptr, MutableInteger ptrResult) { int hhmm = parseBase10(b, ptr, ptrResult); - return ZoneOffset.ofHoursMinutes(hhmm / 100, hhmm % 100); + try { + return ZoneOffset.ofHoursMinutes(hhmm / 100, hhmm % 100); + } catch (DateTimeException e) { + return UTC; + } } /** -- cgit v1.2.3 From 76274b62a2cf4da2feb1ac7e9ed6c982d9d3f087 Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Mon, 9 Dec 2024 13:31:44 -0800 Subject: PersonIdent: Default to UTC in timezone parsing The old method "getTimeZone(int tzOffset)" defaults to UTC if the offset is out of range, but the new "getZoneId(int tzOffset)" throws an exception. Return UTC if the offset is invalid, to keep the behavior consistent with older code. Change-Id: Iffe1980b3bd9c05ef2293635a1cbb493144afb79 --- org.eclipse.jgit/src/org/eclipse/jgit/lib/PersonIdent.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/PersonIdent.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/PersonIdent.java index 5d3db9e6ee..f22642c4ce 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/PersonIdent.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/PersonIdent.java @@ -12,7 +12,10 @@ package org.eclipse.jgit.lib; +import static java.time.ZoneOffset.UTC; + import java.io.Serializable; +import java.time.DateTimeException; import java.time.Instant; import java.time.ZoneId; import java.time.ZoneOffset; @@ -54,11 +57,15 @@ public class PersonIdent implements Serializable { * Translate a minutes offset into a ZoneId * * @param tzOffset as minutes east of UTC - * @return a ZoneId for this offset + * @return a ZoneId for this offset (UTC if invalid) * @since 7.1 */ public static ZoneId getZoneId(int tzOffset) { - return ZoneOffset.ofHoursMinutes(tzOffset / 60, tzOffset % 60); + try { + return ZoneOffset.ofHoursMinutes(tzOffset / 60, tzOffset % 60); + } catch (DateTimeException e) { + return UTC; + } } /** -- cgit v1.2.3 From 2b229df06c94fa351ce00227a107835e3b21f5ce Mon Sep 17 00:00:00 2001 From: Antonio Barone Date: Mon, 9 Dec 2024 16:22:18 +0100 Subject: Advertise "agent" capability when using protocol v2 The "agent" capability was not advertised by the server when executing git-upload-pack over protocol v2. This, in turn, prevented the clients from advertising its client agent too, as documented [1]: "The client may optionally send its own agent string by including the agent capability with a value Y (in the form agent=Y) in its request to the server (but it MUST NOT do so if the server did not advertise the agent capability)." When using jgit with Gerrit this had the effect of preventing the logging of the git client agent in the sshd_log. [1] https://git-scm.com/docs/protocol-v2#_agent Bug: jgit-118 Change-Id: Ifb6ea65fde020425920338f7dd9cc683fed6a4a4 --- .../tst/org/eclipse/jgit/transport/UploadPackTest.java | 12 ++++++------ .../src/org/eclipse/jgit/transport/UploadPack.java | 1 + 2 files changed, 7 insertions(+), 6 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/UploadPackTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/UploadPackTest.java index 017104c527..2bd40031cc 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/UploadPackTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/UploadPackTest.java @@ -502,15 +502,15 @@ public class UploadPackTest { assertThat(hook.capabilitiesRequest, notNullValue()); assertThat(pckIn.readString(), is("version 2")); assertThat( - Arrays.asList(pckIn.readString(), pckIn.readString(), - pckIn.readString()), + Arrays.asList(pckIn.readString(),pckIn.readString(), + pckIn.readString(), pckIn.readString()), // TODO(jonathantanmy) This check is written this way // to make it simple to see that we expect this list of // capabilities, but probably should be loosened to // allow additional commands to be added to the list, // and additional capabilities to be added to existing // commands without requiring test changes. - hasItems("ls-refs", "fetch=shallow", "server-option")); + hasItems("agent=" + UserAgent.get() ,"ls-refs", "fetch=shallow", "server-option")); assertTrue(PacketLineIn.isEnd(pckIn.readString())); } @@ -536,7 +536,7 @@ public class UploadPackTest { lines.add(line); } } - assertThat(lines, containsInAnyOrder("ls-refs", "fetch", "server-option")); + assertThat(lines, containsInAnyOrder("ls-refs", "fetch", "server-option", "agent=" + UserAgent.get())); } private void checkUnadvertisedIfUnallowed(String configSection, @@ -643,9 +643,9 @@ public class UploadPackTest { assertThat(pckIn.readString(), is("version 2")); assertThat( - Arrays.asList(pckIn.readString(), pckIn.readString(), + Arrays.asList(pckIn.readString(),pckIn.readString(), pckIn.readString(), pckIn.readString()), - hasItems("ls-refs", "fetch=shallow", "server-option")); + hasItems("agent="+ UserAgent.get(),"ls-refs", "fetch=shallow", "server-option")); assertTrue(PacketLineIn.isEnd(pckIn.readString())); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java index 968728de8d..fbea7044f3 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java @@ -1441,6 +1441,7 @@ public class UploadPack implements Closeable { if (transferConfig.isAdvertiseObjectInfo()) { caps.add(COMMAND_OBJECT_INFO); } + caps.add(OPTION_AGENT + "=" + UserAgent.get()); return caps; } -- cgit v1.2.3 From e5bb59bc15f09651ffe0fff95dfe5ca5c8470619 Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Tue, 10 Dec 2024 13:15:22 -0800 Subject: FileSnapshot: Delete malformed "catch" line (merge artifact?) The merge of this file left an empty catch block: error: [EmptyCatch] Caught exceptions should not be ignored Delete the catch, as the exception is a subclass of the FileSystemException catched just under it. Change-Id: I78d6b1ca8152b6eee50a69d24ca987868866ba06 --- .../src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java | 2 -- 1 file changed, 2 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java index 8f83a37efc..3eea16b227 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java @@ -16,7 +16,6 @@ import static org.eclipse.jgit.util.FS.FileStoreAttributes.FALLBACK_TIMESTAMP_RE import java.io.File; import java.io.IOException; import java.nio.file.FileSystemException; -import java.nio.file.NoSuchFileException; import java.nio.file.attribute.BasicFileAttributes; import java.time.Duration; import java.time.Instant; @@ -547,7 +546,6 @@ public class FileSnapshot { private static BasicFileAttributes getFileAttributes(File path) throws NoSuchElementException { try { return FS.DETECTED.fileAttributes(path); - } catch (NoSuchFileException e) { } catch (FileSystemException e) { if (!e.getMessage().endsWith("Not a directory")) { LOG.error(e.getMessage(), e); -- cgit v1.2.3 From d2ff398bcbd0177200651f684920b513a923ab82 Mon Sep 17 00:00:00 2001 From: Nasser Grainawi Date: Tue, 10 Dec 2024 17:36:26 -0700 Subject: Fix potential NPE in TreeWalk#getFilterCommandDefinition Change-Id: Ic6f0fd26153b47b4aeeec105bac431225d9bf8bf --- org.eclipse.jgit/src/org/eclipse/jgit/treewalk/TreeWalk.java | 3 +++ 1 file changed, 3 insertions(+) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/TreeWalk.java b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/TreeWalk.java index aaac2a72e0..b789c5120e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/TreeWalk.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/TreeWalk.java @@ -1587,6 +1587,9 @@ public class TreeWalk implements AutoCloseable, AttributesProvider { */ private String getFilterCommandDefinition(String filterDriverName, String filterCommandType) { + if (config == null) { + return null; + } String key = filterDriverName + "." + filterCommandType; //$NON-NLS-1$ String filterCommand = filterCommandsByNameDotType.get(key); if (filterCommand != null) -- cgit v1.2.3 From df7810957c00f3d857c006ff62ea7dd0e5b6fb76 Mon Sep 17 00:00:00 2001 From: Simon Eder Date: Wed, 31 Jul 2024 16:21:46 +0200 Subject: Submodules: use relative paths for worktree and gitdir Currently absolute paths are used for a submodules 'worktree' and 'gitdir'. This has drawbacks if a git repository containing submodules is copied to a different location. Enable using relative paths when creating a file-based repository. Add optional relative path support to the clone and init commands, and use it in the submodule commands to generate relative paths. The new implementation mimics the cgit behavior which also uses relative paths instead of absolute ones. Bug: jgit-78 Change-Id: I31c5a938d5cbc030d273fc649c94ee0c90b4ce01 --- .../tst/org/eclipse/jgit/api/CloneCommandTest.java | 11 +++--- .../eclipse/jgit/submodule/SubmoduleAddTest.java | 7 ++++ .../jgit/submodule/SubmoduleUpdateTest.java | 8 ++++ .../src/org/eclipse/jgit/api/CloneCommand.java | 17 +++++++++ .../src/org/eclipse/jgit/api/InitCommand.java | 23 ++++++++++- .../org/eclipse/jgit/api/SubmoduleAddCommand.java | 5 ++- .../eclipse/jgit/api/SubmoduleUpdateCommand.java | 7 ++-- .../jgit/internal/storage/file/FileRepository.java | 44 ++++++++++++++++++++-- 8 files changed, 107 insertions(+), 15 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CloneCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CloneCommandTest.java index 63ab8094ae..db5b27c2ab 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CloneCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CloneCommandTest.java @@ -647,7 +647,8 @@ public class CloneCommandTest extends RepositoryTestCase { new File(git.getRepository().getWorkTree(), walk.getPath()), subRepo.getWorkTree()); assertEquals(new File(new File(git.getRepository().getDirectory(), - "modules"), walk.getPath()), subRepo.getDirectory()); + "modules"), walk.getPath()).getCanonicalPath(), + subRepo.getDirectory().getCanonicalPath()); } File directory = createTempDirectory("testCloneRepositoryWithSubmodules"); @@ -681,8 +682,8 @@ public class CloneCommandTest extends RepositoryTestCase { walk.getPath()), clonedSub1.getWorkTree()); assertEquals( new File(new File(git2.getRepository().getDirectory(), - "modules"), walk.getPath()), - clonedSub1.getDirectory()); + "modules"), walk.getPath()).getCanonicalPath(), + clonedSub1.getDirectory().getCanonicalPath()); } } @@ -770,8 +771,8 @@ public class CloneCommandTest extends RepositoryTestCase { walk.getPath()), clonedSub1.getWorkTree()); assertEquals( new File(new File(git2.getRepository().getDirectory(), - "modules"), walk.getPath()), - clonedSub1.getDirectory()); + "modules"), walk.getPath()).getCanonicalPath(), + clonedSub1.getDirectory().getCanonicalPath()); status = new SubmoduleStatusCommand(clonedSub1); statuses = status.call(); } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/submodule/SubmoduleAddTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/submodule/SubmoduleAddTest.java index 300c869b78..4306975f7b 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/submodule/SubmoduleAddTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/submodule/SubmoduleAddTest.java @@ -114,6 +114,13 @@ public class SubmoduleAddTest extends RepositoryTestCase { try (Repository subModRepo = generator.getRepository()) { assertNotNull(subModRepo); assertEquals(subCommit, commit); + String worktreeDir = subModRepo.getConfig().getString( + ConfigConstants.CONFIG_CORE_SECTION, null, + ConfigConstants.CONFIG_KEY_WORKTREE); + assertEquals("../../../sub", worktreeDir); + String gitdir = read(new File(subModRepo.getWorkTree(), + Constants.DOT_GIT)); + assertEquals("gitdir: ../.git/modules/sub", gitdir); } } Status status = Git.wrap(db).status().call(); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/submodule/SubmoduleUpdateTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/submodule/SubmoduleUpdateTest.java index b10bd73208..c5e9c2deaa 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/submodule/SubmoduleUpdateTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/submodule/SubmoduleUpdateTest.java @@ -91,6 +91,14 @@ public class SubmoduleUpdateTest extends RepositoryTestCase { try (Repository subRepo = generator.getRepository()) { assertNotNull(subRepo); assertEquals(commit, subRepo.resolve(Constants.HEAD)); + String worktreeDir = subRepo.getConfig().getString( + ConfigConstants.CONFIG_CORE_SECTION, null, + ConfigConstants.CONFIG_KEY_WORKTREE); + assertEquals("../../../sub", worktreeDir); + String gitdir = read( + new File(subRepo.getWorkTree(), Constants.DOT_GIT)); + assertEquals("gitdir: ../.git/modules/sub", gitdir); + } } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/CloneCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/CloneCommand.java index 3e034f1a6a..4a536b9534 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/CloneCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/CloneCommand.java @@ -67,6 +67,8 @@ public class CloneCommand extends TransportCommand { private boolean bare; + private boolean relativePaths; + private FS fs; private String remote = Constants.DEFAULT_REMOTE_NAME; @@ -264,6 +266,7 @@ public class CloneCommand extends TransportCommand { private Repository init() throws GitAPIException { InitCommand command = Git.init(); command.setBare(bare); + command.setRelativeDirs(relativePaths); if (fs != null) { command.setFs(fs); } @@ -554,6 +557,20 @@ public class CloneCommand extends TransportCommand { return this; } + /** + * Set whether the cloned repository shall use relative paths for GIT_DIR + * and GIT_WORK_TREE + * + * @param relativePaths + * if true, use relative paths for GIT_DIR and GIT_WORK_TREE + * @return this instance + * @since 7.2 + */ + public CloneCommand setRelativePaths(boolean relativePaths) { + this.relativePaths = relativePaths; + return this; + } + /** * Set the file system abstraction to be used for repositories created by * this command. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/InitCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/InitCommand.java index 240290f4f9..1da71aa6eb 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/InitCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/InitCommand.java @@ -19,6 +19,7 @@ import org.eclipse.jgit.api.errors.InvalidRefNameException; import org.eclipse.jgit.api.errors.JGitInternalException; import org.eclipse.jgit.errors.ConfigInvalidException; import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.internal.storage.file.FileRepository; import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.Repository; @@ -44,6 +45,8 @@ public class InitCommand implements Callable { private String initialBranch; + private boolean relativePaths; + /** * {@inheritDoc} *

@@ -100,7 +103,11 @@ public class InitCommand implements Callable { : initialBranch); Repository repository = builder.build(); if (!repository.getObjectDatabase().exists()) - repository.create(bare); + if (repository instanceof FileRepository) { + ((FileRepository) repository).create(bare, relativePaths); + } else { + repository.create(bare); + } return new Git(repository, true); } catch (IOException | ConfigInvalidException e) { throw new JGitInternalException(e.getMessage(), e); @@ -214,4 +221,18 @@ public class InitCommand implements Callable { this.initialBranch = branch; return this; } + + /** + * * Set whether the repository shall use relative paths for GIT_DIR and + * GIT_WORK_TREE + * + * @param relativePaths + * if true, use relative paths for GIT_DIR and GIT_WORK_TREE + * @return {@code this} + * @since 7.2 + */ + public InitCommand setRelativeDirs(boolean relativePaths) { + this.relativePaths = relativePaths; + return this; + } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleAddCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleAddCommand.java index 401f069e4e..5105dfc2e5 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleAddCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleAddCommand.java @@ -176,8 +176,9 @@ public class SubmoduleAddCommand extends CloneCommand clone = Git.cloneRepository(); configure(clone); clone.setDirectory(moduleDirectory); - clone.setGitDir(new File(new File(repo.getCommonDirectory(), - Constants.MODULES), path)); + clone.setGitDir(new File( + new File(repo.getCommonDirectory(), Constants.MODULES), path)); + clone.setRelativePaths(true); clone.setURI(resolvedUri); if (monitor != null) clone.setProgressMonitor(monitor); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleUpdateCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleUpdateCommand.java index 751dabcd6b..3524984347 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleUpdateCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleUpdateCommand.java @@ -127,9 +127,10 @@ public class SubmoduleUpdateCommand extends configure(clone); clone.setURI(url); clone.setDirectory(generator.getDirectory()); - clone.setGitDir( - new File(new File(repo.getCommonDirectory(), Constants.MODULES), - generator.getPath())); + clone.setGitDir(new File( + new File(repo.getCommonDirectory(), Constants.MODULES), + generator.getPath())); + clone.setRelativePaths(true); if (monitor != null) { clone.setProgressMonitor(monitor); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java index 84c85659ff..c5c36565d9 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java @@ -2,7 +2,7 @@ * Copyright (C) 2007, Dave Watson * Copyright (C) 2008-2010, Google Inc. * Copyright (C) 2006-2010, Robin Rosenberg - * Copyright (C) 2006-2008, Shawn O. Pearce and others + * Copyright (C) 2006-2024, Shawn O. Pearce and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -215,6 +215,16 @@ public class FileRepository extends Repository { } } + private String getRelativeDir(File base, File other) { + File relPath; + try { + relPath = base.toPath().relativize(other.toPath()).toFile(); + } catch (IllegalArgumentException e) { + relPath = other; + } + return FileUtils.pathToString(relPath); + } + /** * {@inheritDoc} *

@@ -223,6 +233,22 @@ public class FileRepository extends Repository { */ @Override public void create(boolean bare) throws IOException { + create(bare, false); + } + + /** + * Create a new Git repository initializing the necessary files and + * directories. + * + * @param bare + * if true, a bare repository (a repository without a working + * directory) is created. + * @param relativePaths + * if true, relative paths are used for GIT_DIR and GIT_WORK_TREE + * @throws IOException + * in case of IO problem + */ + public void create(boolean bare, boolean relativePaths) throws IOException { final FileBasedConfig cfg = getConfig(); if (cfg.getFile().exists()) { throw new IllegalStateException(MessageFormat.format( @@ -293,15 +319,25 @@ public class FileRepository extends Repository { if (!bare) { File workTree = getWorkTree(); if (!getDirectory().getParentFile().equals(workTree)) { + String workTreePath; + String gitDirPath; + if (relativePaths) { + File canonGitDir = getDirectory().getCanonicalFile(); + File canonWorkTree = getWorkTree().getCanonicalFile(); + workTreePath = getRelativeDir(canonGitDir, canonWorkTree); + gitDirPath = getRelativeDir(canonWorkTree, canonGitDir); + } else { + workTreePath = getWorkTree().getAbsolutePath(); + gitDirPath = getDirectory().getAbsolutePath(); + } cfg.setString(ConfigConstants.CONFIG_CORE_SECTION, null, - ConfigConstants.CONFIG_KEY_WORKTREE, getWorkTree() - .getAbsolutePath()); + ConfigConstants.CONFIG_KEY_WORKTREE, workTreePath); LockFile dotGitLockFile = new LockFile(new File(workTree, Constants.DOT_GIT)); try { if (dotGitLockFile.lock()) { dotGitLockFile.write(Constants.encode(Constants.GITDIR - + getDirectory().getAbsolutePath())); + + gitDirPath)); dotGitLockFile.commit(); } } finally { -- cgit v1.2.3 From e9094fffdb9d5d498cf75584267d1fd7e3690374 Mon Sep 17 00:00:00 2001 From: Thomas Wolf Date: Thu, 5 Dec 2024 20:19:17 +0100 Subject: RevertCommand: use only first line in revert commit message C git uses only the first line of the title paragraph of the reverted commit to build the title of the revert commit. Align the JGit behavior with that. Since git 2.43.0, a revert of a revert uses a title "Reapply "xxx"" instead of "Revert "Revert "xxx""".[1] This is _not_ implemented in this change. [1] https://github.com/git/git/commit/883cb1b8f86d Bug: jgit-117 Change-Id: I030092c6b9447bb738e6d761af5ce50df58cc6d3 --- .../org/eclipse/jgit/api/RevertCommandTest.java | 9 ++++--- .../eclipse/jgit/revwalk/RevCommitParseTest.java | 11 ++++++++- .../src/org/eclipse/jgit/api/RevertCommand.java | 6 ++--- .../src/org/eclipse/jgit/revwalk/RevCommit.java | 28 ++++++++++++++++++++-- 4 files changed, 45 insertions(+), 9 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/RevertCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/RevertCommandTest.java index 4ebe994ef7..afd6708d21 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/RevertCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/RevertCommandTest.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2011, Robin Rosenberg and others + * Copyright (C) 2011, 2024 Robin Rosenberg and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -59,7 +59,9 @@ public class RevertCommandTest extends RepositoryTestCase { writeTrashFile("a", "first line\nsecond line\nthird line\nfourth line\n"); git.add().addFilepattern("a").call(); - RevCommit fixingA = git.commit().setMessage("fixed a").call(); + // Commit message with a non-empty second line on purpose + RevCommit fixingA = git.commit().setMessage("fixed a\nsecond line") + .call(); writeTrashFile("b", "first line\n"); git.add().addFilepattern("b").call(); @@ -78,7 +80,8 @@ public class RevertCommandTest extends RepositoryTestCase { + "This reverts commit " + fixingA.getId().getName() + ".\n"; assertEquals(expectedMessage, revertCommit.getFullMessage()); assertEquals("fixed b", history.next().getFullMessage()); - assertEquals("fixed a", history.next().getFullMessage()); + assertEquals("fixed a\nsecond line", + history.next().getFullMessage()); assertEquals("enlarged a", history.next().getFullMessage()); assertEquals("create b", history.next().getFullMessage()); assertEquals("create a", history.next().getFullMessage()); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/RevCommitParseTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/RevCommitParseTest.java index 6872289a8b..2955516af0 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/RevCommitParseTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/RevCommitParseTest.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2008-2009, Google Inc. and others + * Copyright (C) 2008, 2024 Google Inc. and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -408,6 +408,7 @@ public class RevCommitParseTest extends RepositoryTestCase { final RevCommit c = create(msg); assertEquals(msg, c.getFullMessage()); assertEquals(msg, c.getShortMessage()); + assertEquals(msg, c.getFirstMessageLine()); } @Test @@ -415,6 +416,7 @@ public class RevCommitParseTest extends RepositoryTestCase { final RevCommit c = create("\n"); assertEquals("\n", c.getFullMessage()); assertEquals("", c.getShortMessage()); + assertEquals("", c.getFirstMessageLine()); } @Test @@ -423,6 +425,7 @@ public class RevCommitParseTest extends RepositoryTestCase { final RevCommit c = create(shortMsg); assertEquals(shortMsg, c.getFullMessage()); assertEquals(shortMsg, c.getShortMessage()); + assertEquals(shortMsg, c.getFirstMessageLine()); } @Test @@ -432,6 +435,7 @@ public class RevCommitParseTest extends RepositoryTestCase { final RevCommit c = create(fullMsg); assertEquals(fullMsg, c.getFullMessage()); assertEquals(shortMsg, c.getShortMessage()); + assertEquals(shortMsg, c.getFirstMessageLine()); } @Test @@ -441,6 +445,7 @@ public class RevCommitParseTest extends RepositoryTestCase { final RevCommit c = create(fullMsg); assertEquals(fullMsg, c.getFullMessage()); assertEquals(shortMsg, c.getShortMessage()); + assertEquals("This is a", c.getFirstMessageLine()); } @Test @@ -450,6 +455,7 @@ public class RevCommitParseTest extends RepositoryTestCase { final RevCommit c = create(fullMsg); assertEquals(fullMsg, c.getFullMessage()); assertEquals(shortMsg, c.getShortMessage()); + assertEquals("This is a", c.getFirstMessageLine()); } @Test @@ -461,6 +467,7 @@ public class RevCommitParseTest extends RepositoryTestCase { final RevCommit c = create(fullMsg); assertEquals(fullMsg, c.getFullMessage()); assertEquals(shortMsg, c.getShortMessage()); + assertEquals(shortMsg, c.getFirstMessageLine()); } @Test @@ -480,6 +487,7 @@ public class RevCommitParseTest extends RepositoryTestCase { assertEquals(author, p.getAuthorIdent()); assertEquals(committer, p.getCommitterIdent()); assertEquals("Test commit", p.getShortMessage()); + assertEquals("Test commit", p.getFirstMessageLine()); assertEquals(src.getMessage(), p.getFullMessage()); } @@ -494,6 +502,7 @@ public class RevCommitParseTest extends RepositoryTestCase { final RevCommit c = create(fullMsg); assertEquals(fullMsg, c.getFullMessage()); assertEquals(shortMsg, c.getShortMessage()); + assertEquals("This fixes a", c.getFirstMessageLine()); } private static ObjectId id(String str) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/RevertCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/RevertCommand.java index 855c3b1cf3..6643c83662 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/RevertCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/RevertCommand.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010, Christian Halstrick and others + * Copyright (C) 2010, 2024 Christian Halstrick and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -143,8 +143,8 @@ public class RevertCommand extends GitCommand { merger.setCommitNames(new String[] { "BASE", ourName, revertName }); //$NON-NLS-1$ - String shortMessage = "Revert \"" + srcCommit.getShortMessage() //$NON-NLS-1$ - + "\""; //$NON-NLS-1$ + String shortMessage = "Revert \"" //$NON-NLS-1$ + + srcCommit.getFirstMessageLine() + '"'; String newMessage = shortMessage + "\n\n" //$NON-NLS-1$ + "This reverts commit " + srcCommit.getId().getName() //$NON-NLS-1$ + ".\n"; //$NON-NLS-1$ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevCommit.java b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevCommit.java index 743a8ccce0..55ddebf288 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevCommit.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevCommit.java @@ -1,6 +1,6 @@ /* - * Copyright (C) 2008-2009, Google Inc. - * Copyright (C) 2008, Shawn O. Pearce and others + * Copyright (C) 2008, 2009 Google Inc. + * Copyright (C) 2008, 2024 Shawn O. Pearce and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -523,6 +523,30 @@ public class RevCommit extends RevObject { return false; } + /** + * Parse the commit message and return its first line, i.e., everything up + * to but not including the first newline, if any. + * + * @return the first line of the decoded commit message as a string; never + * {@code null}. + * @since 7.2 + */ + public final String getFirstMessageLine() { + int msgB = RawParseUtils.commitMessage(buffer, 0); + if (msgB < 0) { + return ""; //$NON-NLS-1$ + } + int msgE = msgB; + byte[] raw = buffer; + while (msgE < raw.length && raw[msgE] != '\n') { + msgE++; + } + if (msgE > msgB && msgE > 0 && raw[msgE - 1] == '\r') { + msgE--; + } + return RawParseUtils.decode(guessEncoding(buffer), buffer, msgB, msgE); + } + /** * Determine the encoding of the commit message buffer. *

-- cgit v1.2.3 From ae53d63837857d886cdf15442118597be717ff33 Mon Sep 17 00:00:00 2001 From: Martin Fick Date: Wed, 11 Dec 2024 17:00:17 -0800 Subject: Pack: fix threading bug getting idx When converting to Optionally, a threading bug was introduced, fix it. The Optionally class itself is not thread safe, and previously it was being called from idx() without any thread synchronization mechanism. This was because previously the variable which held the cached index was volatile. The Optionally kept the volatile attribute, but that only synchronizes the reference to the Optionally, and not the element inside the Optionally. A clear() from another thread could thus be missed by the idx() call, potentially allowing the idx() call to interact poorly with the Optionally, potentially even causing a memory leak. To fix this, extend the synchronized inside the idx() method to the entire method. Then, additionally remove the now redundant Optional fetch in idx() and the no longer needed volatile. Change-Id: I6077e1aaed96c53200805b4c87a67afb49c2b373 Signed-off-by: Martin Fick --- .../eclipse/jgit/internal/storage/file/Pack.java | 86 ++++++++++------------ 1 file changed, 40 insertions(+), 46 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java index 61577a956c..8d2a86386f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java @@ -119,7 +119,7 @@ public class Pack implements Iterable { private byte[] packChecksum; - private volatile Optionally loadedIdx = Optionally.empty(); + private Optionally loadedIdx = Optionally.empty(); private Optionally reverseIdx = Optionally.empty(); @@ -159,60 +159,54 @@ public class Pack implements Iterable { length = Long.MAX_VALUE; } - private PackIndex idx() throws IOException { + private synchronized PackIndex idx() throws IOException { Optional optional = loadedIdx.getOptional(); if (optional.isPresent()) { return optional.get(); } - synchronized (this) { - optional = loadedIdx.getOptional(); - if (optional.isPresent()) { - return optional.get(); - } - if (invalid) { - throw new PackInvalidException(packFile, invalidatingCause); + if (invalid) { + throw new PackInvalidException(packFile, invalidatingCause); + } + try { + long start = System.currentTimeMillis(); + PackFile idxFile = packFile.create(INDEX); + PackIndex idx = PackIndex.open(idxFile); + if (LOG.isDebugEnabled()) { + LOG.debug(String.format( + "Opening pack index %s, size %.3f MB took %d ms", //$NON-NLS-1$ + idxFile.getAbsolutePath(), + Float.valueOf(idxFile.length() + / (1024f * 1024)), + Long.valueOf(System.currentTimeMillis() + - start))); } - try { - long start = System.currentTimeMillis(); - PackFile idxFile = packFile.create(INDEX); - PackIndex idx = PackIndex.open(idxFile); - if (LOG.isDebugEnabled()) { - LOG.debug(String.format( - "Opening pack index %s, size %.3f MB took %d ms", //$NON-NLS-1$ - idxFile.getAbsolutePath(), - Float.valueOf(idxFile.length() - / (1024f * 1024)), - Long.valueOf(System.currentTimeMillis() - - start))); - } - if (packChecksum == null) { - packChecksum = idx.getChecksum(); - fileSnapshot.setChecksum( - ObjectId.fromRaw(packChecksum)); - } else if (!Arrays.equals(packChecksum, - idx.getChecksum())) { - throw new PackMismatchException(MessageFormat - .format(JGitText.get().packChecksumMismatch, - packFile.getPath(), - PackExt.PACK.getExtension(), - Hex.toHexString(packChecksum), - PackExt.INDEX.getExtension(), - Hex.toHexString(idx.getChecksum()))); - } - loadedIdx = optionally(idx); - return idx; - } catch (InterruptedIOException e) { - // don't invalidate the pack, we are interrupted from - // another thread - throw e; - } catch (IOException e) { - invalid = true; - invalidatingCause = e; - throw e; + packChecksum = idx.getChecksum(); + fileSnapshot.setChecksum( + ObjectId.fromRaw(packChecksum)); + } else if (!Arrays.equals(packChecksum, + idx.getChecksum())) { + throw new PackMismatchException(MessageFormat + .format(JGitText.get().packChecksumMismatch, + packFile.getPath(), + PackExt.PACK.getExtension(), + Hex.toHexString(packChecksum), + PackExt.INDEX.getExtension(), + Hex.toHexString(idx.getChecksum()))); } + loadedIdx = optionally(idx); + return idx; + } catch (InterruptedIOException e) { + // don't invalidate the pack, we are interrupted from + // another thread + throw e; + } catch (IOException e) { + invalid = true; + invalidatingCause = e; + throw e; } } + /** * Get the File object which locates this pack on disk. * -- cgit v1.2.3 From cdbea5ea96f7e473a48e6bff15e84ecccd521800 Mon Sep 17 00:00:00 2001 From: Martin Fick Date: Wed, 11 Dec 2024 17:26:57 -0800 Subject: Optionally.Hard: avoid Optional creation on every use, Previously the getOptional() call created an new Optional for its element, and this appears to be somewhat expensive. During a time when a server is heavily loaded because of a poorly maintained repository with potentially 2K+ pack files, calls to Optionally.ofNullable() from within the getOptional() method appeared extensively in the Stacktrace. Reduce the getOptional() call overhead by storing an already created Optional of the element instead of the element itself. This trades the extra space of one extra reference for a potential speed gain in a hotspot. Since the current users of Optionally.Hard reference objects significantly larger than a single reference (and most users are likely to be, else why would they be using an Optionally?), this is likely a good tradeoff. Change-Id: I7adccc915480cbb97a43dcbe074cfb620888c936 Signed-off-by: Martin Fick --- .../src/org/eclipse/jgit/internal/util/Optionally.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/util/Optionally.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/util/Optionally.java index 3447f669ab..270b760562 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/util/Optionally.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/util/Optionally.java @@ -53,24 +53,24 @@ public interface Optionally { /** * The mutable optional object */ - protected T element; + protected Optional optional; /** * @param element * the mutable optional object */ public Hard(T element) { - this.element = element; + optional = Optional.ofNullable(element); } @Override public void clear() { - element = null; + optional = Optional.empty(); } @Override public Optional getOptional() { - return Optional.ofNullable(element); + return optional; } } -- cgit v1.2.3 From fef56c3c81cce48819fdc8247203717d87ba17c9 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Sun, 15 Dec 2024 20:51:34 +0100 Subject: FileSnapshot: fix warnings - comment empty code block - suppress non-translatable text warning Change-Id: Id49b4a56bbe5454edfe1ea8b79ceeaf51ceac370 --- .../src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java index d752e315db..7f34aed902 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileSnapshot.java @@ -589,9 +589,10 @@ public class FileSnapshot { } } } catch (NoSuchFileException e) { + // ignore } catch (FileSystemException e) { String msg = e.getMessage(); - if (!msg.endsWith("Not a directory")) { + if (!msg.endsWith("Not a directory")) { //$NON-NLS-1$ LOG.error(msg, e); } } catch (IOException e) { -- cgit v1.2.3 From 937cfdc765c2c523013c00eff7e14cf3c88e94d1 Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Fri, 13 Dec 2024 09:53:17 -0800 Subject: CommitTimeRevFilter: Move to java.time API Deprecate the methods using Date and offer the equivalent as Instant. Keep an int with seconds as internal representation, as it seems more efficient to compare than Instant.before/after. Change-Id: Ie751ab5661c7dafaab58a16c219143b78092f84a --- .../eclipse/jgit/revwalk/RevWalkFilterTest.java | 123 ++++++++++++++++++++- .../org/eclipse/jgit/revwalk/RevWalkTestCase.java | 7 ++ .../jgit/revwalk/filter/CommitTimeRevFilter.java | 83 +++++++++++--- 3 files changed, 194 insertions(+), 19 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/RevWalkFilterTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/RevWalkFilterTest.java index 81ff4a2f92..7fece66bf0 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/RevWalkFilterTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/RevWalkFilterTest.java @@ -14,6 +14,7 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import java.io.IOException; +import java.time.Instant; import java.util.Date; import org.eclipse.jgit.errors.IncorrectObjectTypeException; @@ -217,14 +218,132 @@ public class RevWalkFilterTest extends RevWalkTestCase { final RevCommit b = commit(a); tick(100); - Date since = getDate(); + Instant since = getInstant(); final RevCommit c1 = commit(b); tick(100); final RevCommit c2 = commit(b); tick(100); - Date until = getDate(); + Instant until = getInstant(); + final RevCommit d = commit(c1, c2); + tick(100); + + final RevCommit e = commit(d); + + { + RevFilter after = CommitTimeRevFilter.after(since); + assertNotNull(after); + rw.setRevFilter(after); + markStart(e); + assertCommit(e, rw.next()); + assertCommit(d, rw.next()); + assertCommit(c2, rw.next()); + assertCommit(c1, rw.next()); + assertNull(rw.next()); + } + + { + RevFilter before = CommitTimeRevFilter.before(until); + assertNotNull(before); + rw.reset(); + rw.setRevFilter(before); + markStart(e); + assertCommit(c2, rw.next()); + assertCommit(c1, rw.next()); + assertCommit(b, rw.next()); + assertCommit(a, rw.next()); + assertNull(rw.next()); + } + + { + RevFilter between = CommitTimeRevFilter.between(since, until); + assertNotNull(between); + rw.reset(); + rw.setRevFilter(between); + markStart(e); + assertCommit(c2, rw.next()); + assertCommit(c1, rw.next()); + assertNull(rw.next()); + } + } + + @Test + public void testCommitTimeRevFilter_date() throws Exception { + // Using deprecated Date api for the commit time rev filter. + // Delete this tests when method is removed. + final RevCommit a = commit(); + tick(100); + + final RevCommit b = commit(a); + tick(100); + + Date since = Date.from(getInstant()); + final RevCommit c1 = commit(b); + tick(100); + + final RevCommit c2 = commit(b); + tick(100); + + Date until = Date.from(getInstant()); + final RevCommit d = commit(c1, c2); + tick(100); + + final RevCommit e = commit(d); + + { + RevFilter after = CommitTimeRevFilter.after(since); + assertNotNull(after); + rw.setRevFilter(after); + markStart(e); + assertCommit(e, rw.next()); + assertCommit(d, rw.next()); + assertCommit(c2, rw.next()); + assertCommit(c1, rw.next()); + assertNull(rw.next()); + } + + { + RevFilter before = CommitTimeRevFilter.before(until); + assertNotNull(before); + rw.reset(); + rw.setRevFilter(before); + markStart(e); + assertCommit(c2, rw.next()); + assertCommit(c1, rw.next()); + assertCommit(b, rw.next()); + assertCommit(a, rw.next()); + assertNull(rw.next()); + } + + { + RevFilter between = CommitTimeRevFilter.between(since, until); + assertNotNull(between); + rw.reset(); + rw.setRevFilter(between); + markStart(e); + assertCommit(c2, rw.next()); + assertCommit(c1, rw.next()); + assertNull(rw.next()); + } + } + + @Test + public void testCommitTimeRevFilter_long() throws Exception { + final RevCommit a = commit(); + tick(100); + + final RevCommit b = commit(a); + tick(100); + + long since = getInstant().toEpochMilli(); + final RevCommit c1 = commit(b); + tick(100); + + final RevCommit c2 = commit(b); + tick(100); + + long until = getInstant().toEpochMilli(); final RevCommit d = commit(c1, c2); tick(100); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/RevWalkTestCase.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/RevWalkTestCase.java index b37a19dd7a..8fa6a83670 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/RevWalkTestCase.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/RevWalkTestCase.java @@ -12,6 +12,7 @@ package org.eclipse.jgit.revwalk; import static org.junit.Assert.assertSame; +import java.time.Instant; import java.util.Date; import org.eclipse.jgit.dircache.DirCacheEntry; @@ -38,10 +39,16 @@ public abstract class RevWalkTestCase extends RepositoryTestCase { return new RevWalk(db); } + // Use getInstant() instead + @Deprecated protected Date getDate() { return Date.from(util.getInstant()); } + protected Instant getInstant() { + return util.getInstant(); + } + protected void tick(int secDelta) { util.tick(secDelta); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/filter/CommitTimeRevFilter.java b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/filter/CommitTimeRevFilter.java index 4100e877df..40f56e38ba 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/filter/CommitTimeRevFilter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/filter/CommitTimeRevFilter.java @@ -12,6 +12,7 @@ package org.eclipse.jgit.revwalk.filter; import java.io.IOException; +import java.time.Instant; import java.util.Date; import org.eclipse.jgit.errors.IncorrectObjectTypeException; @@ -30,9 +31,23 @@ public abstract class CommitTimeRevFilter extends RevFilter { * @param ts * the point in time to cut on. * @return a new filter to select commits on or before ts. + * + * @deprecated Use {@link #before(Instant)} instead. */ + @Deprecated(since="7.2") public static final RevFilter before(Date ts) { - return before(ts.getTime()); + return before(ts.toInstant()); + } + + /** + * Create a new filter to select commits before a given date/time. + * + * @param ts + * the point in time to cut on. + * @return a new filter to select commits on or before ts. + */ + public static RevFilter before(Instant ts) { + return new Before(ts); } /** @@ -43,7 +58,7 @@ public abstract class CommitTimeRevFilter extends RevFilter { * @return a new filter to select commits on or before ts. */ public static final RevFilter before(long ts) { - return new Before(ts); + return new Before(Instant.ofEpochMilli(ts)); } /** @@ -52,9 +67,23 @@ public abstract class CommitTimeRevFilter extends RevFilter { * @param ts * the point in time to cut on. * @return a new filter to select commits on or after ts. + * + * @deprecated Use {@link #after(Instant)} instead. */ + @Deprecated(since="7.2") public static final RevFilter after(Date ts) { - return after(ts.getTime()); + return after(ts.toInstant()); + } + + /** + * Create a new filter to select commits after a given date/time. + * + * @param ts + * the point in time to cut on. + * @return a new filter to select commits on or after ts. + */ + public static RevFilter after(Instant ts) { + return new After(ts); } /** @@ -65,7 +94,7 @@ public abstract class CommitTimeRevFilter extends RevFilter { * @return a new filter to select commits on or after ts. */ public static final RevFilter after(long ts) { - return new After(ts); + return after(Instant.ofEpochMilli(ts)); } /** @@ -75,9 +104,24 @@ public abstract class CommitTimeRevFilter extends RevFilter { * @param since the point in time to cut on. * @param until the point in time to cut off. * @return a new filter to select commits between the given date/times. + * + * @deprecated Use {@link #between(Instant, Instant)} instead. */ + @Deprecated(since="7.2") public static final RevFilter between(Date since, Date until) { - return between(since.getTime(), until.getTime()); + return between(since.toInstant(), until.toInstant()); + } + + /** + * Create a new filter to select commits after or equal a given date/time since + * and before or equal a given date/time until. + * + * @param since the point in time to cut on. + * @param until the point in time to cut off. + * @return a new filter to select commits between the given date/times. + */ + public static RevFilter between(Instant since, Instant until) { + return new Between(since, until); } /** @@ -87,9 +131,12 @@ public abstract class CommitTimeRevFilter extends RevFilter { * @param since the point in time to cut on, in milliseconds. * @param until the point in time to cut off, in millisconds. * @return a new filter to select commits between the given date/times. + * + * @deprecated Use {@link #between(Instant, Instant)} instead. */ + @Deprecated(since="7.2") public static final RevFilter between(long since, long until) { - return new Between(since, until); + return new Between(Instant.ofEpochMilli(since), Instant.ofEpochMilli(until)); } final int when; @@ -98,6 +145,10 @@ public abstract class CommitTimeRevFilter extends RevFilter { when = (int) (ts / 1000); } + CommitTimeRevFilter(Instant t) { + when = (int) t.getEpochSecond(); + } + @Override public RevFilter clone() { return this; @@ -109,8 +160,8 @@ public abstract class CommitTimeRevFilter extends RevFilter { } private static class Before extends CommitTimeRevFilter { - Before(long ts) { - super(ts); + Before(Instant t) { + super(t); } @Override @@ -123,14 +174,12 @@ public abstract class CommitTimeRevFilter extends RevFilter { @SuppressWarnings("nls") @Override public String toString() { - return super.toString() + "(" + new Date(when * 1000L) + ")"; + return super.toString() + "(" + Instant.ofEpochSecond(when) + ")"; } } private static class After extends CommitTimeRevFilter { - After(long ts) { - super(ts); - } + After(Instant t) { super(t); } @Override public boolean include(RevWalk walker, RevCommit cmit) @@ -148,16 +197,16 @@ public abstract class CommitTimeRevFilter extends RevFilter { @SuppressWarnings("nls") @Override public String toString() { - return super.toString() + "(" + new Date(when * 1000L) + ")"; + return super.toString() + "(" + Instant.ofEpochSecond(when) + ")"; } } private static class Between extends CommitTimeRevFilter { private final int until; - Between(long since, long until) { + Between(Instant since, Instant until) { super(since); - this.until = (int) (until / 1000); + this.until = (int) until.getEpochSecond(); } @Override @@ -170,8 +219,8 @@ public abstract class CommitTimeRevFilter extends RevFilter { @SuppressWarnings("nls") @Override public String toString() { - return super.toString() + "(" + new Date(when * 1000L) + " - " - + new Date(until * 1000L) + ")"; + return super.toString() + "(" + Instant.ofEpochSecond(when) + " - " + + Instant.ofEpochSecond(until) + ")"; } } -- cgit v1.2.3 From a783bb76caa148577d2f5ce11ef037a547d51c34 Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Fri, 13 Dec 2024 10:47:08 -0800 Subject: transport: Remove usage of CommitTimeRevFilter with long and Dates [errorprone] The Date API is full of major design flaws and pitfalls and should be avoided at all costs. Prefer the java.time APIs, specifically, java.time.Instant (for physical time) and java.time.LocalDate[Time] (for civil time). Move callers to the CommitTimeRevFilter java.time.Instant API introduced in the parent commit. Change-Id: Ib92c56ae09f1ad3df7628a8c61212eb84250700d --- .../src/org/eclipse/jgit/transport/BasePackFetchConnection.java | 5 ++--- org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackFetchConnection.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackFetchConnection.java index 469a3d6015..be0d37b96e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackFetchConnection.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackFetchConnection.java @@ -12,10 +12,10 @@ package org.eclipse.jgit.transport; -import static org.eclipse.jgit.transport.GitProtocolConstants.PACKET_DELIM; import static org.eclipse.jgit.transport.GitProtocolConstants.PACKET_DEEPEN; import static org.eclipse.jgit.transport.GitProtocolConstants.PACKET_DEEPEN_NOT; import static org.eclipse.jgit.transport.GitProtocolConstants.PACKET_DEEPEN_SINCE; +import static org.eclipse.jgit.transport.GitProtocolConstants.PACKET_DELIM; import static org.eclipse.jgit.transport.GitProtocolConstants.PACKET_DONE; import static org.eclipse.jgit.transport.GitProtocolConstants.PACKET_END; import static org.eclipse.jgit.transport.GitProtocolConstants.PACKET_ERR; @@ -32,7 +32,6 @@ import java.time.Instant; import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.Date; import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; @@ -715,7 +714,7 @@ public abstract class BasePackFetchConnection extends BasePackConnection // wind up later matching up against things we want and we // can avoid asking for something we already happen to have. // - final Date maxWhen = new Date(maxTime * 1000L); + Instant maxWhen = Instant.ofEpochSecond(maxTime); walk.sort(RevSort.COMMIT_TIME_DESC); walk.markStart(reachableCommits); walk.setRevFilter(CommitTimeRevFilter.after(maxWhen)); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java index d972067e40..41ab8acf05 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java @@ -2226,7 +2226,7 @@ public class UploadPack implements Closeable { walk.resetRetain(SAVE); walk.markStart((RevCommit) want); if (oldestTime != 0) - walk.setRevFilter(CommitTimeRevFilter.after(oldestTime * 1000L)); + walk.setRevFilter(CommitTimeRevFilter.after(Instant.ofEpochSecond(oldestTime))); for (;;) { final RevCommit c = walk.next(); if (c == null) -- cgit v1.2.3 From 7fc6382689f95961e5a6f86046be6da1f86f7bc4 Mon Sep 17 00:00:00 2001 From: Thomas Wolf Date: Thu, 19 Dec 2024 20:54:33 +0100 Subject: CommitTimeRevFilter: add missing @since Annotate the new methods using Instant. Change-Id: I1dc4eec59b44e784bcc151bd6a28347f6fd0de03 --- .../eclipse/jgit/revwalk/filter/CommitTimeRevFilter.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/filter/CommitTimeRevFilter.java b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/filter/CommitTimeRevFilter.java index 40f56e38ba..c9186b5491 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/filter/CommitTimeRevFilter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/filter/CommitTimeRevFilter.java @@ -45,6 +45,7 @@ public abstract class CommitTimeRevFilter extends RevFilter { * @param ts * the point in time to cut on. * @return a new filter to select commits on or before ts. + * @since 7.2 */ public static RevFilter before(Instant ts) { return new Before(ts); @@ -81,6 +82,7 @@ public abstract class CommitTimeRevFilter extends RevFilter { * @param ts * the point in time to cut on. * @return a new filter to select commits on or after ts. + * @since 7.2 */ public static RevFilter after(Instant ts) { return new After(ts); @@ -113,12 +115,16 @@ public abstract class CommitTimeRevFilter extends RevFilter { } /** - * Create a new filter to select commits after or equal a given date/time since - * and before or equal a given date/time until. + * Create a new filter to select commits after or equal a given date/time + * since and before or equal a given date/time + * until. * - * @param since the point in time to cut on. - * @param until the point in time to cut off. + * @param since + * the point in time to cut on. + * @param until + * the point in time to cut off. * @return a new filter to select commits between the given date/times. + * @since 7.2 */ public static RevFilter between(Instant since, Instant until) { return new Between(since, until); -- cgit v1.2.3 From 1177e1e4a9b6fa52f4e40fe961263cd510114f81 Mon Sep 17 00:00:00 2001 From: Martin Fick Date: Wed, 4 Dec 2024 09:33:00 -0800 Subject: WindowCache: share removal work among multiple threads Split the removal process into blocks so that it can be shared by multiple threads. This potential work sharing can provide 2 optimizations for removals: 1) It provides an opportunity for separate removal requests to be consolidated into one removal pass. 2) It can reduce removing thread latencies by sharing the removal work with other removing threads which otherwise might not have any work to do due to their removal request being consolidated. This makes the system more efficient and can actually reduce latencies as system load increases due to pack removals! The optimizations above are all achieved without blockng threads to wait for other threads to complete (although naturally there are some synchronization points), and while ensuring that no threads do more work than if they were the only thread available to perform a removal. Change-Id: Ic6809a8abf056299abde0f0c58c77aaf245a8df5 Signed-off-by: Martin Fick --- .../jgit/internal/storage/file/WindowCache.java | 131 ++++++++++++++++++++- 1 file changed, 126 insertions(+), 5 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCache.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCache.java index 3ec7e6e666..fd80faf4ed 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCache.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCache.java @@ -15,6 +15,7 @@ import java.io.IOException; import java.lang.ref.ReferenceQueue; import java.lang.ref.SoftReference; import java.util.Collections; +import java.util.HashSet; import java.util.Map; import java.util.Random; import java.util.Set; @@ -402,7 +403,7 @@ public class WindowCache { } static final void purge(Set packs) { - cache.removeAll(packs); + cache.queueRemoveAll(packs); } /** cleanup released and/or garbage collected windows. */ @@ -446,6 +447,29 @@ public class WindowCache { private final boolean useStrongIndexRefs; + /** Removers are purely CPU/mem bound (no I/O), so likely should not go above # CPUs */ + private final int idealNumRemovers; + + /** Number of blocks to split the Pack removal into. + * + * Consolidation is better with more blocks since it increases + * the wait before moving to the next set by allowing more work + * to accumulate in the next set. On the flip side, the more + * blocks, the more synchronization overhead increasing each + * removers latency. + */ + private final int numRemovalBlocks; + + private final int removalBlockSize; + + private Set packsToRemove = new HashSet<>(); + + private Set packsBeingRemoved; + + private int numRemovers; + + private int blockBeingRemoved; + private WindowCache(WindowCacheConfig cfg) { tableSize = tableSize(cfg); final int lockCount = lockCount(cfg); @@ -484,6 +508,23 @@ public class WindowCache { statsRecorder = mbean; publishMBean.set(cfg.getExposeStatsViaJmx()); + /* Since each worker will only process up to one full set of blocks, at least 2 + * workers are needed anytime there are queued removals to ensure that all the + * blocks will get processed. However, if workers are maxed out at only 2, then + * enough newer workers will never start in order to make it safe for older + * workers to quit early. At least 3 workers are needed to make older worker + * relief transitions possible. + */ + idealNumRemovers = Math.max(3, Runtime.getRuntime().availableProcessors()); + + int bs = 1024; + if (tableSize < 2 * bs) { + bs = tableSize / 2; + } + removalBlockSize = bs; + numRemovalBlocks = tableSize / removalBlockSize; + blockBeingRemoved = numRemovalBlocks - 1; + if (maxFiles < 1) throw new IllegalArgumentException(JGitText.get().openFilesMustBeAtLeast1); if (maxBytes < windowSize) @@ -713,7 +754,7 @@ public class WindowCache { } /** - * Clear all entries related to a single file. + * Asynchronously clear all entries related to files. *

* Typically this method is invoked during {@link Pack#close()}, or * {@link Pack#close(Set)}, when we know the packs are never going to be @@ -721,11 +762,91 @@ public class WindowCache { * A concurrent reader loading an entry from these same packs may cause a * pack to become stuck in the cache anyway. * + * Work on clearing files will be split up into blocks so that removing + * can be shared by more than one thread. This potential work sharing + * can provide 2 optimizations for removals: + *

    + *
  1. It provides an opportunity for separate removal requests to be + * consolidated into one removal pass.
    2. + *
  2. It can reduce removing thread latencies by sharing the removal work + * with other removing threads which otherwise might not have any work to do + * due to their removal request being consolidated. This makes the system + * more efficient and can actually reduce latencies as system load increases + * due to pack removals!
    3. + *
+ * The optimizations above are all achieved without blockng threads to wait + * for other threads to complete (although naturally there are some + * synchronization points), and while ensuring that no threads do more work + * than if they were the only thread available to perform a removal. + * * @param packs - * the files to purge all entries of. + * the files to purge all entries of */ - private void removeAll(Set packs) { - for (int s = 0; s < tableSize; s++) { + private void queueRemoveAll(Set packs) { + synchronized (this) { + packsToRemove.addAll(packs); + if (numRemovers >= idealNumRemovers) { + return; + } + numRemovers++; + } + for (int numRemoved = 0; removeNextBlock(numRemoved); numRemoved++); + synchronized (this) { + if (numRemovers > 0) { + numRemovers--; + } + } + } + + /** Determine which block to remove next, if any, and do so. + * + * @param numRemoved + * the number of already processed block removals by the current thread + * @return whether more processing should be done by the current thread + */ + private boolean removeNextBlock(int numRemoved) { + Set toRemove; + int block; + synchronized (this) { + if (packsBeingRemoved == null || blockBeingRemoved >= numRemovalBlocks - 1) { + if (packsToRemove.isEmpty()) { + return false; + } + + blockBeingRemoved = 0; + packsBeingRemoved = packsToRemove; + packsToRemove = new HashSet<>(); + } else { + blockBeingRemoved++; + } + + toRemove = packsBeingRemoved; + block = blockBeingRemoved; + } + + removeBlock(toRemove, block); + numRemoved++; + + /* Ensure threads never work on more than a full set of blocks (the equivalent + * of removing one Pack) */ + boolean isLast = numRemoved >= numRemovalBlocks; + synchronized (this) { + if (numRemovers >= idealNumRemovers) { + isLast = true; + } + } + return !isLast; + } + + /** Remove a block of entries for a Set of files + * @param packs + * the files to purge all entries of + * @param block + * the specific block to process removals for + */ + private void removeBlock(Set packs, int block) { + int starting = block * removalBlockSize; + for (int s = starting; s < starting + removalBlockSize && s < tableSize; s++) { final Entry e1 = table.get(s); boolean hasDead = false; for (Entry e = e1; e != null; e = e.next) { -- cgit v1.2.3 From 2ec1dc20ec71daa33007fb0fa62eff96eabf8254 Mon Sep 17 00:00:00 2001 From: Thomas Wolf Date: Fri, 20 Dec 2024 19:26:40 +0100 Subject: SignatureUtils.toString(): allow null signature creation date Some signatures may not have a "created at" timestamp, or the tool used to verify the signature does not report it. If we get none, do not report anything about the signature creation time. This can happen for instance if 'smimesign' is used for verifying x509 signatures. Change-Id: I1a63aa62ffe173e00f27e8aea539b26cd40387c0 --- .../src/org/eclipse/jgit/util/SignatureUtils.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/SignatureUtils.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/SignatureUtils.java index 90524db20a..820ac2db91 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/SignatureUtils.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/SignatureUtils.java @@ -45,12 +45,15 @@ public final class SignatureUtils { public static String toString(SignatureVerification verification, PersonIdent creator, GitDateFormatter formatter) { StringBuilder result = new StringBuilder(); - // Use the creator's timezone for the signature date - PersonIdent dateId = new PersonIdent(creator, - verification.creationDate()); - result.append(MessageFormat.format(JGitText.get().verifySignatureMade, - formatter.formatDate(dateId))); - result.append('\n'); + if (verification.creationDate() != null) { + // Use the creator's timezone for the signature date + PersonIdent dateId = new PersonIdent(creator, + verification.creationDate()); + result.append( + MessageFormat.format(JGitText.get().verifySignatureMade, + formatter.formatDate(dateId))); + result.append('\n'); + } result.append(MessageFormat.format( JGitText.get().verifySignatureKey, verification.keyFingerprint().toUpperCase(Locale.ROOT))); -- cgit v1.2.3 From 51d6c63fe1602a9e812dc3610c10d7e6e31eeafd Mon Sep 17 00:00:00 2001 From: Martin Fick Date: Tue, 17 Dec 2024 17:30:17 -0800 Subject: Pack: separate an open/close accounting lock Previously the open/close accounting code used whole Pack object synchronization for locking. Unfortunately, there are other unrelated methods which use whole Pack object synchronization also, mostly to avoid concurrent loading of these independent indices, and they do not touch or need to coordinate with the open/close accounting data. During heavy load when a new file appears after repacking the readFully() threads could uselessly block on threads reading the reverse index. These threads could have been reading from the Pack file instead of waiting for the reverse index to be read. Use a new lock to make this locking more fine grained to prevent the readFully() calling threads from getting blocked in beginWindowCache() while the reverse index or bitmaps are being loaded. Change-Id: I7ac9067ca10cd6d6be0ab25148d99da3ace7ba36 Signed-off-by: Martin Fick --- .../eclipse/jgit/internal/storage/file/Pack.java | 56 ++++++++++++++-------- 1 file changed, 35 insertions(+), 21 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java index 8d2a86386f..1a7b5de1d7 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java @@ -95,6 +95,9 @@ public class Pack implements Iterable { private RandomAccessFile fd; + /** For managing open/close accounting of {@link #fd}. */ + private final Object activeLock = new Object(); + /** Serializes reads performed against {@link #fd}. */ private final Object readLock = new Object(); @@ -645,37 +648,48 @@ public class Pack implements Iterable { throw new EOFException(); } - private synchronized void beginCopyAsIs() + private void beginCopyAsIs() throws StoredObjectRepresentationNotAvailableException { - if (++activeCopyRawData == 1 && activeWindows == 0) { - try { - doOpen(); - } catch (IOException thisPackNotValid) { - throw new StoredObjectRepresentationNotAvailableException( - thisPackNotValid); + synchronized (activeLock) { + if (++activeCopyRawData == 1 && activeWindows == 0) { + try { + doOpen(); + } catch (IOException thisPackNotValid) { + throw new StoredObjectRepresentationNotAvailableException( + thisPackNotValid); + } } } } - private synchronized void endCopyAsIs() { - if (--activeCopyRawData == 0 && activeWindows == 0) - doClose(); + private void endCopyAsIs() { + synchronized (activeLock) { + if (--activeCopyRawData == 0 && activeWindows == 0) { + doClose(); + } + } } - synchronized boolean beginWindowCache() throws IOException { - if (++activeWindows == 1) { - if (activeCopyRawData == 0) - doOpen(); - return true; + boolean beginWindowCache() throws IOException { + synchronized (activeLock) { + if (++activeWindows == 1) { + if (activeCopyRawData == 0) { + doOpen(); + } + return true; + } + return false; } - return false; } - synchronized boolean endWindowCache() { - final boolean r = --activeWindows == 0; - if (r && activeCopyRawData == 0) - doClose(); - return r; + boolean endWindowCache() { + synchronized (activeLock) { + boolean r = --activeWindows == 0; + if (r && activeCopyRawData == 0) { + doClose(); + } + return r; + } } private void doOpen() throws IOException { -- cgit v1.2.3 From cef16a6105fe26847a9ad71f53cb7ffd778c1170 Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Mon, 23 Dec 2024 09:59:01 -0800 Subject: PushCertificateStore: Use Instant, unused var, boxing PendingCerts are ordered using PersonIdent#getWhen, which returns the time as java.util.Date. That method is deprecated, as we are moving to the new java.time.Instant API. Sort the pending certs using the Instant time (PersonIdent#getWhenAsInstant). Take the chance to fix two minor warnings: * the inserter is never used in the try{} block * Unnecessary boxing in message format Change-Id: I211e5319dc9865bd5a3468c7f5bbc141a6d5e71e --- .../src/org/eclipse/jgit/transport/PushCertificateStore.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/PushCertificateStore.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/PushCertificateStore.java index a9e93b6be6..6bdaf0e234 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/PushCertificateStore.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/PushCertificateStore.java @@ -24,6 +24,7 @@ import java.text.MessageFormat; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -329,7 +330,7 @@ public class PushCertificateStore implements AutoCloseable { if (newId == null) { return RefUpdate.Result.NO_CHANGE; } - try (ObjectInserter inserter = db.newObjectInserter()) { + try { RefUpdate.Result result = updateRef(newId); switch (result) { case FAST_FORWARD: @@ -404,8 +405,8 @@ public class PushCertificateStore implements AutoCloseable { } private static void sortPending(List pending) { - Collections.sort(pending, (PendingCert a, PendingCert b) -> Long.signum( - a.ident.getWhen().getTime() - b.ident.getWhen().getTime())); + Collections.sort(pending, + Comparator.comparing((PendingCert a) -> a.ident.getWhenAsInstant())); } private DirCache newDirCache() throws IOException { @@ -503,7 +504,7 @@ public class PushCertificateStore implements AutoCloseable { } else { sb.append(MessageFormat.format( JGitText.get().storePushCertMultipleRefs, - Integer.valueOf(cert.getCommands().size()))); + cert.getCommands().size())); } return sb.append('\n').toString(); } -- cgit v1.2.3 From ac269d4ae2433c2c65ff0d026589741c8029ac24 Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Fri, 15 Nov 2024 12:20:05 -0800 Subject: ProposedTimestamp: mark unused methods as deprecated date() and timestamp() don't seem to have any use. I find only calls to instant(). Mark as deprecated to remove usages of java.util.Date and related classes. Change-Id: I4e77c64678295b5ea83ada253c0e120f7bae5843 --- .../src/org/eclipse/jgit/util/time/ProposedTimestamp.java | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/time/ProposedTimestamp.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/time/ProposedTimestamp.java index a5ee1070d0..a20eaaf908 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/time/ProposedTimestamp.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/time/ProposedTimestamp.java @@ -138,7 +138,10 @@ public abstract class ProposedTimestamp implements AutoCloseable { * Get time since epoch, with up to microsecond resolution. * * @return time since epoch, with up to microsecond resolution. + * + * @deprecated Use instant() instead */ + @Deprecated(since = "7.2") public Timestamp timestamp() { return Timestamp.from(instant()); } @@ -147,7 +150,10 @@ public abstract class ProposedTimestamp implements AutoCloseable { * Get time since epoch, with up to millisecond resolution. * * @return time since epoch, with up to millisecond resolution. + * + * @deprecated Use instant() instead */ + @Deprecated(since = "7.2") public Date date() { return new Date(millis()); } -- cgit v1.2.3 From 9219a7724e4f8b46130577acd24fee8cd7956099 Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Wed, 6 Nov 2024 14:54:52 -0800 Subject: RelativeDateFormatter: Use Instant instead of date Errorprone suggest to use the more modern java.time.Instant instead of the older Date API. Offer a variant of RelativeDateFormatter#format method for Instants. Change-Id: I536230327ec7b617958191fbe36f98b1294f6d2e --- .../jgit/util/RelativeDateFormatterTest.java | 8 ++++---- .../org/eclipse/jgit/util/GitDateFormatter.java | 2 +- .../eclipse/jgit/util/RelativeDateFormatter.java | 23 ++++++++++++++++++++-- 3 files changed, 26 insertions(+), 7 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/RelativeDateFormatterTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/RelativeDateFormatterTest.java index 214bbca944..a927d8dbef 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/RelativeDateFormatterTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/RelativeDateFormatterTest.java @@ -16,7 +16,7 @@ import static org.eclipse.jgit.util.RelativeDateFormatter.SECOND_IN_MILLIS; import static org.eclipse.jgit.util.RelativeDateFormatter.YEAR_IN_MILLIS; import static org.junit.Assert.assertEquals; -import java.util.Date; +import java.time.Instant; import org.eclipse.jgit.junit.MockSystemReader; import org.junit.After; @@ -37,9 +37,9 @@ public class RelativeDateFormatterTest { private static void assertFormat(long ageFromNow, long timeUnit, String expectedFormat) { - Date d = new Date(SystemReader.getInstance().getCurrentTime() - - ageFromNow * timeUnit); - String s = RelativeDateFormatter.format(d); + long millis = ageFromNow * timeUnit; + Instant aTime = SystemReader.getInstance().now().minusMillis(millis); + String s = RelativeDateFormatter.format(aTime); assertEquals(expectedFormat, s); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/GitDateFormatter.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/GitDateFormatter.java index e6bf497ac4..524126b098 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/GitDateFormatter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/GitDateFormatter.java @@ -148,7 +148,7 @@ public class GitDateFormatter { return String.format("%d %s%02d%02d", //$NON-NLS-1$ ident.getWhen().getTime() / 1000, sign, hours, minutes); case RELATIVE: - return RelativeDateFormatter.format(ident.getWhen()); + return RelativeDateFormatter.format(ident.getWhenAsInstant()); case LOCALELOCAL: case LOCAL: dateTimeInstance.setTimeZone(SystemReader.getInstance() diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/RelativeDateFormatter.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/RelativeDateFormatter.java index 5611b1e78e..b6b19e0e7b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/RelativeDateFormatter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/RelativeDateFormatter.java @@ -10,6 +10,8 @@ package org.eclipse.jgit.util; import java.text.MessageFormat; +import java.time.Duration; +import java.time.Instant; import java.util.Date; import org.eclipse.jgit.internal.JGitText; @@ -42,12 +44,29 @@ public class RelativeDateFormatter { * @return age of given {@link java.util.Date} compared to now formatted in * the same relative format as returned by * {@code git log --relative-date} + * @deprecated Use {@link #format(Instant)} instead. */ + @Deprecated(since = "7.2") @SuppressWarnings("boxing") public static String format(Date when) { + return format(when.toInstant()); + } - long ageMillis = SystemReader.getInstance().getCurrentTime() - - when.getTime(); + /** + * Get age of given {@link java.time.Instant} compared to now formatted in the + * same relative format as returned by {@code git log --relative-date} + * + * @param when + * an instant to format + * @return age of given instant compared to now formatted in + * the same relative format as returned by + * {@code git log --relative-date} + * @since 7.2 + */ + @SuppressWarnings("boxing") + public static String format(Instant when) { + long ageMillis = Duration + .between(when, SystemReader.getInstance().now()).toMillis(); // shouldn't happen in a perfect world if (ageMillis < 0) -- cgit v1.2.3 From 478c91216ff1867b747bbe6cfe0c180b4e0bbdf4 Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Tue, 17 Dec 2024 10:19:48 -0800 Subject: RecursiveMerger: Create PersonIdent with java.time API We are moving away from the old java.util.Date api to the java.time.Instant version. This class uses a deprecated constructor in PersonIdent. Use the new constructor for PersonIdent with Instant/ZoneId. Change-Id: I50e2a643ad17a5c0722f87b1bb8abcad286745a5 --- .../src/org/eclipse/jgit/merge/RecursiveMerger.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/merge/RecursiveMerger.java b/org.eclipse.jgit/src/org/eclipse/jgit/merge/RecursiveMerger.java index fc5ab62898..f58ef4faba 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/merge/RecursiveMerger.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/merge/RecursiveMerger.java @@ -18,10 +18,10 @@ package org.eclipse.jgit.merge; import java.io.IOException; import java.text.MessageFormat; +import java.time.Instant; +import java.time.ZoneOffset; import java.util.ArrayList; -import java.util.Date; import java.util.List; -import java.util.TimeZone; import java.util.stream.Collectors; import org.eclipse.jgit.dircache.DirCache; @@ -233,12 +233,11 @@ public class RecursiveMerger extends ResolveMerger { private static PersonIdent mockAuthor(List parents) { String name = RecursiveMerger.class.getSimpleName(); int time = 0; - for (RevCommit p : parents) + for (RevCommit p : parents) { time = Math.max(time, p.getCommitTime()); - return new PersonIdent( - name, name + "@JGit", //$NON-NLS-1$ - new Date((time + 1) * 1000L), - TimeZone.getTimeZone("GMT+0000")); //$NON-NLS-1$ + } + return new PersonIdent(name, name + "@JGit", //$NON-NLS-1$ + Instant.ofEpochSecond(time+1), ZoneOffset.UTC); } private String failingPathsMessage() { -- cgit v1.2.3 From 0be7e96ed1aef31acc945974aa1f0fd190a3f795 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Tue, 24 Dec 2024 17:10:29 +0100 Subject: Mark static method PersonIdent#getTimeZone as deprecated since it's using java.util.Timezone. Instead promote usage of #getZoneId which uses ZoneId from the preferred java.time API. Change-Id: I15dee551a81acf0b8aafdbfcfcf200c0d38069f4 --- org.eclipse.jgit/src/org/eclipse/jgit/lib/PersonIdent.java | 2 ++ 1 file changed, 2 insertions(+) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/PersonIdent.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/PersonIdent.java index f22642c4ce..50f4a83b93 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/PersonIdent.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/PersonIdent.java @@ -45,7 +45,9 @@ public class PersonIdent implements Serializable { * timezone offset as in {@link #getTimeZoneOffset()}. * @return time zone object for the given offset. * @since 4.1 + * @deprecated use {@link #getZoneId(int)} instead */ + @Deprecated(since = "7.2") public static TimeZone getTimeZone(int tzOffset) { StringBuilder tzId = new StringBuilder(8); tzId.append("GMT"); //$NON-NLS-1$ -- cgit v1.2.3 From fc5cbf1d778f705acd1c388208846040cb7fa70a Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Wed, 25 Dec 2024 00:49:30 +0100 Subject: GitDateFormatter: use java.time API We are moving away from the old java.util.Date API to the java.time API. Change-Id: I904556f94cb28677cce565054bd2cd0f4fb2a096 --- .../org/eclipse/jgit/util/GitDateFormatter.java | 86 ++++++++++++---------- 1 file changed, 48 insertions(+), 38 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/GitDateFormatter.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/GitDateFormatter.java index 524126b098..332e65985e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/GitDateFormatter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/GitDateFormatter.java @@ -10,10 +10,10 @@ package org.eclipse.jgit.util; -import java.text.DateFormat; -import java.text.SimpleDateFormat; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.time.format.FormatStyle; import java.util.Locale; -import java.util.TimeZone; import org.eclipse.jgit.lib.PersonIdent; @@ -26,9 +26,9 @@ import org.eclipse.jgit.lib.PersonIdent; */ public class GitDateFormatter { - private DateFormat dateTimeInstance; + private DateTimeFormatter dateTimeFormat; - private DateFormat dateTimeInstance2; + private DateTimeFormatter dateTimeFormat2; private final Format format; @@ -96,30 +96,34 @@ public class GitDateFormatter { default: break; case DEFAULT: // Not default: - dateTimeInstance = new SimpleDateFormat( + dateTimeFormat = DateTimeFormatter.ofPattern( "EEE MMM dd HH:mm:ss yyyy Z", Locale.US); //$NON-NLS-1$ break; case ISO: - dateTimeInstance = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss Z", //$NON-NLS-1$ + dateTimeFormat = DateTimeFormatter.ofPattern( + "yyyy-MM-dd HH:mm:ss Z", //$NON-NLS-1$ Locale.US); break; case LOCAL: - dateTimeInstance = new SimpleDateFormat("EEE MMM dd HH:mm:ss yyyy", //$NON-NLS-1$ + dateTimeFormat = DateTimeFormatter.ofPattern( + "EEE MMM dd HH:mm:ss yyyy", //$NON-NLS-1$ Locale.US); break; case RFC: - dateTimeInstance = new SimpleDateFormat( + dateTimeFormat = DateTimeFormatter.ofPattern( "EEE, dd MMM yyyy HH:mm:ss Z", Locale.US); //$NON-NLS-1$ break; case SHORT: - dateTimeInstance = new SimpleDateFormat("yyyy-MM-dd", Locale.US); //$NON-NLS-1$ + dateTimeFormat = DateTimeFormatter.ofPattern("yyyy-MM-dd", //$NON-NLS-1$ + Locale.US); break; case LOCALE: case LOCALELOCAL: - SystemReader systemReader = SystemReader.getInstance(); - dateTimeInstance = systemReader.getDateTimeInstance( - DateFormat.DEFAULT, DateFormat.DEFAULT); - dateTimeInstance2 = systemReader.getSimpleDateFormat("Z"); //$NON-NLS-1$ + dateTimeFormat = DateTimeFormatter + .ofLocalizedDateTime(FormatStyle.MEDIUM) + .withLocale(Locale.US); + dateTimeFormat2 = DateTimeFormatter.ofPattern("Z", //$NON-NLS-1$ + Locale.US); break; } } @@ -135,39 +139,45 @@ public class GitDateFormatter { @SuppressWarnings("boxing") public String formatDate(PersonIdent ident) { switch (format) { - case RAW: - int offset = ident.getTimeZoneOffset(); + case RAW: { + int offset = ident.getZoneOffset().getTotalSeconds(); String sign = offset < 0 ? "-" : "+"; //$NON-NLS-1$ //$NON-NLS-2$ int offset2; - if (offset < 0) + if (offset < 0) { offset2 = -offset; - else + } else { offset2 = offset; - int hours = offset2 / 60; - int minutes = offset2 % 60; + } + int minutes = (offset2 / 60) % 60; + int hours = offset2 / 60 / 60; return String.format("%d %s%02d%02d", //$NON-NLS-1$ - ident.getWhen().getTime() / 1000, sign, hours, minutes); + ident.getWhenAsInstant().getEpochSecond(), sign, hours, + minutes); + } case RELATIVE: return RelativeDateFormatter.format(ident.getWhenAsInstant()); case LOCALELOCAL: case LOCAL: - dateTimeInstance.setTimeZone(SystemReader.getInstance() - .getTimeZone()); - return dateTimeInstance.format(ident.getWhen()); - case LOCALE: - TimeZone tz = ident.getTimeZone(); - if (tz == null) - tz = SystemReader.getInstance().getTimeZone(); - dateTimeInstance.setTimeZone(tz); - dateTimeInstance2.setTimeZone(tz); - return dateTimeInstance.format(ident.getWhen()) + " " //$NON-NLS-1$ - + dateTimeInstance2.format(ident.getWhen()); - default: - tz = ident.getTimeZone(); - if (tz == null) - tz = SystemReader.getInstance().getTimeZone(); - dateTimeInstance.setTimeZone(ident.getTimeZone()); - return dateTimeInstance.format(ident.getWhen()); + return dateTimeFormat + .withZone(SystemReader.getInstance().getTimeZoneId()) + .format(ident.getWhenAsInstant()); + case LOCALE: { + ZoneId tz = ident.getZoneId(); + if (tz == null) { + tz = SystemReader.getInstance().getTimeZoneId(); + } + return dateTimeFormat.withZone(tz).format(ident.getWhenAsInstant()) + + " " //$NON-NLS-1$ + + dateTimeFormat2.withZone(tz) + .format(ident.getWhenAsInstant()); + } + default: { + ZoneId tz = ident.getZoneId(); + if (tz == null) { + tz = SystemReader.getInstance().getTimeZoneId(); + } + return dateTimeFormat.withZone(tz).format(ident.getWhenAsInstant()); + } } } } -- cgit v1.2.3 From d32a770f144d3b8a87eeb811e4c8987ae9145966 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Wed, 25 Dec 2024 01:26:46 +0100 Subject: SystemReader: add version when annotated methods became deprecated Change-Id: I2a4e8c18378c1bcbf01c12a79d31cb5a2a050e97 --- org.eclipse.jgit/src/org/eclipse/jgit/util/SystemReader.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/SystemReader.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/SystemReader.java index 55cc878e02..22b82b3610 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/SystemReader.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/SystemReader.java @@ -523,7 +523,7 @@ public abstract class SystemReader { * * @deprecated Use {@link #now()} */ - @Deprecated + @Deprecated(since = "7.1") public abstract long getCurrentTime(); /** @@ -569,7 +569,7 @@ public abstract class SystemReader { * * @deprecated Use {@link #getTimeZoneAt(Instant)} instead. */ - @Deprecated + @Deprecated(since = "7.1") public abstract int getTimezone(long when); /** @@ -592,7 +592,7 @@ public abstract class SystemReader { * * @deprecated Use {@link #getTimeZoneId()} */ - @Deprecated + @Deprecated(since = "7.1") public TimeZone getTimeZone() { return TimeZone.getDefault(); } -- cgit v1.2.3 From 5a56be277dac6328c9aec8ada60ab89c9750b1da Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Wed, 25 Dec 2024 21:54:48 +0100 Subject: DescribeCommand: use java.time API We are moving away from the old java.util.Date API to the java.time API. Change-Id: I46b1f12192fe40d3bb740c3ce8632fffcd5fc5de --- org.eclipse.jgit/src/org/eclipse/jgit/api/DescribeCommand.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/DescribeCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/DescribeCommand.java index 805a886392..934245781f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/DescribeCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/DescribeCommand.java @@ -15,11 +15,11 @@ import static org.eclipse.jgit.lib.TypedConfigGetter.UNSET_INT; import java.io.IOException; import java.text.MessageFormat; +import java.time.Instant; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Comparator; -import java.util.Date; import java.util.List; import java.util.Map; import java.util.Optional; @@ -274,10 +274,10 @@ public class DescribeCommand extends GitCommand { } } - private Date tagDate(Ref tag) throws IOException { + private Instant tagDate(Ref tag) throws IOException { RevTag t = w.parseTag(tag.getObjectId()); w.parseBody(t); - return t.getTaggerIdent().getWhen(); + return t.getTaggerIdent().getWhenAsInstant(); } }; -- cgit v1.2.3 From 8497c89192b80fd333b6227a7e222fbd4d1697e7 Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Fri, 27 Dec 2024 16:45:38 -0800 Subject: DfsGarbageCollector: Use SystemReader.now() instead of getCurrentTime() getCurrentTime() is deprecated. Use now() that returns an Instant. Change-Id: I107af03342e6815d97181d6dccd94e2fae2d3b21 --- .../eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java index e6068a15ec..199481cf33 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsGarbageCollector.java @@ -90,7 +90,7 @@ public class DfsGarbageCollector { private long coalesceGarbageLimit = 50 << 20; private long garbageTtlMillis = TimeUnit.DAYS.toMillis(1); - private long startTimeMillis; + private Instant startTime; private List packsBefore; private List reftablesBefore; private List expiredGarbagePacks; @@ -352,7 +352,7 @@ public class DfsGarbageCollector { throw new IllegalStateException( JGitText.get().supportOnlyPackIndexVersion2); - startTimeMillis = SystemReader.getInstance().getCurrentTime(); + startTime = SystemReader.getInstance().now(); ctx = objdb.newReader(); try { refdb.refresh(); @@ -435,7 +435,7 @@ public class DfsGarbageCollector { packsBefore = new ArrayList<>(packs.length); expiredGarbagePacks = new ArrayList<>(packs.length); - long now = SystemReader.getInstance().getCurrentTime(); + long now = SystemReader.getInstance().now().toEpochMilli(); for (DfsPackFile p : packs) { DfsPackDescription d = p.getPackDescription(); if (d.getPackSource() != UNREACHABLE_GARBAGE) { @@ -723,7 +723,7 @@ public class DfsGarbageCollector { PackStatistics stats = pw.getStatistics(); pack.setPackStats(stats); - pack.setLastModified(startTimeMillis); + pack.setLastModified(startTime.toEpochMilli()); newPackDesc.add(pack); newPackStats.add(stats); newPackObj.add(pw.getObjectSet()); -- cgit v1.2.3 From 57dbbfe98b7f80830fbfb49aa35335052d2c6f9f Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Fri, 27 Dec 2024 17:07:38 -0800 Subject: PackWriterBitmapPreparer: Use SystemReader.now() instead of deprecated method SystemReader.getCurrentTime() is deprecated. Change-Id: If4d683633a77eec9b4cc32daed00254fbaa3114b --- .../jgit/internal/storage/pack/PackWriterBitmapPreparer.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriterBitmapPreparer.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriterBitmapPreparer.java index dabc1f0c5f..bf87c4c9d6 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriterBitmapPreparer.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriterBitmapPreparer.java @@ -14,6 +14,7 @@ import static org.eclipse.jgit.internal.storage.file.PackBitmapIndex.FLAG_REUSE; import static org.eclipse.jgit.revwalk.RevFlag.SEEN; import java.io.IOException; +import java.time.Instant; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -28,16 +29,16 @@ import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.revwalk.AddUnseenToBitmapFilter; import org.eclipse.jgit.internal.storage.file.BitmapIndexImpl; +import org.eclipse.jgit.internal.storage.file.BitmapIndexImpl.CompressedBitmap; import org.eclipse.jgit.internal.storage.file.PackBitmapIndex; import org.eclipse.jgit.internal.storage.file.PackBitmapIndexBuilder; import org.eclipse.jgit.internal.storage.file.PackBitmapIndexRemapper; -import org.eclipse.jgit.internal.storage.file.BitmapIndexImpl.CompressedBitmap; import org.eclipse.jgit.lib.AnyObjectId; +import org.eclipse.jgit.lib.BitmapIndex.BitmapBuilder; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectReader; import org.eclipse.jgit.lib.ProgressMonitor; -import org.eclipse.jgit.lib.BitmapIndex.BitmapBuilder; import org.eclipse.jgit.revwalk.BitmapWalker; import org.eclipse.jgit.revwalk.ObjectWalk; import org.eclipse.jgit.revwalk.RevCommit; @@ -99,10 +100,10 @@ class PackWriterBitmapPreparer { this.excessiveBranchCount = config.getBitmapExcessiveBranchCount(); this.excessiveBranchTipCount = Math.max(excessiveBranchCount, config.getBitmapExcessiveBranchTipCount()); - long now = SystemReader.getInstance().getCurrentTime(); + Instant now = SystemReader.getInstance().now(); long ageInSeconds = (long) config.getBitmapInactiveBranchAgeInDays() * DAY_IN_SECONDS; - this.inactiveBranchTimestamp = (now / 1000) - ageInSeconds; + this.inactiveBranchTimestamp = now.getEpochSecond() - ageInSeconds; } /** -- cgit v1.2.3 From 300a53dd6f84a630c73754632de0b4a1651ff890 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Wed, 25 Dec 2024 23:05:42 +0100 Subject: GarbageCollectCommand, GC: use java.time API We are moving away from the old java.util.Date API to the java.time API. Add GitTimeParser#parseInstant to support this. Change-Id: I3baeafeda5b65421dc94c1045b0ba576d4f79662 --- .../org/eclipse/jgit/api/GitConstructionTest.java | 3 +- .../tst/org/eclipse/jgit/api/PushCommandTest.java | 3 +- .../eclipse/jgit/api/GarbageCollectCommand.java | 28 +++++++- .../org/eclipse/jgit/internal/storage/file/GC.java | 78 ++++++++++++++++------ .../src/org/eclipse/jgit/util/GitTimeParser.java | 35 ++++++++++ 5 files changed, 122 insertions(+), 25 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/GitConstructionTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/GitConstructionTest.java index 76934343da..e847e72415 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/GitConstructionTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/GitConstructionTest.java @@ -14,6 +14,7 @@ import static org.junit.Assert.fail; import java.io.File; import java.io.IOException; +import java.time.Instant; import org.eclipse.jgit.api.ListBranchCommand.ListMode; import org.eclipse.jgit.api.errors.GitAPIException; @@ -100,7 +101,7 @@ public class GitConstructionTest extends RepositoryTestCase { GitAPIException { File workTree = db.getWorkTree(); Git git = Git.open(workTree); - git.gc().setExpire(null).call(); + git.gc().setExpire((Instant) null).call(); git.checkout().setName(git.getRepository().resolve("HEAD^").getName()) .call(); try { diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/PushCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/PushCommandTest.java index 70e990dedf..d1696d62a8 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/PushCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/PushCommandTest.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.io.PrintStream; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; +import java.time.Instant; import java.util.Properties; import org.eclipse.jgit.api.errors.DetachedHeadException; @@ -1146,7 +1147,7 @@ public class PushCommandTest extends RepositoryTestCase { RevCommit commit2 = git2.commit().setMessage("adding a").call(); // run a gc to ensure we have a bitmap index - Properties res = git1.gc().setExpire(null).call(); + Properties res = git1.gc().setExpire((Instant) null).call(); assertEquals(8, res.size()); // create another commit so we have something else to push diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/GarbageCollectCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/GarbageCollectCommand.java index 88d7e91860..f6935e1c67 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/GarbageCollectCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/GarbageCollectCommand.java @@ -12,6 +12,7 @@ package org.eclipse.jgit.api; import java.io.IOException; import java.text.MessageFormat; import java.text.ParseException; +import java.time.Instant; import java.util.Date; import java.util.Properties; import java.util.concurrent.ExecutionException; @@ -59,7 +60,7 @@ public class GarbageCollectCommand extends GitCommand { private ProgressMonitor monitor; - private Date expire; + private Instant expire; private PackConfig pconfig; @@ -98,8 +99,29 @@ public class GarbageCollectCommand extends GitCommand { * @param expire * minimal age of objects to be pruned. * @return this instance + * @deprecated use {@link #setExpire(Instant)} instead */ + @Deprecated(since = "7.2") public GarbageCollectCommand setExpire(Date expire) { + if (expire != null) { + this.expire = expire.toInstant(); + } + return this; + } + + /** + * During gc() or prune() each unreferenced, loose object which has been + * created or modified after expire will not be pruned. Only + * older objects may be pruned. If set to null then every object is a + * candidate for pruning. Use {@link org.eclipse.jgit.util.GitTimeParser} to + * parse time formats used by git gc. + * + * @param expire + * minimal age of objects to be pruned. + * @return this instance + * @since 7.2 + */ + public GarbageCollectCommand setExpire(Instant expire) { this.expire = expire; return this; } @@ -108,8 +130,8 @@ public class GarbageCollectCommand extends GitCommand { * Whether to use aggressive mode or not. If set to true JGit behaves more * similar to native git's "git gc --aggressive". If set to * true compressed objects found in old packs are not reused - * but every object is compressed again. Configuration variables - * pack.window and pack.depth are set to 250 for this GC. + * but every object is compressed again. Configuration variables pack.window + * and pack.depth are set to 250 for this GC. * * @since 3.6 * @param aggressive diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java index 7f3369364b..09cf35a0cf 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java @@ -102,9 +102,8 @@ import org.eclipse.jgit.treewalk.filter.TreeFilter; import org.eclipse.jgit.util.FS; import org.eclipse.jgit.util.FS.LockToken; import org.eclipse.jgit.util.FileUtils; -import org.eclipse.jgit.util.GitDateParser; +import org.eclipse.jgit.util.GitTimeParser; import org.eclipse.jgit.util.StringUtils; -import org.eclipse.jgit.util.SystemReader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -160,11 +159,11 @@ public class GC { private long expireAgeMillis = -1; - private Date expire; + private Instant expire; private long packExpireAgeMillis = -1; - private Date packExpire; + private Instant packExpire; private Boolean packKeptObjects; @@ -698,16 +697,18 @@ public class GC { if (expire == null && expireAgeMillis == -1) { String pruneExpireStr = getPruneExpireStr(); - if (pruneExpireStr == null) + if (pruneExpireStr == null) { pruneExpireStr = PRUNE_EXPIRE_DEFAULT; - expire = GitDateParser.parse(pruneExpireStr, null, SystemReader - .getInstance().getLocale()); + } + expire = GitTimeParser.parseInstant(pruneExpireStr); expireAgeMillis = -1; } - if (expire != null) - expireDate = expire.getTime(); - if (expireAgeMillis != -1) + if (expire != null) { + expireDate = expire.toEpochMilli(); + } + if (expireAgeMillis != -1) { expireDate = System.currentTimeMillis() - expireAgeMillis; + } return expireDate; } @@ -724,16 +725,18 @@ public class GC { String prunePackExpireStr = repo.getConfig().getString( ConfigConstants.CONFIG_GC_SECTION, null, ConfigConstants.CONFIG_KEY_PRUNEPACKEXPIRE); - if (prunePackExpireStr == null) + if (prunePackExpireStr == null) { prunePackExpireStr = PRUNE_PACK_EXPIRE_DEFAULT; - packExpire = GitDateParser.parse(prunePackExpireStr, null, - SystemReader.getInstance().getLocale()); + } + packExpire = GitTimeParser.parseInstant(prunePackExpireStr); packExpireAgeMillis = -1; } - if (packExpire != null) - packExpireDate = packExpire.getTime(); - if (packExpireAgeMillis != -1) + if (packExpire != null) { + packExpireDate = packExpire.toEpochMilli(); + } + if (packExpireAgeMillis != -1) { packExpireDate = System.currentTimeMillis() - packExpireAgeMillis; + } return packExpireDate; } @@ -1655,12 +1658,31 @@ public class GC { * candidate for pruning. * * @param expire - * instant in time which defines object expiration - * objects with modification time before this instant are expired - * objects with modification time newer or equal to this instant - * are not expired + * instant in time which defines object expiration objects with + * modification time before this instant are expired objects with + * modification time newer or equal to this instant are not + * expired + * @deprecated use {@link #setExpire(Instant)} instead */ + @Deprecated(since = "7.2") public void setExpire(Date expire) { + this.expire = expire.toInstant(); + expireAgeMillis = -1; + } + + /** + * During gc() or prune() each unreferenced, loose object which has been + * created or modified after or at expire will not be pruned. + * Only older objects may be pruned. If set to null then every object is a + * candidate for pruning. + * + * @param expire + * instant in time which defines object expiration objects with + * modification time before this instant are expired objects with + * modification time newer or equal to this instant are not + * expired + */ + public void setExpire(Instant expire) { this.expire = expire; expireAgeMillis = -1; } @@ -1673,8 +1695,24 @@ public class GC { * * @param packExpire * instant in time which defines packfile expiration + * @deprecated use {@link #setPackExpire(Instant)} instead */ + @Deprecated(since = "7.2") public void setPackExpire(Date packExpire) { + this.packExpire = packExpire.toInstant(); + packExpireAgeMillis = -1; + } + + /** + * During gc() or prune() packfiles which are created or modified after or + * at packExpire will not be deleted. Only older packfiles may + * be deleted. If set to null then every packfile is a candidate for + * deletion. + * + * @param packExpire + * instant in time which defines packfile expiration + */ + public void setPackExpire(Instant packExpire) { this.packExpire = packExpire; packExpireAgeMillis = -1; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/GitTimeParser.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/GitTimeParser.java index 7d00fcd5ed..acaa1ce563 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/GitTimeParser.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/GitTimeParser.java @@ -11,6 +11,7 @@ package org.eclipse.jgit.util; import java.text.MessageFormat; import java.text.ParseException; +import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; @@ -97,6 +98,40 @@ public class GitTimeParser { return parse(dateStr, SystemReader.getInstance().civilNow()); } + /** + * Parses a string into a {@link java.time.Instant} using the default + * locale. Since this parser also supports relative formats (e.g. + * "yesterday") the caller can specify the reference date. These types of + * strings can be parsed: + *
    + *
  • "never"
    • + *
  • "now"
    • + *
  • "yesterday"
    • + *
  • "(x) years|months|weeks|days|hours|minutes|seconds ago"
    + * Multiple specs can be combined like in "2 weeks 3 days ago". Instead of ' + * ' one can use '.' to separate the words
    • + *
  • "yyyy-MM-dd HH:mm:ss Z" (ISO)
    • + *
  • "EEE, dd MMM yyyy HH:mm:ss Z" (RFC)
    • + *
  • "yyyy-MM-dd"
    • + *
  • "yyyy.MM.dd"
    • + *
  • "MM/dd/yyyy",
    • + *
  • "dd.MM.yyyy"
    • + *
  • "EEE MMM dd HH:mm:ss yyyy Z" (DEFAULT)
    • + *
  • "EEE MMM dd HH:mm:ss yyyy" (LOCAL)
    • + *
+ * + * @param dateStr + * the string to be parsed + * @return the parsed {@link java.time.Instant} + * @throws java.text.ParseException + * if the given dateStr was not recognized + * @since 7.2 + */ + public static Instant parseInstant(String dateStr) throws ParseException { + return parse(dateStr).atZone(SystemReader.getInstance().getTimeZoneId()) + .toInstant(); + } + // Only tests seem to use this method static LocalDateTime parse(String dateStr, LocalDateTime now) throws ParseException { -- cgit v1.2.3 From 1b39ee910b8c8f36916468204d126664366e915d Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Sat, 28 Dec 2024 23:19:00 +0100 Subject: GcLog: replace deprecated GitDateParser with GitTimeParser We are moving from java.util.Date to java.time APIs. Change-Id: I6f99d52c2e5bd4734ac1fe5378eed29b398e8c25 --- .../src/org/eclipse/jgit/internal/storage/file/GcLog.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GcLog.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GcLog.java index 8647b3e664..862aaab0ee 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GcLog.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GcLog.java @@ -23,8 +23,7 @@ import java.time.Instant; import org.eclipse.jgit.api.errors.JGitInternalException; import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.util.FileUtils; -import org.eclipse.jgit.util.GitDateParser; -import org.eclipse.jgit.util.SystemReader; +import org.eclipse.jgit.util.GitTimeParser; /** * This class manages the gc.log file for a {@link FileRepository}. @@ -62,8 +61,7 @@ class GcLog { if (logExpiryStr == null) { logExpiryStr = LOG_EXPIRY_DEFAULT; } - gcLogExpire = GitDateParser.parse(logExpiryStr, null, - SystemReader.getInstance().getLocale()).toInstant(); + gcLogExpire = GitTimeParser.parseInstant(logExpiryStr); } return gcLogExpire; } -- cgit v1.2.3 From 0e70ee5befd80990dda96e8ced17d4d1825e1085 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Sat, 28 Dec 2024 23:26:22 +0100 Subject: RevCommit: replace deprecated RawParseUtils#headerEnd Change-Id: I4e61af621b63b0715596b5a8c7d8eb985d64ce26 --- org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevCommit.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevCommit.java b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevCommit.java index 55ddebf288..871545fca2 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevCommit.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevCommit.java @@ -401,13 +401,13 @@ public class RevCommit extends RevObject { * @since 5.1 */ public final byte[] getRawGpgSignature() { - final byte[] raw = buffer; - final byte[] header = { 'g', 'p', 'g', 's', 'i', 'g' }; - final int start = RawParseUtils.headerStart(header, raw, 0); + byte[] raw = buffer; + byte[] header = { 'g', 'p', 'g', 's', 'i', 'g' }; + int start = RawParseUtils.headerStart(header, raw, 0); if (start < 0) { return null; } - final int end = RawParseUtils.headerEnd(raw, start); + int end = RawParseUtils.nextLfSkippingSplitLines(raw, start); return RawParseUtils.headerValue(raw, start, end); } -- cgit v1.2.3 From 873648956f83931c23330d0814224410c3bdd225 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Sat, 28 Dec 2024 23:29:32 +0100 Subject: WindowCache: fix "empty control-flow statement" warning Change-Id: I61259fc76454b473aee92dc96180219e5f4a4084 --- .../src/org/eclipse/jgit/internal/storage/file/WindowCache.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCache.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCache.java index fd80faf4ed..15c125c684 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCache.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCache.java @@ -790,7 +790,9 @@ public class WindowCache { } numRemovers++; } - for (int numRemoved = 0; removeNextBlock(numRemoved); numRemoved++); + for (int numRemoved = 0; removeNextBlock(numRemoved); numRemoved++) { + // empty + } synchronized (this) { if (numRemovers > 0) { numRemovers--; -- cgit v1.2.3 From 085b24aaf416308634388038ab186d22cc51a806 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Sat, 28 Dec 2024 23:39:25 +0100 Subject: SignatureUtils: replace deprecated PersonIdent constructor Change-Id: Ibf38c44d6893a1ce3baf83f8f0dbc06d1c8405ad --- org.eclipse.jgit/src/org/eclipse/jgit/util/SignatureUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/SignatureUtils.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/SignatureUtils.java index 820ac2db91..e3e3e04fd9 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/SignatureUtils.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/SignatureUtils.java @@ -48,7 +48,7 @@ public final class SignatureUtils { if (verification.creationDate() != null) { // Use the creator's timezone for the signature date PersonIdent dateId = new PersonIdent(creator, - verification.creationDate()); + verification.creationDate().toInstant()); result.append( MessageFormat.format(JGitText.get().verifySignatureMade, formatter.formatDate(dateId))); -- cgit v1.2.3 From 7c37899eaefbe43d1ac8eeb40f27cec39f567c31 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Sun, 29 Dec 2024 00:09:34 +0100 Subject: BlockReader: use java.time instead of java.util.Date API Change-Id: Iea1147f89fb14821b7d052d4e9c3397033dfade4 --- .../org/eclipse/jgit/internal/storage/reftable/BlockReader.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/BlockReader.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/BlockReader.java index d07713db8e..e9ff02700d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/BlockReader.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/BlockReader.java @@ -32,6 +32,8 @@ import static org.eclipse.jgit.lib.Ref.Storage.PACKED; import java.io.IOException; import java.nio.ByteBuffer; +import java.time.Instant; +import java.time.ZoneOffset; import java.util.Arrays; import java.util.zip.DataFormatException; import java.util.zip.Inflater; @@ -245,9 +247,9 @@ class BlockReader { private PersonIdent readPersonIdent() { String name = readValueString(); String email = readValueString(); - long ms = readVarint64() * 1000; - int tz = readInt16(); - return new PersonIdent(name, email, ms, tz); + long epochSeconds = readVarint64(); + ZoneOffset tz = ZoneOffset.ofTotalSeconds(readInt16() * 60); + return new PersonIdent(name, email, Instant.ofEpochSecond(epochSeconds), tz); } void readBlock(BlockSource src, long pos, int fileBlockSize) -- cgit v1.2.3 From f8ee2ba845fcb5ae5dd6b462c182e49ddea34945 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Sun, 29 Dec 2024 00:20:50 +0100 Subject: RebaseCommand: use java.time instead of java.util.Date API Change-Id: I3004d5b586b674e81cc76d0b01cef424d0b1f7f8 --- .../src/org/eclipse/jgit/api/RebaseCommand.java | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/RebaseCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/RebaseCommand.java index 858bd961cd..3ae7a6c81e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/RebaseCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/RebaseCommand.java @@ -18,6 +18,8 @@ import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.text.MessageFormat; +import java.time.Instant; +import java.time.ZoneOffset; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -1835,23 +1837,26 @@ public class RebaseCommand extends GitCommand { // the time is saved as int timeStart = 0; - if (time.startsWith("@")) //$NON-NLS-1$ + if (time.startsWith("@")) { //$NON-NLS-1$ timeStart = 1; - else + } else { timeStart = 0; - long when = Long - .parseLong(time.substring(timeStart, time.indexOf(' '))) * 1000; + } + Instant when = Instant.ofEpochSecond( + Long.parseLong(time.substring(timeStart, time.indexOf(' ')))); String tzOffsetString = time.substring(time.indexOf(' ') + 1); int multiplier = -1; - if (tzOffsetString.charAt(0) == '+') + if (tzOffsetString.charAt(0) == '+') { multiplier = 1; + } int hours = Integer.parseInt(tzOffsetString.substring(1, 3)); int minutes = Integer.parseInt(tzOffsetString.substring(3, 5)); // this is in format (+/-)HHMM (hours and minutes) - // we need to convert into minutes - int tz = (hours * 60 + minutes) * multiplier; - if (name != null && email != null) + ZoneOffset tz = ZoneOffset.ofHoursMinutes(hours * multiplier, + minutes * multiplier); + if (name != null && email != null) { return new PersonIdent(name, email, when, tz); + } return null; } -- cgit v1.2.3 From e5e2e026987f9f5ae9abf25e9de366e3c59a5dd3 Mon Sep 17 00:00:00 2001 From: youtirsin Date: Wed, 8 Jan 2025 16:26:55 +0800 Subject: Set repositoryformatversion=0 when converting refStorage to files See https://git-scm.com/docs/repository-version#_version_1 Change-Id: Ic71e55f5dfd4dacedabe54a5881a56b72659f3bc Signed-off-by: youtirsin --- .../src/org/eclipse/jgit/internal/storage/file/FileRepository.java | 2 ++ 1 file changed, 2 insertions(+) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java index c5c36565d9..207ced0314 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java @@ -743,6 +743,8 @@ public class FileRepository extends Repository { } repoConfig.unset(ConfigConstants.CONFIG_EXTENSIONS_SECTION, null, ConfigConstants.CONFIG_KEY_REF_STORAGE); + repoConfig.setLong(ConfigConstants.CONFIG_CORE_SECTION, null, + ConfigConstants.CONFIG_KEY_REPO_FORMAT_VERSION, 0); repoConfig.save(); } -- cgit v1.2.3 From fda44a444ed266a6bff352526fed7b2ff3169489 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Sun, 5 Jan 2025 17:58:30 +0100 Subject: Add RefDatabase#getReflogReader methods to fix broken abstraction in FileRepository#getReflogReader(String). How to get a ReflogReader depends on the chosen RefDatabase implementation, hence the #getReflogReader methods should be there. This also fixes a bug in FileRepository#getReflogReader(Ref) which didn't work if FileReftableDatabase was used as RefDatabase since it always returned the implementation only suitable for RefDirectory. Change-Id: I0859990f9390ab96596b6c344966c687dfbbf167 --- .../jgit/lib/RefDatabaseConflictingNamesTest.java | 5 +++ org.eclipse.jgit/.settings/.api_filters | 17 ++++++++++ .../internal/storage/dfs/DfsReftableDatabase.java | 6 ++++ .../storage/file/FileReftableDatabase.java | 8 ++++- .../jgit/internal/storage/file/FileRepository.java | 28 ++--------------- .../jgit/internal/storage/file/RefDirectory.java | 6 ++++ .../src/org/eclipse/jgit/lib/RefDatabase.java | 36 ++++++++++++++++++++++ .../src/org/eclipse/jgit/lib/Repository.java | 13 ++++---- 8 files changed, 86 insertions(+), 33 deletions(-) create mode 100644 org.eclipse.jgit/.settings/.api_filters (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/RefDatabaseConflictingNamesTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/RefDatabaseConflictingNamesTest.java index b02f245865..85f9612b6a 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/RefDatabaseConflictingNamesTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/RefDatabaseConflictingNamesTest.java @@ -70,6 +70,11 @@ public class RefDatabaseConflictingNamesTest { return null; } + @Override + public ReflogReader getReflogReader(Ref ref) throws IOException { + return null; + } + @Override public void create() throws IOException { // Not needed diff --git a/org.eclipse.jgit/.settings/.api_filters b/org.eclipse.jgit/.settings/.api_filters new file mode 100644 index 0000000000..61d66582e0 --- /dev/null +++ b/org.eclipse.jgit/.settings/.api_filters @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReftableDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReftableDatabase.java index 3ba74b26fc..2751cd2969 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReftableDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReftableDatabase.java @@ -28,6 +28,7 @@ import org.eclipse.jgit.lib.BatchRefUpdate; import org.eclipse.jgit.lib.NullProgressMonitor; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.lib.ReflogReader; import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.transport.ReceiveCommand; import org.eclipse.jgit.util.RefList; @@ -176,6 +177,11 @@ public class DfsReftableDatabase extends DfsRefDatabase { return reftableDatabase.getRefsByPrefixWithExclusions(include, excludes); } + @Override + public ReflogReader getReflogReader(Ref ref) throws IOException { + return reftableDatabase.getReflogReader(ref.getName()); + } + @Override public Set getTipsWithSha1(ObjectId id) throws IOException { if (!getReftableConfig().isIndexObjects()) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java index 25b7583b95..81e8c0306d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java @@ -90,7 +90,13 @@ public class FileReftableDatabase extends RefDatabase { }; } - ReflogReader getReflogReader(String refname) throws IOException { + @Override + public ReflogReader getReflogReader(Ref ref) throws IOException { + return reftableDatabase.getReflogReader(ref.getName()); + } + + @Override + public ReflogReader getReflogReader(String refname) throws IOException { return reftableDatabase.getReflogReader(refname); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java index 207ced0314..bcf9f1efdf 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileRepository.java @@ -31,7 +31,6 @@ import java.util.Locale; import java.util.Objects; import java.util.Set; -import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.api.errors.JGitInternalException; @@ -542,29 +541,6 @@ public class FileRepository extends Repository { fireEvent(new IndexChangedEvent(internal)); } - @Override - public ReflogReader getReflogReader(String refName) throws IOException { - if (refs instanceof FileReftableDatabase) { - // Cannot use findRef: reftable stores log data for deleted or renamed - // branches. - return ((FileReftableDatabase)refs).getReflogReader(refName); - } - - // TODO: use exactRef here, which offers more predictable and therefore preferable - // behavior. - Ref ref = findRef(refName); - if (ref == null) { - return null; - } - return new ReflogReaderImpl(this, ref.getName()); - } - - @Override - public @NonNull ReflogReader getReflogReader(@NonNull Ref ref) - throws IOException { - return new ReflogReaderImpl(this, ref.getName()); - } - @Override public AttributesNodeProvider createAttributesNodeProvider() { return new AttributesNodeProviderImpl(this); @@ -697,8 +673,8 @@ public class FileRepository extends Repository { } if (writeLogs) { - List logs = oldDb.getReflogReader(r.getName()) - .getReverseEntries(); + ReflogReader reflogReader = oldDb.getReflogReader(r); + List logs = reflogReader.getReverseEntries(); Collections.reverse(logs); for (ReflogEntry e : logs) { logWriter.log(r.getName(), e); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java index 6aa1157e37..33996519f7 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java @@ -76,6 +76,7 @@ import org.eclipse.jgit.lib.RefComparator; import org.eclipse.jgit.lib.RefDatabase; import org.eclipse.jgit.lib.RefUpdate; import org.eclipse.jgit.lib.RefWriter; +import org.eclipse.jgit.lib.ReflogReader; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.lib.SymbolicRef; import org.eclipse.jgit.revwalk.RevObject; @@ -456,6 +457,11 @@ public class RefDirectory extends RefDatabase { return ret; } + @Override + public ReflogReader getReflogReader(Ref ref) throws IOException { + return new ReflogReaderImpl(getRepository(), ref.getName()); + } + @SuppressWarnings("unchecked") private RefList upcast(RefList loose) { return (RefList) loose; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefDatabase.java index 09cb5a83dd..11d1732c77 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefDatabase.java @@ -355,6 +355,42 @@ public abstract class RefDatabase { return getRefsByPrefix(ALL); } + /** + * Get the reflog reader + * + * @param refName + * a {@link java.lang.String} object. + * @return a {@link org.eclipse.jgit.lib.ReflogReader} for the supplied + * refname, or {@code null} if the named ref does not exist. + * @throws java.io.IOException + * the ref could not be accessed. + * @since 7.2 + */ + @Nullable + public ReflogReader getReflogReader(String refName) throws IOException { + // TODO: use exactRef here, which offers more predictable and therefore + // preferable behavior. + Ref ref = findRef(refName); + if (ref == null) { + return null; + } + return getReflogReader(ref); + } + + /** + * Get the reflog reader. + * + * @param ref + * a Ref + * @return a {@link org.eclipse.jgit.lib.ReflogReader} for the supplied ref. + * @throws IOException + * if an IO error occurred + * @since 7.2 + */ + @NonNull + public abstract ReflogReader getReflogReader(@NonNull Ref ref) + throws IOException; + /** * Get a section of the reference namespace. * diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java index 0562840915..a7bba23996 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java @@ -1694,8 +1694,9 @@ public abstract class Repository implements AutoCloseable { * @since 3.0 */ @Nullable - public abstract ReflogReader getReflogReader(String refName) - throws IOException; + public ReflogReader getReflogReader(String refName) throws IOException { + return getRefDatabase().getReflogReader(refName); + } /** * Get the reflog reader. Subclasses should override this method and provide @@ -1703,15 +1704,15 @@ public abstract class Repository implements AutoCloseable { * * @param ref * a Ref - * @return a {@link org.eclipse.jgit.lib.ReflogReader} for the supplied ref, - * or {@code null} if the ref does not exist. + * @return a {@link org.eclipse.jgit.lib.ReflogReader} for the supplied ref. * @throws IOException * if an IO error occurred * @since 5.13.2 */ - public @Nullable ReflogReader getReflogReader(@NonNull Ref ref) + @NonNull + public ReflogReader getReflogReader(@NonNull Ref ref) throws IOException { - return getReflogReader(ref.getName()); + return getRefDatabase().getReflogReader(ref); } /** -- cgit v1.2.3 From e93294f013fcc76a8e20684e277833394be93c6d Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Sun, 5 Jan 2025 18:48:44 +0100 Subject: Deprecate Repository#getReflogReader methods instead use Repository.getRefDatabase().getReflogReader(). Change-Id: I5e66a512c12e11d0ec3275fffde4adb8483430f2 --- .../jgit/http/test/SmartClientSmartServerTest.java | 3 +- .../eclipse/jgit/api/CherryPickCommandTest.java | 6 +- .../tst/org/eclipse/jgit/api/CloneCommandTest.java | 3 +- .../eclipse/jgit/api/CommitAndLogCommandTest.java | 11 +- .../org/eclipse/jgit/api/CommitCommandTest.java | 19 ++- .../tst/org/eclipse/jgit/api/FetchCommandTest.java | 4 +- .../tst/org/eclipse/jgit/api/MergeCommandTest.java | 59 ++++---- .../org/eclipse/jgit/api/RebaseCommandTest.java | 49 ++++--- .../tst/org/eclipse/jgit/api/ResetCommandTest.java | 27 ++-- .../org/eclipse/jgit/api/RevertCommandTest.java | 29 ++-- .../eclipse/jgit/api/StashCreateCommandTest.java | 4 +- .../org/eclipse/jgit/api/StashDropCommandTest.java | 20 +-- .../internal/storage/file/BatchRefUpdateTest.java | 2 +- .../internal/storage/file/FileReftableTest.java | 31 +++-- .../jgit/internal/storage/file/RefUpdateTest.java | 150 +++++++++++++-------- .../internal/storage/file/ReflogReaderTest.java | 27 ++-- .../tst/org/eclipse/jgit/lib/ReflogConfigTest.java | 19 +-- .../src/org/eclipse/jgit/api/ReflogCommand.java | 2 +- .../src/org/eclipse/jgit/api/StashDropCommand.java | 3 +- .../storage/file/FileReftableDatabase.java | 5 +- .../org/eclipse/jgit/internal/storage/file/GC.java | 2 +- .../src/org/eclipse/jgit/lib/Repository.java | 4 + 22 files changed, 288 insertions(+), 191 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/SmartClientSmartServerTest.java b/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/SmartClientSmartServerTest.java index 850e895790..b0d17adb3a 100644 --- a/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/SmartClientSmartServerTest.java +++ b/org.eclipse.jgit.http.test/tst/org/eclipse/jgit/http/test/SmartClientSmartServerTest.java @@ -1728,7 +1728,8 @@ public class SmartClientSmartServerTest extends AllProtocolsHttpTestCase { assertEquals(Q, remoteRepository.exactRef(dstName).getObjectId()); fsck(remoteRepository, Q); - final ReflogReader log = remoteRepository.getReflogReader(dstName); + final ReflogReader log = remoteRepository.getRefDatabase() + .getReflogReader(dstName); assertNotNull("has log for " + dstName, log); final ReflogEntry last = log.getLastEntry(); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CherryPickCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CherryPickCommandTest.java index be3b33a9c5..a9bd126ff3 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CherryPickCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CherryPickCommandTest.java @@ -34,6 +34,7 @@ import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.FileMode; import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.RefDatabase; import org.eclipse.jgit.lib.ReflogReader; import org.eclipse.jgit.lib.RepositoryState; import org.eclipse.jgit.merge.ContentMergeStrategy; @@ -529,10 +530,11 @@ public class CherryPickCommandTest extends RepositoryTestCase { assertEquals(RepositoryState.SAFE, db.getRepositoryState()); if (reason == null) { - ReflogReader reader = db.getReflogReader(Constants.HEAD); + RefDatabase refDb = db.getRefDatabase(); + ReflogReader reader = refDb.getReflogReader(Constants.HEAD); assertTrue(reader.getLastEntry().getComment() .startsWith("cherry-pick: ")); - reader = db.getReflogReader(db.getBranch()); + reader = refDb.getReflogReader(db.getBranch()); assertTrue(reader.getLastEntry().getComment() .startsWith("cherry-pick: ")); } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CloneCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CloneCommandTest.java index db5b27c2ab..f70ada4e30 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CloneCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CloneCommandTest.java @@ -182,7 +182,8 @@ public class CloneCommandTest extends RepositoryTestCase { private static boolean hasRefLog(Repository repo, Ref ref) { try { - return repo.getReflogReader(ref.getName()).getLastEntry() != null; + return repo.getRefDatabase().getReflogReader(ref.getName()) + .getLastEntry() != null; } catch (IOException ioe) { throw new IllegalStateException(ioe); } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CommitAndLogCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CommitAndLogCommandTest.java index 57e5d4958f..ced24b5928 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CommitAndLogCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CommitAndLogCommandTest.java @@ -26,6 +26,7 @@ import org.eclipse.jgit.junit.RepositoryTestCase; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.PersonIdent; +import org.eclipse.jgit.lib.RefDatabase; import org.eclipse.jgit.lib.RefUpdate; import org.eclipse.jgit.lib.ReflogReader; import org.eclipse.jgit.revwalk.RevCommit; @@ -69,10 +70,11 @@ public class CommitAndLogCommandTest extends RepositoryTestCase { l--; } assertEquals(l, -1); - ReflogReader reader = db.getReflogReader(Constants.HEAD); + RefDatabase refDb = db.getRefDatabase(); + ReflogReader reader = refDb.getReflogReader(Constants.HEAD); assertTrue( reader.getLastEntry().getComment().startsWith("commit:")); - reader = db.getReflogReader(db.getBranch()); + reader = refDb.getReflogReader(db.getBranch()); assertTrue( reader.getLastEntry().getComment().startsWith("commit:")); } @@ -248,10 +250,11 @@ public class CommitAndLogCommandTest extends RepositoryTestCase { c++; } assertEquals(1, c); - ReflogReader reader = db.getReflogReader(Constants.HEAD); + RefDatabase refDb = db.getRefDatabase(); + ReflogReader reader = refDb.getReflogReader(Constants.HEAD); assertTrue(reader.getLastEntry().getComment() .startsWith("commit (amend):")); - reader = db.getReflogReader(db.getBranch()); + reader = refDb.getReflogReader(db.getBranch()); assertTrue(reader.getLastEntry().getComment() .startsWith("commit (amend):")); } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CommitCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CommitCommandTest.java index 2cbd223ee8..d40df26690 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CommitCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CommitCommandTest.java @@ -435,10 +435,12 @@ public class CommitCommandTest extends RepositoryTestCase { assertEquals(1, squashedCommit.getParentCount()); assertNull(db.readSquashCommitMsg()); - assertEquals("commit: Squashed commit of the following:", db - .getReflogReader(Constants.HEAD).getLastEntry().getComment()); - assertEquals("commit: Squashed commit of the following:", db - .getReflogReader(db.getBranch()).getLastEntry().getComment()); + assertEquals("commit: Squashed commit of the following:", + db.getRefDatabase().getReflogReader(Constants.HEAD) + .getLastEntry().getComment()); + assertEquals("commit: Squashed commit of the following:", + db.getRefDatabase().getReflogReader(db.getBranch()) + .getLastEntry().getComment()); } } @@ -455,12 +457,15 @@ public class CommitCommandTest extends RepositoryTestCase { git.commit().setMessage("c3").setAll(true) .setReflogComment("testRl").call(); - db.getReflogReader(Constants.HEAD).getReverseEntries(); + db.getRefDatabase().getReflogReader(Constants.HEAD) + .getReverseEntries(); assertEquals("testRl;commit (initial): c1;", reflogComments( - db.getReflogReader(Constants.HEAD).getReverseEntries())); + db.getRefDatabase().getReflogReader(Constants.HEAD) + .getReverseEntries())); assertEquals("testRl;commit (initial): c1;", reflogComments( - db.getReflogReader(db.getBranch()).getReverseEntries())); + db.getRefDatabase().getReflogReader(db.getBranch()) + .getReverseEntries())); } } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/FetchCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/FetchCommandTest.java index 3ec454cfc3..3731347f11 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/FetchCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/FetchCommandTest.java @@ -92,8 +92,8 @@ public class FetchCommandTest extends RepositoryTestCase { assertTrue(remoteRef.getName().startsWith(Constants.R_REMOTES)); assertEquals(defaultBranchSha1, remoteRef.getObjectId()); - assertNotNull(git.getRepository().getReflogReader(remoteRef.getName()) - .getLastEntry()); + assertNotNull(git.getRepository().getRefDatabase() + .getReflogReader(remoteRef.getName()).getLastEntry()); } @Test diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/MergeCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/MergeCommandTest.java index 917b6c3297..ba13f8ac1f 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/MergeCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/MergeCommandTest.java @@ -33,6 +33,7 @@ import org.eclipse.jgit.junit.TestRepository; import org.eclipse.jgit.junit.TestRepository.BranchBuilder; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.lib.RefDatabase; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.lib.RepositoryState; import org.eclipse.jgit.lib.Sets; @@ -76,11 +77,10 @@ public class MergeCommandTest extends RepositoryTestCase { assertEquals(MergeResult.MergeStatus.ALREADY_UP_TO_DATE, result.getMergeStatus()); } // no reflog entry written by merge - assertEquals("commit (initial): initial commit", - db + RefDatabase refDb = db.getRefDatabase(); + assertEquals("commit (initial): initial commit", refDb .getReflogReader(Constants.HEAD).getLastEntry().getComment()); - assertEquals("commit (initial): initial commit", - db + assertEquals("commit (initial): initial commit", refDb .getReflogReader(db.getBranch()).getLastEntry().getComment()); } @@ -96,9 +96,9 @@ public class MergeCommandTest extends RepositoryTestCase { assertEquals(second, result.getNewHead()); } // no reflog entry written by merge - assertEquals("commit: second commit", db + assertEquals("commit: second commit", db.getRefDatabase() .getReflogReader(Constants.HEAD).getLastEntry().getComment()); - assertEquals("commit: second commit", db + assertEquals("commit: second commit", db.getRefDatabase() .getReflogReader(db.getBranch()).getLastEntry().getComment()); } @@ -117,10 +117,13 @@ public class MergeCommandTest extends RepositoryTestCase { assertEquals(MergeResult.MergeStatus.FAST_FORWARD, result.getMergeStatus()); assertEquals(second, result.getNewHead()); } + RefDatabase refDb = db.getRefDatabase(); assertEquals("merge refs/heads/master: Fast-forward", - db.getReflogReader(Constants.HEAD).getLastEntry().getComment()); + refDb.getReflogReader(Constants.HEAD) + .getLastEntry().getComment()); assertEquals("merge refs/heads/master: Fast-forward", - db.getReflogReader(db.getBranch()).getLastEntry().getComment()); + refDb.getReflogReader(db.getBranch()) + .getLastEntry().getComment()); } @Test @@ -140,9 +143,10 @@ public class MergeCommandTest extends RepositoryTestCase { result.getMergeStatus()); assertEquals(second, result.getNewHead()); } - assertEquals("merge refs/heads/master: Fast-forward", db + RefDatabase refDb = db.getRefDatabase(); + assertEquals("merge refs/heads/master: Fast-forward", refDb .getReflogReader(Constants.HEAD).getLastEntry().getComment()); - assertEquals("merge refs/heads/master: Fast-forward", db + assertEquals("merge refs/heads/master: Fast-forward", refDb .getReflogReader(db.getBranch()).getLastEntry().getComment()); } @@ -171,10 +175,11 @@ public class MergeCommandTest extends RepositoryTestCase { assertEquals(MergeResult.MergeStatus.FAST_FORWARD, result.getMergeStatus()); assertEquals(second, result.getNewHead()); } - assertEquals("merge refs/heads/master: Fast-forward", - db.getReflogReader(Constants.HEAD).getLastEntry().getComment()); - assertEquals("merge refs/heads/master: Fast-forward", - db.getReflogReader(db.getBranch()).getLastEntry().getComment()); + RefDatabase refDb = db.getRefDatabase(); + assertEquals("merge refs/heads/master: Fast-forward", refDb + .getReflogReader(Constants.HEAD).getLastEntry().getComment()); + assertEquals("merge refs/heads/master: Fast-forward", refDb + .getReflogReader(db.getBranch()).getLastEntry().getComment()); } @Test @@ -229,14 +234,17 @@ public class MergeCommandTest extends RepositoryTestCase { .include(db.exactRef(R_HEADS + MASTER)).call(); assertEquals(MergeStatus.MERGED, result.getMergeStatus()); } + RefDatabase refDb = db.getRefDatabase(); assertEquals( "merge refs/heads/master: Merge made by " + mergeStrategy.getName() + ".", - db.getReflogReader(Constants.HEAD).getLastEntry().getComment()); + refDb.getReflogReader(Constants.HEAD).getLastEntry() + .getComment()); assertEquals( "merge refs/heads/master: Merge made by " + mergeStrategy.getName() + ".", - db.getReflogReader(db.getBranch()).getLastEntry().getComment()); + refDb.getReflogReader(db.getBranch()).getLastEntry() + .getComment()); } @Theory @@ -662,14 +670,17 @@ public class MergeCommandTest extends RepositoryTestCase { .setStrategy(MergeStrategy.RESOLVE).call(); assertEquals(MergeStatus.MERGED, result.getMergeStatus()); assertEquals("1\nb(1)\n3\n", read(new File(db.getWorkTree(), "b"))); - assertEquals("merge " + secondCommit.getId().getName() - + ": Merge made by resolve.", db - .getReflogReader(Constants.HEAD) - .getLastEntry().getComment()); - assertEquals("merge " + secondCommit.getId().getName() - + ": Merge made by resolve.", db - .getReflogReader(db.getBranch()) - .getLastEntry().getComment()); + RefDatabase refDb = db.getRefDatabase(); + assertEquals( + "merge " + secondCommit.getId().getName() + + ": Merge made by resolve.", + refDb.getReflogReader(Constants.HEAD).getLastEntry() + .getComment()); + assertEquals( + "merge " + secondCommit.getId().getName() + + ": Merge made by resolve.", + refDb.getReflogReader(db.getBranch()).getLastEntry() + .getComment()); } } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/RebaseCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/RebaseCommandTest.java index 84aa4b58a8..4c8cf06a67 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/RebaseCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/RebaseCommandTest.java @@ -57,6 +57,7 @@ import org.eclipse.jgit.lib.ObjectLoader; import org.eclipse.jgit.lib.PersonIdent; import org.eclipse.jgit.lib.RebaseTodoLine; import org.eclipse.jgit.lib.RebaseTodoLine.Action; +import org.eclipse.jgit.lib.RefDatabase; import org.eclipse.jgit.lib.RefUpdate; import org.eclipse.jgit.lib.ReflogEntry; import org.eclipse.jgit.lib.RepositoryState; @@ -133,11 +134,12 @@ public class RebaseCommandTest extends RepositoryTestCase { checkFile(file2, "file2"); assertEquals(Status.FAST_FORWARD, res.getStatus()); - List headLog = db.getReflogReader(Constants.HEAD) + RefDatabase refDb = db.getRefDatabase(); + List headLog = refDb.getReflogReader(Constants.HEAD) .getReverseEntries(); - List topicLog = db.getReflogReader("refs/heads/topic") + List topicLog = refDb.getReflogReader("refs/heads/topic") .getReverseEntries(); - List masterLog = db.getReflogReader("refs/heads/master") + List masterLog = refDb.getReflogReader("refs/heads/master") .getReverseEntries(); assertEquals("rebase finished: returning to refs/heads/topic", headLog .get(0).getComment()); @@ -179,11 +181,12 @@ public class RebaseCommandTest extends RepositoryTestCase { checkFile(file2, "file2 new content"); assertEquals(Status.FAST_FORWARD, res.getStatus()); - List headLog = db.getReflogReader(Constants.HEAD) + RefDatabase refDb = db.getRefDatabase(); + List headLog = refDb.getReflogReader(Constants.HEAD) .getReverseEntries(); - List topicLog = db.getReflogReader("refs/heads/topic") + List topicLog = refDb.getReflogReader("refs/heads/topic") .getReverseEntries(); - List masterLog = db.getReflogReader("refs/heads/master") + List masterLog = refDb.getReflogReader("refs/heads/master") .getReverseEntries(); assertEquals("rebase finished: returning to refs/heads/topic", headLog .get(0).getComment()); @@ -447,13 +450,14 @@ public class RebaseCommandTest extends RepositoryTestCase { assertEquals(a, rw.next()); } - List headLog = db.getReflogReader(Constants.HEAD) + RefDatabase refDb = db.getRefDatabase(); + List headLog = refDb.getReflogReader(Constants.HEAD) .getReverseEntries(); - List sideLog = db.getReflogReader("refs/heads/side") + List sideLog = refDb.getReflogReader("refs/heads/side") .getReverseEntries(); - List topicLog = db.getReflogReader("refs/heads/topic") + List topicLog = refDb.getReflogReader("refs/heads/topic") .getReverseEntries(); - List masterLog = db.getReflogReader("refs/heads/master") + List masterLog = refDb.getReflogReader("refs/heads/master") .getReverseEntries(); assertEquals("rebase finished: returning to refs/heads/topic", headLog .get(0).getComment()); @@ -768,9 +772,10 @@ public class RebaseCommandTest extends RepositoryTestCase { RebaseResult result = git.rebase().setUpstream(parent).call(); assertEquals(Status.UP_TO_DATE, result.getStatus()); - assertEquals(2, db.getReflogReader(Constants.HEAD).getReverseEntries() - .size()); - assertEquals(2, db.getReflogReader("refs/heads/master") + RefDatabase refDb = db.getRefDatabase(); + assertEquals(2, refDb.getReflogReader(Constants.HEAD) + .getReverseEntries().size()); + assertEquals(2, refDb.getReflogReader("refs/heads/master") .getReverseEntries().size()); } @@ -786,9 +791,10 @@ public class RebaseCommandTest extends RepositoryTestCase { RebaseResult res = git.rebase().setUpstream(first).call(); assertEquals(Status.UP_TO_DATE, res.getStatus()); - assertEquals(1, db.getReflogReader(Constants.HEAD).getReverseEntries() - .size()); - assertEquals(1, db.getReflogReader("refs/heads/master") + RefDatabase refDb = db.getRefDatabase(); + assertEquals(1, refDb.getReflogReader(Constants.HEAD) + .getReverseEntries().size()); + assertEquals(1, refDb.getReflogReader("refs/heads/master") .getReverseEntries().size()); } @@ -846,11 +852,12 @@ public class RebaseCommandTest extends RepositoryTestCase { db.resolve(Constants.HEAD)).getParent(0)); } assertEquals(origHead, db.readOrigHead()); - List headLog = db.getReflogReader(Constants.HEAD) + RefDatabase refDb = db.getRefDatabase(); + List headLog = refDb.getReflogReader(Constants.HEAD) .getReverseEntries(); - List topicLog = db.getReflogReader("refs/heads/topic") + List topicLog = refDb.getReflogReader("refs/heads/topic") .getReverseEntries(); - List masterLog = db.getReflogReader("refs/heads/master") + List masterLog = refDb.getReflogReader("refs/heads/master") .getReverseEntries(); assertEquals(2, masterLog.size()); assertEquals(3, topicLog.size()); @@ -898,8 +905,8 @@ public class RebaseCommandTest extends RepositoryTestCase { db.resolve(Constants.HEAD)).getParent(0)); } - List headLog = db.getReflogReader(Constants.HEAD) - .getReverseEntries(); + List headLog = db.getRefDatabase() + .getReflogReader(Constants.HEAD).getReverseEntries(); assertEquals(8, headLog.size()); assertEquals("rebase: change file1 in topic", headLog.get(0) .getComment()); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/ResetCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/ResetCommandTest.java index 8a479a0ca0..4265806078 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/ResetCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/ResetCommandTest.java @@ -36,6 +36,7 @@ import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.FileMode; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.lib.RefDatabase; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.treewalk.TreeWalk; @@ -557,43 +558,45 @@ public class ResetCommandTest extends RepositoryTestCase { private void assertReflog(ObjectId prevHead, ObjectId head) throws IOException { // Check the reflog for HEAD - String actualHeadMessage = db.getReflogReader(Constants.HEAD) + RefDatabase refDb = db.getRefDatabase(); + String actualHeadMessage = refDb.getReflogReader(Constants.HEAD) .getLastEntry().getComment(); String expectedHeadMessage = head.getName() + ": updating HEAD"; assertEquals(expectedHeadMessage, actualHeadMessage); - assertEquals(head.getName(), db.getReflogReader(Constants.HEAD) + assertEquals(head.getName(), refDb.getReflogReader(Constants.HEAD) .getLastEntry().getNewId().getName()); - assertEquals(prevHead.getName(), db.getReflogReader(Constants.HEAD) + assertEquals(prevHead.getName(), refDb.getReflogReader(Constants.HEAD) .getLastEntry().getOldId().getName()); // The reflog for master contains the same as the one for HEAD - String actualMasterMessage = db.getReflogReader("refs/heads/master") + String actualMasterMessage = refDb.getReflogReader("refs/heads/master") .getLastEntry().getComment(); String expectedMasterMessage = head.getName() + ": updating HEAD"; // yes! assertEquals(expectedMasterMessage, actualMasterMessage); - assertEquals(head.getName(), db.getReflogReader(Constants.HEAD) + assertEquals(head.getName(), refDb.getReflogReader(Constants.HEAD) .getLastEntry().getNewId().getName()); - assertEquals(prevHead.getName(), db - .getReflogReader("refs/heads/master").getLastEntry().getOldId() - .getName()); + assertEquals(prevHead.getName(), + refDb.getReflogReader("refs/heads/master").getLastEntry() + .getOldId().getName()); } private void assertReflogDisabled(ObjectId head) throws IOException { + RefDatabase refDb = db.getRefDatabase(); // Check the reflog for HEAD - String actualHeadMessage = db.getReflogReader(Constants.HEAD) + String actualHeadMessage = refDb.getReflogReader(Constants.HEAD) .getLastEntry().getComment(); String expectedHeadMessage = "commit: adding a.txt and dir/b.txt"; assertEquals(expectedHeadMessage, actualHeadMessage); - assertEquals(head.getName(), db.getReflogReader(Constants.HEAD) + assertEquals(head.getName(), refDb.getReflogReader(Constants.HEAD) .getLastEntry().getOldId().getName()); // The reflog for master contains the same as the one for HEAD - String actualMasterMessage = db.getReflogReader("refs/heads/master") + String actualMasterMessage = refDb.getReflogReader("refs/heads/master") .getLastEntry().getComment(); String expectedMasterMessage = "commit: adding a.txt and dir/b.txt"; assertEquals(expectedMasterMessage, actualMasterMessage); - assertEquals(head.getName(), db.getReflogReader(Constants.HEAD) + assertEquals(head.getName(), refDb.getReflogReader(Constants.HEAD) .getLastEntry().getOldId().getName()); } /** diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/RevertCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/RevertCommandTest.java index afd6708d21..e4d0eb76f7 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/RevertCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/RevertCommandTest.java @@ -29,6 +29,7 @@ import org.eclipse.jgit.junit.RepositoryTestCase; import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.FileMode; +import org.eclipse.jgit.lib.RefDatabase; import org.eclipse.jgit.lib.ReflogReader; import org.eclipse.jgit.lib.RepositoryState; import org.eclipse.jgit.merge.ResolveMerger.MergeFailureReason; @@ -87,10 +88,11 @@ public class RevertCommandTest extends RepositoryTestCase { assertEquals("create a", history.next().getFullMessage()); assertFalse(history.hasNext()); - ReflogReader reader = db.getReflogReader(Constants.HEAD); + RefDatabase refDb = db.getRefDatabase(); + ReflogReader reader = refDb.getReflogReader(Constants.HEAD); assertTrue(reader.getLastEntry().getComment() .startsWith("revert: Revert \"")); - reader = db.getReflogReader(db.getBranch()); + reader = refDb.getReflogReader(db.getBranch()); assertTrue(reader.getLastEntry().getComment() .startsWith("revert: Revert \"")); } @@ -170,10 +172,11 @@ public class RevertCommandTest extends RepositoryTestCase { assertEquals("add first", history.next().getFullMessage()); assertFalse(history.hasNext()); - ReflogReader reader = db.getReflogReader(Constants.HEAD); + RefDatabase refDb = db.getRefDatabase(); + ReflogReader reader = refDb.getReflogReader(Constants.HEAD); assertTrue(reader.getLastEntry().getComment() .startsWith("revert: Revert \"")); - reader = db.getReflogReader(db.getBranch()); + reader = refDb.getReflogReader(db.getBranch()); assertTrue(reader.getLastEntry().getComment() .startsWith("revert: Revert \"")); } @@ -223,10 +226,11 @@ public class RevertCommandTest extends RepositoryTestCase { assertEquals("add first", history.next().getFullMessage()); assertFalse(history.hasNext()); - ReflogReader reader = db.getReflogReader(Constants.HEAD); + RefDatabase refDb = db.getRefDatabase(); + ReflogReader reader = refDb.getReflogReader(Constants.HEAD); assertTrue(reader.getLastEntry().getComment() .startsWith("revert: Revert \"")); - reader = db.getReflogReader(db.getBranch()); + reader = refDb.getReflogReader(db.getBranch()); assertTrue(reader.getLastEntry().getComment() .startsWith("revert: Revert \"")); } @@ -431,12 +435,13 @@ public class RevertCommandTest extends RepositoryTestCase { assertEquals(RepositoryState.SAFE, db.getRepositoryState()); if (reason == null) { - ReflogReader reader = db.getReflogReader(Constants.HEAD); - assertTrue(reader.getLastEntry().getComment() - .startsWith("revert: ")); - reader = db.getReflogReader(db.getBranch()); - assertTrue(reader.getLastEntry().getComment() - .startsWith("revert: ")); + RefDatabase refDb = db.getRefDatabase(); + ReflogReader reader = refDb.getReflogReader(Constants.HEAD); + assertTrue( + reader.getLastEntry().getComment().startsWith("revert: ")); + reader = refDb.getReflogReader(db.getBranch()); + assertTrue( + reader.getLastEntry().getComment().startsWith("revert: ")); } } } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/StashCreateCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/StashCreateCommandTest.java index 5d0ab05174..18cd21a5d7 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/StashCreateCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/StashCreateCommandTest.java @@ -409,8 +409,8 @@ public class StashCreateCommandTest extends RepositoryTestCase { assertEquals("content", read(committedFile)); validateStashedCommit(stashed); - ReflogReader reader = git.getRepository().getReflogReader( - Constants.R_STASH); + ReflogReader reader = git.getRepository().getRefDatabase() + .getReflogReader(Constants.R_STASH); ReflogEntry entry = reader.getLastEntry(); assertNotNull(entry); assertEquals(ObjectId.zeroId(), entry.getOldId()); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/StashDropCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/StashDropCommandTest.java index c81731d746..d937579283 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/StashDropCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/StashDropCommandTest.java @@ -92,8 +92,8 @@ public class StashDropCommandTest extends RepositoryTestCase { stashRef = git.getRepository().exactRef(Constants.R_STASH); assertNull(stashRef); - ReflogReader reader = git.getRepository().getReflogReader( - Constants.R_STASH); + ReflogReader reader = git.getRepository().getRefDatabase() + .getReflogReader(Constants.R_STASH); assertNull(reader); } @@ -120,8 +120,8 @@ public class StashDropCommandTest extends RepositoryTestCase { assertNull(git.stashDrop().setAll(true).call()); assertNull(git.getRepository().exactRef(Constants.R_STASH)); - ReflogReader reader = git.getRepository().getReflogReader( - Constants.R_STASH); + ReflogReader reader = git.getRepository().getRefDatabase() + .getReflogReader(Constants.R_STASH); assertNull(reader); } @@ -150,8 +150,8 @@ public class StashDropCommandTest extends RepositoryTestCase { assertNotNull(stashRef); assertEquals(firstStash, stashRef.getObjectId()); - ReflogReader reader = git.getRepository().getReflogReader( - Constants.R_STASH); + ReflogReader reader = git.getRepository().getRefDatabase() + .getReflogReader(Constants.R_STASH); List entries = reader.getReverseEntries(); assertEquals(1, entries.size()); assertEquals(ObjectId.zeroId(), entries.get(0).getOldId()); @@ -192,8 +192,8 @@ public class StashDropCommandTest extends RepositoryTestCase { assertNotNull(stashRef); assertEquals(thirdStash, stashRef.getObjectId()); - ReflogReader reader = git.getRepository().getReflogReader( - Constants.R_STASH); + ReflogReader reader = git.getRepository().getRefDatabase() + .getReflogReader(Constants.R_STASH); List entries = reader.getReverseEntries(); assertEquals(2, entries.size()); assertEquals(ObjectId.zeroId(), entries.get(1).getOldId()); @@ -250,8 +250,8 @@ public class StashDropCommandTest extends RepositoryTestCase { assertNotNull(stashRef); assertEquals(thirdStash, stashRef.getObjectId()); - ReflogReader reader = git.getRepository().getReflogReader( - Constants.R_STASH); + ReflogReader reader = git.getRepository().getRefDatabase() + .getReflogReader(Constants.R_STASH); List entries = reader.getReverseEntries(); assertEquals(2, entries.size()); assertEquals(ObjectId.zeroId(), entries.get(1).getOldId()); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/BatchRefUpdateTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/BatchRefUpdateTest.java index 1af42cb229..a0afc3ef13 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/BatchRefUpdateTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/BatchRefUpdateTest.java @@ -1263,7 +1263,7 @@ public class BatchRefUpdateTest extends LocalDiskRepositoryTestCase { } private ReflogEntry getLastReflog(String name) throws IOException { - ReflogReader r = diskRepo.getReflogReader(name); + ReflogReader r = diskRepo.getRefDatabase().getReflogReader(name); if (r == null) { return null; } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileReftableTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileReftableTest.java index 01cbcfd20f..758ee93c30 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileReftableTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileReftableTest.java @@ -171,7 +171,8 @@ public class FileReftableTest extends SampleDataRepositoryTestCase { v.update(); db.convertToPackedRefs(true, false); - List logs = db.getReflogReader("refs/heads/master").getReverseEntries(2); + List logs = db.getRefDatabase() + .getReflogReader("refs/heads/master").getReverseEntries(2); assertEquals("banana", logs.get(0).getComment()); assertEquals("apple", logs.get(1).getComment()); } @@ -185,8 +186,9 @@ public class FileReftableTest extends SampleDataRepositoryTestCase { ReceiveCommand rc1 = new ReceiveCommand(ObjectId.zeroId(), cur, "refs/heads/batch1"); ReceiveCommand rc2 = new ReceiveCommand(ObjectId.zeroId(), prev, "refs/heads/batch2"); String msg = "message"; + RefDatabase refDb = db.getRefDatabase(); try (RevWalk rw = new RevWalk(db)) { - db.getRefDatabase().newBatchUpdate() + refDb.newBatchUpdate() .addCommand(rc1, rc2) .setAtomic(true) .setRefLogIdent(person) @@ -197,12 +199,14 @@ public class FileReftableTest extends SampleDataRepositoryTestCase { assertEquals(ReceiveCommand.Result.OK, rc1.getResult()); assertEquals(ReceiveCommand.Result.OK, rc2.getResult()); - ReflogEntry e = db.getReflogReader("refs/heads/batch1").getLastEntry(); + ReflogEntry e = refDb.getReflogReader("refs/heads/batch1") + .getLastEntry(); assertEquals(msg, e.getComment()); assertEquals(person, e.getWho()); assertEquals(cur, e.getNewId()); - e = db.getReflogReader("refs/heads/batch2").getLastEntry(); + e = refDb.getReflogReader("refs/heads/batch2") + .getLastEntry(); assertEquals(msg, e.getComment()); assertEquals(person, e.getWho()); assertEquals(prev, e.getNewId()); @@ -309,7 +313,7 @@ public class FileReftableTest extends SampleDataRepositoryTestCase { // the branch HEAD referred to is left untouched assertEquals(pid, db.resolve("refs/heads/master")); - ReflogReader reflogReader = db.getReflogReader("HEAD"); + ReflogReader reflogReader = db.getRefDatabase().getReflogReader("HEAD"); ReflogEntry e = reflogReader.getReverseEntries().get(0); assertEquals(ppid, e.getNewId()); assertEquals("GIT_COMMITTER_EMAIL", e.getWho().getEmailAddress()); @@ -330,7 +334,8 @@ public class FileReftableTest extends SampleDataRepositoryTestCase { updateRef.setForceUpdate(true); RefUpdate.Result update = updateRef.update(); assertEquals(FORCED, update); // internal - ReflogReader r = db.getReflogReader("refs/heads/master"); + ReflogReader r = db.getRefDatabase() + .getReflogReader("refs/heads/master"); ReflogEntry e = r.getLastEntry(); assertEquals(pid, e.getNewId()); @@ -355,7 +360,8 @@ public class FileReftableTest extends SampleDataRepositoryTestCase { assertEquals(RefUpdate.Result.NO_CHANGE, ref.delete()); // Differs from RefupdateTest. Deleting a loose ref leaves reflog trail. - ReflogReader reader = db.getReflogReader("refs/heads/abc"); + ReflogReader reader = db.getRefDatabase() + .getReflogReader("refs/heads/abc"); assertEquals(ObjectId.zeroId(), reader.getReverseEntry(1).getOldId()); assertEquals(nonZero, reader.getReverseEntry(1).getNewId()); assertEquals(nonZero, reader.getReverseEntry(0).getOldId()); @@ -382,8 +388,9 @@ public class FileReftableTest extends SampleDataRepositoryTestCase { assertNotSame(newid, r.getObjectId()); assertSame(ObjectId.class, r.getObjectId().getClass()); assertEquals(newid, r.getObjectId()); - List reverseEntries1 = db.getReflogReader("refs/heads/abc") - .getReverseEntries(); + RefDatabase refDb = db.getRefDatabase(); + List reverseEntries1 = refDb + .getReflogReader("refs/heads/abc").getReverseEntries(); ReflogEntry entry1 = reverseEntries1.get(0); assertEquals(1, reverseEntries1.size()); assertEquals(ObjectId.zeroId(), entry1.getOldId()); @@ -392,7 +399,7 @@ public class FileReftableTest extends SampleDataRepositoryTestCase { assertEquals(new PersonIdent(db).toString(), entry1.getWho().toString()); assertEquals("", entry1.getComment()); - List reverseEntries2 = db.getReflogReader("HEAD") + List reverseEntries2 = refDb.getReflogReader("HEAD") .getReverseEntries(); assertEquals(0, reverseEntries2.size()); } @@ -455,7 +462,7 @@ public class FileReftableTest extends SampleDataRepositoryTestCase { // the branch HEAD referred to is left untouched assertNull(db.resolve("refs/heads/unborn")); - ReflogReader reflogReader = db.getReflogReader("HEAD"); + ReflogReader reflogReader = db.getRefDatabase().getReflogReader("HEAD"); ReflogEntry e = reflogReader.getReverseEntries().get(0); assertEquals(ObjectId.zeroId(), e.getOldId()); assertEquals(ppid, e.getNewId()); @@ -499,7 +506,7 @@ public class FileReftableTest extends SampleDataRepositoryTestCase { names.add("refs/heads/new/name"); for (String nm : names) { - ReflogReader rd = db.getReflogReader(nm); + ReflogReader rd = db.getRefDatabase().getReflogReader(nm); assertNotNull(rd); ReflogEntry last = rd.getLastEntry(); ObjectId id = last.getNewId(); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/RefUpdateTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/RefUpdateTest.java index cb977bd601..0a9aa40d04 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/RefUpdateTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/RefUpdateTest.java @@ -40,6 +40,7 @@ import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectInserter; import org.eclipse.jgit.lib.PersonIdent; import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.lib.RefDatabase; import org.eclipse.jgit.lib.RefRename; import org.eclipse.jgit.lib.RefUpdate; import org.eclipse.jgit.lib.RefUpdate.Result; @@ -111,16 +112,17 @@ public class RefUpdateTest extends SampleDataRepositoryTestCase { assertNotSame(newid, r.getObjectId()); assertSame(ObjectId.class, r.getObjectId().getClass()); assertEquals(newid, r.getObjectId()); - List reverseEntries1 = db + List reverseEntries1 = db.getRefDatabase() .getReflogReader("refs/heads/abc").getReverseEntries(); ReflogEntry entry1 = reverseEntries1.get(0); assertEquals(1, reverseEntries1.size()); assertEquals(ObjectId.zeroId(), entry1.getOldId()); assertEquals(r.getObjectId(), entry1.getNewId()); - assertEquals(new PersonIdent(db).toString(), entry1.getWho().toString()); + assertEquals(new PersonIdent(db).toString(), + entry1.getWho().toString()); assertEquals("", entry1.getComment()); - List reverseEntries2 = db.getReflogReader("HEAD") - .getReverseEntries(); + List reverseEntries2 = db.getRefDatabase() + .getReflogReader("HEAD").getReverseEntries(); assertEquals(0, reverseEntries2.size()); } @@ -136,8 +138,11 @@ public class RefUpdateTest extends SampleDataRepositoryTestCase { final RefUpdate ru2 = updateRef(newRef2); Result update2 = ru2.update(); assertEquals(Result.LOCK_FAILURE, update2); - assertEquals(1, db.getReflogReader("refs/heads/z").getReverseEntries().size()); - assertEquals(0, db.getReflogReader("HEAD").getReverseEntries().size()); + RefDatabase refDb = db.getRefDatabase(); + assertEquals(1, refDb.getReflogReader("refs/heads/z") + .getReverseEntries().size()); + assertEquals(0, + refDb.getReflogReader("HEAD").getReverseEntries().size()); } @Test @@ -147,8 +152,10 @@ public class RefUpdateTest extends SampleDataRepositoryTestCase { final RefUpdate ru = updateRef(newRef); Result update = ru.update(); assertEquals(Result.LOCK_FAILURE, update); - assertNull(db.getReflogReader("refs/heads/master/x")); - assertEquals(0, db.getReflogReader("HEAD").getReverseEntries().size()); + RefDatabase refDb = db.getRefDatabase(); + assertNull(refDb.getReflogReader("refs/heads/master/x")); + assertEquals(0, + refDb.getReflogReader("HEAD").getReverseEntries().size()); } @Test @@ -163,9 +170,12 @@ public class RefUpdateTest extends SampleDataRepositoryTestCase { final RefUpdate ru2 = updateRef(newRef2); Result update2 = ru2.update(); assertEquals(Result.LOCK_FAILURE, update2); - assertEquals(1, db.getReflogReader("refs/heads/z/a").getReverseEntries().size()); - assertNull(db.getReflogReader("refs/heads/z")); - assertEquals(0, db.getReflogReader("HEAD").getReverseEntries().size()); + RefDatabase refDb = db.getRefDatabase(); + assertEquals(1, refDb.getReflogReader("refs/heads/z/a") + .getReverseEntries().size()); + assertNull(refDb.getReflogReader("refs/heads/z")); + assertEquals(0, + refDb.getReflogReader("HEAD").getReverseEntries().size()); } @Test @@ -175,8 +185,10 @@ public class RefUpdateTest extends SampleDataRepositoryTestCase { final RefUpdate ru = updateRef(newRef); Result update = ru.update(); assertEquals(Result.LOCK_FAILURE, update); - assertNull(db.getReflogReader("refs/heads/prefix")); - assertEquals(0, db.getReflogReader("HEAD").getReverseEntries().size()); + RefDatabase refDb = db.getRefDatabase(); + assertNull(refDb.getReflogReader("refs/heads/prefix")); + assertEquals(0, + refDb.getReflogReader("HEAD").getReverseEntries().size()); } /** @@ -197,8 +209,11 @@ public class RefUpdateTest extends SampleDataRepositoryTestCase { Result delete = updateRef2.delete(); assertEquals(Result.REJECTED_CURRENT_BRANCH, delete); assertEquals(pid, db.resolve("refs/heads/master")); - assertEquals(1,db.getReflogReader("refs/heads/master").getReverseEntries().size()); - assertEquals(0,db.getReflogReader("HEAD").getReverseEntries().size()); + RefDatabase refDb = db.getRefDatabase(); + assertEquals(1, refDb.getReflogReader("refs/heads/master") + .getReverseEntries().size()); + assertEquals(0, + refDb.getReflogReader("HEAD").getReverseEntries().size()); } @Test @@ -209,7 +224,8 @@ public class RefUpdateTest extends SampleDataRepositoryTestCase { updateRef.setForceUpdate(true); Result update = updateRef.update(); assertEquals(Result.FORCED, update); - assertEquals(1,db.getReflogReader("refs/heads/master").getReverseEntries().size()); + assertEquals(1, db.getRefDatabase().getReflogReader("refs/heads/master") + .getReverseEntries().size()); } @Test @@ -219,15 +235,18 @@ public class RefUpdateTest extends SampleDataRepositoryTestCase { ref.update(); // create loose ref ref = updateRef(newRef); // refresh delete(ref, Result.NO_CHANGE); - assertNull(db.getReflogReader("refs/heads/abc")); + assertNull(db.getRefDatabase().getReflogReader("refs/heads/abc")); } @Test public void testDeleteHead() throws IOException { final RefUpdate ref = updateRef(Constants.HEAD); delete(ref, Result.REJECTED_CURRENT_BRANCH, true, false); - assertEquals(0, db.getReflogReader("refs/heads/master").getReverseEntries().size()); - assertEquals(0, db.getReflogReader("HEAD").getReverseEntries().size()); + RefDatabase refDb = db.getRefDatabase(); + assertEquals(0, refDb.getReflogReader("refs/heads/master") + .getReverseEntries().size()); + assertEquals(0, + refDb.getReflogReader("HEAD").getReverseEntries().size()); } @Test @@ -423,7 +442,7 @@ public class RefUpdateTest extends SampleDataRepositoryTestCase { // the branch HEAD referred to is left untouched assertEquals(pid, db.resolve("refs/heads/master")); - ReflogReader reflogReader = db.getReflogReader("HEAD"); + ReflogReader reflogReader = db.getRefDatabase().getReflogReader("HEAD"); ReflogEntry e = reflogReader.getReverseEntries().get(0); assertEquals(pid, e.getOldId()); assertEquals(ppid, e.getNewId()); @@ -453,7 +472,7 @@ public class RefUpdateTest extends SampleDataRepositoryTestCase { // the branch HEAD referred to is left untouched assertNull(db.resolve("refs/heads/unborn")); - ReflogReader reflogReader = db.getReflogReader("HEAD"); + ReflogReader reflogReader = db.getRefDatabase().getReflogReader("HEAD"); ReflogEntry e = reflogReader.getReverseEntries().get(0); assertEquals(ObjectId.zeroId(), e.getOldId()); assertEquals(ppid, e.getNewId()); @@ -691,9 +710,11 @@ public class RefUpdateTest extends SampleDataRepositoryTestCase { assertEquals(Result.RENAMED, result); assertEquals(rb, db.resolve("refs/heads/new/name")); assertNull(db.resolve("refs/heads/b")); - assertEquals(1, db.getReflogReader("new/name").getReverseEntries().size()); - assertEquals("Branch: renamed b to new/name", db.getReflogReader("new/name") - .getLastEntry().getComment()); + RefDatabase refDb = db.getRefDatabase(); + assertEquals(1, + refDb.getReflogReader("new/name").getReverseEntries().size()); + assertEquals("Branch: renamed b to new/name", + refDb.getReflogReader("new/name").getLastEntry().getComment()); assertFalse(new File(db.getDirectory(), "logs/refs/heads/b").exists()); assertEquals(oldHead, db.resolve(Constants.HEAD)); // unchanged } @@ -713,10 +734,12 @@ public class RefUpdateTest extends SampleDataRepositoryTestCase { assertEquals(Result.RENAMED, result); assertEquals(rb, db.resolve("refs/heads/new/name")); assertNull(db.resolve("refs/heads/b")); - assertEquals(2, db.getReflogReader("new/name").getReverseEntries().size()); - assertEquals("Branch: renamed b to new/name", db.getReflogReader("new/name") - .getLastEntry().getComment()); - assertEquals("Just a message", db.getReflogReader("new/name") + RefDatabase refDb = db.getRefDatabase(); + assertEquals(2, + refDb.getReflogReader("new/name").getReverseEntries().size()); + assertEquals("Branch: renamed b to new/name", + refDb.getReflogReader("new/name").getLastEntry().getComment()); + assertEquals("Just a message", refDb.getReflogReader("new/name") .getReverseEntries().get(1).getComment()); assertFalse(new File(db.getDirectory(), "logs/refs/heads/b").exists()); assertEquals(oldHead, db.resolve(Constants.HEAD)); // unchanged @@ -737,13 +760,18 @@ public class RefUpdateTest extends SampleDataRepositoryTestCase { assertEquals(Result.RENAMED, result); assertEquals(rb, db.resolve("refs/heads/new/name")); assertNull(db.resolve("refs/heads/b")); - assertEquals("Branch: renamed b to new/name", db.getReflogReader( - "new/name").getLastEntry().getComment()); + RefDatabase refDb = db.getRefDatabase(); + assertEquals("Branch: renamed b to new/name", + refDb.getReflogReader("new/name").getLastEntry().getComment()); assertFalse(new File(db.getDirectory(), "logs/refs/heads/b").exists()); assertEquals(rb, db.resolve(Constants.HEAD)); - assertEquals(2, db.getReflogReader("new/name").getReverseEntries().size()); - assertEquals("Branch: renamed b to new/name", db.getReflogReader("new/name").getReverseEntries().get(0).getComment()); - assertEquals("Just a message", db.getReflogReader("new/name").getReverseEntries().get(1).getComment()); + assertEquals(2, + refDb.getReflogReader("new/name").getReverseEntries().size()); + assertEquals("Branch: renamed b to new/name", + refDb.getReflogReader("new/name").getReverseEntries().get(0) + .getComment()); + assertEquals("Just a message", refDb.getReflogReader("new/name") + .getReverseEntries().get(1).getComment()); } @Test @@ -766,11 +794,16 @@ public class RefUpdateTest extends SampleDataRepositoryTestCase { assertEquals(Result.RENAMED, result); assertEquals(rb2, db.resolve("refs/heads/new/name")); assertNull(db.resolve("refs/heads/b")); - assertEquals("Branch: renamed b to new/name", db.getReflogReader( - "new/name").getLastEntry().getComment()); - assertEquals(3, db.getReflogReader("refs/heads/new/name").getReverseEntries().size()); - assertEquals("Branch: renamed b to new/name", db.getReflogReader("refs/heads/new/name").getReverseEntries().get(0).getComment()); - assertEquals(0, db.getReflogReader("HEAD").getReverseEntries().size()); + RefDatabase refDb = db.getRefDatabase(); + assertEquals("Branch: renamed b to new/name", + refDb.getReflogReader("new/name").getLastEntry().getComment()); + assertEquals(3, refDb.getReflogReader("refs/heads/new/name") + .getReverseEntries().size()); + assertEquals("Branch: renamed b to new/name", + refDb.getReflogReader("refs/heads/new/name").getReverseEntries() + .get(0).getComment()); + assertEquals(0, + refDb.getReflogReader("HEAD").getReverseEntries().size()); // make sure b's log file is gone too. assertFalse(new File(db.getDirectory(), "logs/refs/heads/b").exists()); @@ -789,9 +822,10 @@ public class RefUpdateTest extends SampleDataRepositoryTestCase { ObjectId oldfromId = db.resolve(fromName); ObjectId oldHeadId = db.resolve(Constants.HEAD); writeReflog(db, oldfromId, "Just a message", fromName); - List oldFromLog = db + RefDatabase refDb = db.getRefDatabase(); + List oldFromLog = refDb .getReflogReader(fromName).getReverseEntries(); - List oldHeadLog = oldHeadId != null ? db + List oldHeadLog = oldHeadId != null ? refDb .getReflogReader(Constants.HEAD).getReverseEntries() : null; assertTrue("internal check, we have a log", new File(db.getDirectory(), @@ -818,10 +852,10 @@ public class RefUpdateTest extends SampleDataRepositoryTestCase { assertEquals(oldHeadId, db.resolve(Constants.HEAD)); assertEquals(oldfromId, db.resolve(fromName)); assertNull(db.resolve(toName)); - assertEquals(oldFromLog.toString(), db.getReflogReader(fromName) + assertEquals(oldFromLog.toString(), refDb.getReflogReader(fromName) .getReverseEntries().toString()); if (oldHeadId != null && oldHeadLog != null) - assertEquals(oldHeadLog.toString(), db.getReflogReader( + assertEquals(oldHeadLog.toString(), refDb.getReflogReader( Constants.HEAD).getReverseEntries().toString()); } finally { lockFile.unlock(); @@ -942,15 +976,17 @@ public class RefUpdateTest extends SampleDataRepositoryTestCase { assertEquals(Result.RENAMED, result); assertNull(db.resolve("refs/heads/a")); assertEquals(rb, db.resolve("refs/heads/a/b")); - assertEquals(3, db.getReflogReader("a/b").getReverseEntries().size()); - assertEquals("Branch: renamed a to a/b", db.getReflogReader("a/b") + RefDatabase refDb = db.getRefDatabase(); + assertEquals(3, + refDb.getReflogReader("a/b").getReverseEntries().size()); + assertEquals("Branch: renamed a to a/b", refDb.getReflogReader("a/b") .getReverseEntries().get(0).getComment()); - assertEquals("Just a message", db.getReflogReader("a/b") + assertEquals("Just a message", refDb.getReflogReader("a/b") .getReverseEntries().get(1).getComment()); - assertEquals("Setup", db.getReflogReader("a/b").getReverseEntries() + assertEquals("Setup", refDb.getReflogReader("a/b").getReverseEntries() .get(2).getComment()); // same thing was logged to HEAD - assertEquals("Branch: renamed a to a/b", db.getReflogReader("HEAD") + assertEquals("Branch: renamed a to a/b", refDb.getReflogReader("HEAD") .getReverseEntries().get(0).getComment()); } @@ -978,15 +1014,19 @@ public class RefUpdateTest extends SampleDataRepositoryTestCase { assertNull(db.resolve("refs/heads/prefix/a")); assertEquals(rb, db.resolve("refs/heads/prefix")); - assertEquals(3, db.getReflogReader("prefix").getReverseEntries().size()); - assertEquals("Branch: renamed prefix/a to prefix", db.getReflogReader( - "prefix").getReverseEntries().get(0).getComment()); - assertEquals("Just a message", db.getReflogReader("prefix") + RefDatabase refDb = db.getRefDatabase(); + assertEquals(3, + refDb.getReflogReader("prefix").getReverseEntries().size()); + assertEquals("Branch: renamed prefix/a to prefix", + refDb.getReflogReader("prefix").getReverseEntries().get(0) + .getComment()); + assertEquals("Just a message", refDb.getReflogReader("prefix") .getReverseEntries().get(1).getComment()); - assertEquals("Setup", db.getReflogReader("prefix").getReverseEntries() - .get(2).getComment()); - assertEquals("Branch: renamed prefix/a to prefix", db.getReflogReader( - "HEAD").getReverseEntries().get(0).getComment()); + assertEquals("Setup", refDb.getReflogReader("prefix") + .getReverseEntries().get(2).getComment()); + assertEquals("Branch: renamed prefix/a to prefix", + refDb.getReflogReader("HEAD").getReverseEntries().get(0) + .getComment()); } @Test diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/ReflogReaderTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/ReflogReaderTest.java index eb521ff9eb..23c279eabe 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/ReflogReaderTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/ReflogReaderTest.java @@ -27,6 +27,7 @@ import org.eclipse.jgit.lib.CheckoutEntry; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.PersonIdent; +import org.eclipse.jgit.lib.RefDatabase; import org.eclipse.jgit.lib.ReflogEntry; import org.eclipse.jgit.lib.ReflogReader; import org.eclipse.jgit.test.resources.SampleDataRepositoryTestCase; @@ -154,18 +155,20 @@ public class ReflogReaderTest extends SampleDataRepositoryTestCase { setupReflog("logs/refs/heads/a", aLine); setupReflog("logs/refs/heads/master", masterLine); setupReflog("logs/HEAD", headLine); - assertEquals("branch: change to master", db.getReflogReader("master") - .getLastEntry().getComment()); - assertEquals("branch: change to a", db.getReflogReader("a") - .getLastEntry().getComment()); - assertEquals("branch: change to HEAD", db.getReflogReader("HEAD") - .getLastEntry().getComment()); + RefDatabase refDb = db.getRefDatabase(); + assertEquals("branch: change to master", + refDb.getReflogReader("master").getLastEntry().getComment()); + assertEquals("branch: change to a", + refDb.getReflogReader("a").getLastEntry().getComment()); + assertEquals("branch: change to HEAD", + refDb.getReflogReader("HEAD").getLastEntry().getComment()); } @Test public void testReadLineWithMissingComment() throws Exception { setupReflog("logs/refs/heads/master", oneLineWithoutComment); - final ReflogReader reader = db.getReflogReader("master"); + final ReflogReader reader = db.getRefDatabase() + .getReflogReader("master"); ReflogEntry e = reader.getLastEntry(); assertEquals(ObjectId .fromString("da85355dfc525c9f6f3927b876f379f46ccf826e"), e @@ -183,15 +186,17 @@ public class ReflogReaderTest extends SampleDataRepositoryTestCase { @Test public void testNoLog() throws Exception { - assertEquals(0, db.getReflogReader("master").getReverseEntries().size()); - assertNull(db.getReflogReader("master").getLastEntry()); + RefDatabase refDb = db.getRefDatabase(); + assertEquals(0, + refDb.getReflogReader("master").getReverseEntries().size()); + assertNull(refDb.getReflogReader("master").getLastEntry()); } @Test public void testCheckout() throws Exception { setupReflog("logs/HEAD", switchBranch); - List entries = db.getReflogReader(Constants.HEAD) - .getReverseEntries(); + List entries = db.getRefDatabase() + .getReflogReader(Constants.HEAD).getReverseEntries(); assertEquals(1, entries.size()); ReflogEntry entry = entries.get(0); CheckoutEntry checkout = entry.parseCheckout(); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/ReflogConfigTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/ReflogConfigTest.java index ad47cef0f4..a93937eeea 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/ReflogConfigTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/ReflogConfigTest.java @@ -32,7 +32,8 @@ public class ReflogConfigTest extends RepositoryTestCase { // check that there are no entries in the reflog and turn off writing // reflogs - assertTrue(db.getReflogReader(Constants.HEAD).getReverseEntries() + RefDatabase refDb = db.getRefDatabase(); + assertTrue(refDb.getReflogReader(Constants.HEAD).getReverseEntries() .isEmpty()); FileBasedConfig cfg = db.getConfig(); cfg.setBoolean("core", null, "logallrefupdates", false); @@ -42,7 +43,7 @@ public class ReflogConfigTest extends RepositoryTestCase { // written commit("A Commit\n", commitTime, tz); commitTime = commitTime.plus(Duration.ofMinutes(1)); - assertTrue("Reflog for HEAD still contain no entry", db + assertTrue("Reflog for HEAD still contain no entry", refDb .getReflogReader(Constants.HEAD).getReverseEntries().isEmpty()); // set the logAllRefUpdates parameter to true and check it @@ -56,9 +57,9 @@ public class ReflogConfigTest extends RepositoryTestCase { // do one commit and check that reflog size is increased to 1 commit("A Commit\n", commitTime, tz); commitTime = commitTime.plus(Duration.ofMinutes(1)); - assertTrue( - "Reflog for HEAD should contain one entry", - db.getReflogReader(Constants.HEAD).getReverseEntries().size() == 1); + assertTrue("Reflog for HEAD should contain one entry", + refDb.getReflogReader(Constants.HEAD).getReverseEntries() + .size() == 1); // set the logAllRefUpdates parameter to false and check it cfg.setBoolean("core", null, "logallrefupdates", false); @@ -71,9 +72,9 @@ public class ReflogConfigTest extends RepositoryTestCase { // do one commit and check that reflog size is 2 commit("A Commit\n", commitTime, tz); commitTime = commitTime.plus(Duration.ofMinutes(1)); - assertTrue( - "Reflog for HEAD should contain two entries", - db.getReflogReader(Constants.HEAD).getReverseEntries().size() == 2); + assertTrue("Reflog for HEAD should contain two entries", + refDb.getReflogReader(Constants.HEAD).getReverseEntries() + .size() == 2); // set the logAllRefUpdates parameter to false and check it cfg.setEnum("core", null, "logallrefupdates", @@ -87,7 +88,7 @@ public class ReflogConfigTest extends RepositoryTestCase { // do one commit and check that reflog size is 3 commit("A Commit\n", commitTime, tz); assertTrue("Reflog for HEAD should contain three entries", - db.getReflogReader(Constants.HEAD).getReverseEntries() + refDb.getReflogReader(Constants.HEAD).getReverseEntries() .size() == 3); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/ReflogCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/ReflogCommand.java index dead2749b7..a149649004 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/ReflogCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/ReflogCommand.java @@ -68,7 +68,7 @@ public class ReflogCommand extends GitCommand> { checkCallable(); try { - ReflogReader reader = repo.getReflogReader(ref); + ReflogReader reader = repo.getRefDatabase().getReflogReader(ref); if (reader == null) throw new RefNotFoundException(MessageFormat.format( JGitText.get().refNotResolved, ref)); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/StashDropCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/StashDropCommand.java index 23fbe0197f..2dba0ef0f2 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/StashDropCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/StashDropCommand.java @@ -165,7 +165,8 @@ public class StashDropCommand extends GitCommand { List entries; try { - ReflogReader reader = repo.getReflogReader(R_STASH); + ReflogReader reader = repo.getRefDatabase() + .getReflogReader(R_STASH); if (reader == null) { throw new RefNotFoundException(MessageFormat .format(JGitText.get().refNotResolved, stashRef)); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java index 81e8c0306d..3b09d3a82f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java @@ -563,9 +563,10 @@ public class FileReftableDatabase extends RefDatabase { boolean writeLogs) throws IOException { int size = 0; List refs = repo.getRefDatabase().getRefs(); + RefDatabase refDb = repo.getRefDatabase(); if (writeLogs) { for (Ref r : refs) { - ReflogReader rlr = repo.getReflogReader(r.getName()); + ReflogReader rlr = refDb.getReflogReader(r.getName()); if (rlr != null) { size = Math.max(rlr.getReverseEntries().size(), size); } @@ -588,7 +589,7 @@ public class FileReftableDatabase extends RefDatabase { if (writeLogs) { for (Ref r : refs) { long idx = size; - ReflogReader reader = repo.getReflogReader(r.getName()); + ReflogReader reader = refDb.getReflogReader(r.getName()); if (reader == null) { continue; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java index 09cf35a0cf..15679aa64e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java @@ -1157,7 +1157,7 @@ public class GC { * if an IO error occurred */ private Set listRefLogObjects(Ref ref, long minTime) throws IOException { - ReflogReader reflogReader = repo.getReflogReader(ref); + ReflogReader reflogReader = repo.getRefDatabase().getReflogReader(ref); List rlEntries = reflogReader .getReverseEntries(); if (rlEntries == null || rlEntries.isEmpty()) diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java index a7bba23996..757473878b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java @@ -1692,7 +1692,9 @@ public abstract class Repository implements AutoCloseable { * @throws java.io.IOException * the ref could not be accessed. * @since 3.0 + * @deprecated use {@code #getRefDatabase().getReflogReader(String)} instead */ + @Deprecated(since = "7.2") @Nullable public ReflogReader getReflogReader(String refName) throws IOException { return getRefDatabase().getReflogReader(refName); @@ -1708,7 +1710,9 @@ public abstract class Repository implements AutoCloseable { * @throws IOException * if an IO error occurred * @since 5.13.2 + * @deprecated use {@code #getRefDatabase().getReflogReader(Ref)} instead */ + @Deprecated(since = "7.2") @NonNull public ReflogReader getReflogReader(@NonNull Ref ref) throws IOException { -- cgit v1.2.3 From 7b8ddc2ce46500d1ee6e4a8b325dbf54262a88bb Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Sun, 5 Jan 2025 22:02:33 +0100 Subject: RefDatabase#getReflogReader(String): use #exactRef to resolve refName Don't accept short ref names anymore which is more predictable. Change-Id: I5e7323c610c68b25facd6f2286456716d8e6cf1a --- .../eclipse/jgit/api/CherryPickCommandTest.java | 2 +- .../tst/org/eclipse/jgit/api/CloneCommandTest.java | 2 +- .../eclipse/jgit/api/CommitAndLogCommandTest.java | 4 +- .../org/eclipse/jgit/api/CommitCommandTest.java | 4 +- .../tst/org/eclipse/jgit/api/MergeCommandTest.java | 18 +++--- .../org/eclipse/jgit/api/RevertCommandTest.java | 8 +-- .../jgit/internal/storage/file/RefUpdateTest.java | 68 ++++++++++++---------- .../internal/storage/file/ReflogReaderTest.java | 13 +++-- .../storage/file/FileReftableDatabase.java | 7 +-- .../src/org/eclipse/jgit/lib/RefDatabase.java | 4 +- 10 files changed, 70 insertions(+), 60 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CherryPickCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CherryPickCommandTest.java index a9bd126ff3..3f5c5da55a 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CherryPickCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CherryPickCommandTest.java @@ -534,7 +534,7 @@ public class CherryPickCommandTest extends RepositoryTestCase { ReflogReader reader = refDb.getReflogReader(Constants.HEAD); assertTrue(reader.getLastEntry().getComment() .startsWith("cherry-pick: ")); - reader = refDb.getReflogReader(db.getBranch()); + reader = refDb.getReflogReader(db.getFullBranch()); assertTrue(reader.getLastEntry().getComment() .startsWith("cherry-pick: ")); } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CloneCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CloneCommandTest.java index f70ada4e30..f5fd73ba38 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CloneCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CloneCommandTest.java @@ -182,7 +182,7 @@ public class CloneCommandTest extends RepositoryTestCase { private static boolean hasRefLog(Repository repo, Ref ref) { try { - return repo.getRefDatabase().getReflogReader(ref.getName()) + return repo.getRefDatabase().getReflogReader(ref) .getLastEntry() != null; } catch (IOException ioe) { throw new IllegalStateException(ioe); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CommitAndLogCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CommitAndLogCommandTest.java index ced24b5928..4e5f44e5a6 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CommitAndLogCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CommitAndLogCommandTest.java @@ -74,7 +74,7 @@ public class CommitAndLogCommandTest extends RepositoryTestCase { ReflogReader reader = refDb.getReflogReader(Constants.HEAD); assertTrue( reader.getLastEntry().getComment().startsWith("commit:")); - reader = refDb.getReflogReader(db.getBranch()); + reader = refDb.getReflogReader(db.getFullBranch()); assertTrue( reader.getLastEntry().getComment().startsWith("commit:")); } @@ -254,7 +254,7 @@ public class CommitAndLogCommandTest extends RepositoryTestCase { ReflogReader reader = refDb.getReflogReader(Constants.HEAD); assertTrue(reader.getLastEntry().getComment() .startsWith("commit (amend):")); - reader = refDb.getReflogReader(db.getBranch()); + reader = refDb.getReflogReader(db.getFullBranch()); assertTrue(reader.getLastEntry().getComment() .startsWith("commit (amend):")); } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CommitCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CommitCommandTest.java index d40df26690..21cfcc4e34 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CommitCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CommitCommandTest.java @@ -439,7 +439,7 @@ public class CommitCommandTest extends RepositoryTestCase { db.getRefDatabase().getReflogReader(Constants.HEAD) .getLastEntry().getComment()); assertEquals("commit: Squashed commit of the following:", - db.getRefDatabase().getReflogReader(db.getBranch()) + db.getRefDatabase().getReflogReader(db.getFullBranch()) .getLastEntry().getComment()); } } @@ -464,7 +464,7 @@ public class CommitCommandTest extends RepositoryTestCase { db.getRefDatabase().getReflogReader(Constants.HEAD) .getReverseEntries())); assertEquals("testRl;commit (initial): c1;", reflogComments( - db.getRefDatabase().getReflogReader(db.getBranch()) + db.getRefDatabase().getReflogReader(db.getFullBranch()) .getReverseEntries())); } } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/MergeCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/MergeCommandTest.java index ba13f8ac1f..503fef9916 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/MergeCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/MergeCommandTest.java @@ -81,7 +81,8 @@ public class MergeCommandTest extends RepositoryTestCase { assertEquals("commit (initial): initial commit", refDb .getReflogReader(Constants.HEAD).getLastEntry().getComment()); assertEquals("commit (initial): initial commit", refDb - .getReflogReader(db.getBranch()).getLastEntry().getComment()); + .getReflogReader(db.getFullBranch()).getLastEntry() + .getComment()); } @Test @@ -99,7 +100,8 @@ public class MergeCommandTest extends RepositoryTestCase { assertEquals("commit: second commit", db.getRefDatabase() .getReflogReader(Constants.HEAD).getLastEntry().getComment()); assertEquals("commit: second commit", db.getRefDatabase() - .getReflogReader(db.getBranch()).getLastEntry().getComment()); + .getReflogReader(db.getFullBranch()).getLastEntry() + .getComment()); } @Test @@ -122,7 +124,7 @@ public class MergeCommandTest extends RepositoryTestCase { refDb.getReflogReader(Constants.HEAD) .getLastEntry().getComment()); assertEquals("merge refs/heads/master: Fast-forward", - refDb.getReflogReader(db.getBranch()) + refDb.getReflogReader(db.getFullBranch()) .getLastEntry().getComment()); } @@ -147,7 +149,8 @@ public class MergeCommandTest extends RepositoryTestCase { assertEquals("merge refs/heads/master: Fast-forward", refDb .getReflogReader(Constants.HEAD).getLastEntry().getComment()); assertEquals("merge refs/heads/master: Fast-forward", refDb - .getReflogReader(db.getBranch()).getLastEntry().getComment()); + .getReflogReader(db.getFullBranch()).getLastEntry() + .getComment()); } @Test @@ -179,7 +182,8 @@ public class MergeCommandTest extends RepositoryTestCase { assertEquals("merge refs/heads/master: Fast-forward", refDb .getReflogReader(Constants.HEAD).getLastEntry().getComment()); assertEquals("merge refs/heads/master: Fast-forward", refDb - .getReflogReader(db.getBranch()).getLastEntry().getComment()); + .getReflogReader(db.getFullBranch()).getLastEntry() + .getComment()); } @Test @@ -243,7 +247,7 @@ public class MergeCommandTest extends RepositoryTestCase { assertEquals( "merge refs/heads/master: Merge made by " + mergeStrategy.getName() + ".", - refDb.getReflogReader(db.getBranch()).getLastEntry() + refDb.getReflogReader(db.getFullBranch()).getLastEntry() .getComment()); } @@ -679,7 +683,7 @@ public class MergeCommandTest extends RepositoryTestCase { assertEquals( "merge " + secondCommit.getId().getName() + ": Merge made by resolve.", - refDb.getReflogReader(db.getBranch()).getLastEntry() + refDb.getReflogReader(db.getFullBranch()).getLastEntry() .getComment()); } } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/RevertCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/RevertCommandTest.java index e4d0eb76f7..89fdb32220 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/RevertCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/RevertCommandTest.java @@ -92,7 +92,7 @@ public class RevertCommandTest extends RepositoryTestCase { ReflogReader reader = refDb.getReflogReader(Constants.HEAD); assertTrue(reader.getLastEntry().getComment() .startsWith("revert: Revert \"")); - reader = refDb.getReflogReader(db.getBranch()); + reader = refDb.getReflogReader(db.getFullBranch()); assertTrue(reader.getLastEntry().getComment() .startsWith("revert: Revert \"")); } @@ -176,7 +176,7 @@ public class RevertCommandTest extends RepositoryTestCase { ReflogReader reader = refDb.getReflogReader(Constants.HEAD); assertTrue(reader.getLastEntry().getComment() .startsWith("revert: Revert \"")); - reader = refDb.getReflogReader(db.getBranch()); + reader = refDb.getReflogReader(db.getFullBranch()); assertTrue(reader.getLastEntry().getComment() .startsWith("revert: Revert \"")); } @@ -230,7 +230,7 @@ public class RevertCommandTest extends RepositoryTestCase { ReflogReader reader = refDb.getReflogReader(Constants.HEAD); assertTrue(reader.getLastEntry().getComment() .startsWith("revert: Revert \"")); - reader = refDb.getReflogReader(db.getBranch()); + reader = refDb.getReflogReader(db.getFullBranch()); assertTrue(reader.getLastEntry().getComment() .startsWith("revert: Revert \"")); } @@ -439,7 +439,7 @@ public class RevertCommandTest extends RepositoryTestCase { ReflogReader reader = refDb.getReflogReader(Constants.HEAD); assertTrue( reader.getLastEntry().getComment().startsWith("revert: ")); - reader = refDb.getReflogReader(db.getBranch()); + reader = refDb.getReflogReader(db.getFullBranch()); assertTrue( reader.getLastEntry().getComment().startsWith("revert: ")); } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/RefUpdateTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/RefUpdateTest.java index 0a9aa40d04..acc36d76f4 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/RefUpdateTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/RefUpdateTest.java @@ -711,10 +711,11 @@ public class RefUpdateTest extends SampleDataRepositoryTestCase { assertEquals(rb, db.resolve("refs/heads/new/name")); assertNull(db.resolve("refs/heads/b")); RefDatabase refDb = db.getRefDatabase(); - assertEquals(1, - refDb.getReflogReader("new/name").getReverseEntries().size()); + assertEquals(1, refDb.getReflogReader("refs/heads/new/name") + .getReverseEntries().size()); assertEquals("Branch: renamed b to new/name", - refDb.getReflogReader("new/name").getLastEntry().getComment()); + refDb.getReflogReader("refs/heads/new/name").getLastEntry() + .getComment()); assertFalse(new File(db.getDirectory(), "logs/refs/heads/b").exists()); assertEquals(oldHead, db.resolve(Constants.HEAD)); // unchanged } @@ -735,12 +736,14 @@ public class RefUpdateTest extends SampleDataRepositoryTestCase { assertEquals(rb, db.resolve("refs/heads/new/name")); assertNull(db.resolve("refs/heads/b")); RefDatabase refDb = db.getRefDatabase(); - assertEquals(2, - refDb.getReflogReader("new/name").getReverseEntries().size()); + assertEquals(2, refDb.getReflogReader("refs/heads/new/name") + .getReverseEntries().size()); assertEquals("Branch: renamed b to new/name", - refDb.getReflogReader("new/name").getLastEntry().getComment()); - assertEquals("Just a message", refDb.getReflogReader("new/name") - .getReverseEntries().get(1).getComment()); + refDb.getReflogReader("refs/heads/new/name").getLastEntry() + .getComment()); + assertEquals("Just a message", + refDb.getReflogReader("refs/heads/new/name").getReverseEntries() + .get(1).getComment()); assertFalse(new File(db.getDirectory(), "logs/refs/heads/b").exists()); assertEquals(oldHead, db.resolve(Constants.HEAD)); // unchanged } @@ -762,16 +765,18 @@ public class RefUpdateTest extends SampleDataRepositoryTestCase { assertNull(db.resolve("refs/heads/b")); RefDatabase refDb = db.getRefDatabase(); assertEquals("Branch: renamed b to new/name", - refDb.getReflogReader("new/name").getLastEntry().getComment()); + refDb.getReflogReader("refs/heads/new/name").getLastEntry() + .getComment()); assertFalse(new File(db.getDirectory(), "logs/refs/heads/b").exists()); assertEquals(rb, db.resolve(Constants.HEAD)); - assertEquals(2, - refDb.getReflogReader("new/name").getReverseEntries().size()); + assertEquals(2, refDb.getReflogReader("refs/heads/new/name") + .getReverseEntries().size()); assertEquals("Branch: renamed b to new/name", - refDb.getReflogReader("new/name").getReverseEntries().get(0) - .getComment()); - assertEquals("Just a message", refDb.getReflogReader("new/name") - .getReverseEntries().get(1).getComment()); + refDb.getReflogReader("refs/heads/new/name").getReverseEntries() + .get(0).getComment()); + assertEquals("Just a message", + refDb.getReflogReader("refs/heads/new/name").getReverseEntries() + .get(1).getComment()); } @Test @@ -796,7 +801,8 @@ public class RefUpdateTest extends SampleDataRepositoryTestCase { assertNull(db.resolve("refs/heads/b")); RefDatabase refDb = db.getRefDatabase(); assertEquals("Branch: renamed b to new/name", - refDb.getReflogReader("new/name").getLastEntry().getComment()); + refDb.getReflogReader("refs/heads/new/name").getLastEntry() + .getComment()); assertEquals(3, refDb.getReflogReader("refs/heads/new/name") .getReverseEntries().size()); assertEquals("Branch: renamed b to new/name", @@ -977,14 +983,15 @@ public class RefUpdateTest extends SampleDataRepositoryTestCase { assertNull(db.resolve("refs/heads/a")); assertEquals(rb, db.resolve("refs/heads/a/b")); RefDatabase refDb = db.getRefDatabase(); - assertEquals(3, - refDb.getReflogReader("a/b").getReverseEntries().size()); - assertEquals("Branch: renamed a to a/b", refDb.getReflogReader("a/b") - .getReverseEntries().get(0).getComment()); - assertEquals("Just a message", refDb.getReflogReader("a/b") + assertEquals(3, refDb.getReflogReader("refs/heads/a/b") + .getReverseEntries().size()); + assertEquals("Branch: renamed a to a/b", + refDb.getReflogReader("refs/heads/a/b").getReverseEntries() + .get(0).getComment()); + assertEquals("Just a message", refDb.getReflogReader("refs/heads/a/b") .getReverseEntries().get(1).getComment()); - assertEquals("Setup", refDb.getReflogReader("a/b").getReverseEntries() - .get(2).getComment()); + assertEquals("Setup", refDb.getReflogReader("refs/heads/a/b") + .getReverseEntries().get(2).getComment()); // same thing was logged to HEAD assertEquals("Branch: renamed a to a/b", refDb.getReflogReader("HEAD") .getReverseEntries().get(0).getComment()); @@ -1015,14 +1022,15 @@ public class RefUpdateTest extends SampleDataRepositoryTestCase { assertNull(db.resolve("refs/heads/prefix/a")); assertEquals(rb, db.resolve("refs/heads/prefix")); RefDatabase refDb = db.getRefDatabase(); - assertEquals(3, - refDb.getReflogReader("prefix").getReverseEntries().size()); + assertEquals(3, refDb.getReflogReader("refs/heads/prefix") + .getReverseEntries().size()); assertEquals("Branch: renamed prefix/a to prefix", - refDb.getReflogReader("prefix").getReverseEntries().get(0) - .getComment()); - assertEquals("Just a message", refDb.getReflogReader("prefix") - .getReverseEntries().get(1).getComment()); - assertEquals("Setup", refDb.getReflogReader("prefix") + refDb.getReflogReader("refs/heads/prefix").getReverseEntries() + .get(0).getComment()); + assertEquals("Just a message", + refDb.getReflogReader("refs/heads/prefix").getReverseEntries() + .get(1).getComment()); + assertEquals("Setup", refDb.getReflogReader("refs/heads/prefix") .getReverseEntries().get(2).getComment()); assertEquals("Branch: renamed prefix/a to prefix", refDb.getReflogReader("HEAD").getReverseEntries().get(0) diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/ReflogReaderTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/ReflogReaderTest.java index 23c279eabe..16645cbcd7 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/ReflogReaderTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/ReflogReaderTest.java @@ -157,9 +157,11 @@ public class ReflogReaderTest extends SampleDataRepositoryTestCase { setupReflog("logs/HEAD", headLine); RefDatabase refDb = db.getRefDatabase(); assertEquals("branch: change to master", - refDb.getReflogReader("master").getLastEntry().getComment()); + refDb.getReflogReader("refs/heads/master").getLastEntry() + .getComment()); assertEquals("branch: change to a", - refDb.getReflogReader("a").getLastEntry().getComment()); + refDb.getReflogReader("refs/heads/a").getLastEntry() + .getComment()); assertEquals("branch: change to HEAD", refDb.getReflogReader("HEAD").getLastEntry().getComment()); } @@ -168,7 +170,7 @@ public class ReflogReaderTest extends SampleDataRepositoryTestCase { public void testReadLineWithMissingComment() throws Exception { setupReflog("logs/refs/heads/master", oneLineWithoutComment); final ReflogReader reader = db.getRefDatabase() - .getReflogReader("master"); + .getReflogReader("refs/heads/master"); ReflogEntry e = reader.getLastEntry(); assertEquals(ObjectId .fromString("da85355dfc525c9f6f3927b876f379f46ccf826e"), e @@ -188,8 +190,9 @@ public class ReflogReaderTest extends SampleDataRepositoryTestCase { public void testNoLog() throws Exception { RefDatabase refDb = db.getRefDatabase(); assertEquals(0, - refDb.getReflogReader("master").getReverseEntries().size()); - assertNull(refDb.getReflogReader("master").getLastEntry()); + refDb.getReflogReader("refs/heads/master").getReverseEntries() + .size()); + assertNull(refDb.getReflogReader("refs/heads/master").getLastEntry()); } @Test diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java index 3b09d3a82f..6040ad554d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java @@ -566,7 +566,7 @@ public class FileReftableDatabase extends RefDatabase { RefDatabase refDb = repo.getRefDatabase(); if (writeLogs) { for (Ref r : refs) { - ReflogReader rlr = refDb.getReflogReader(r.getName()); + ReflogReader rlr = refDb.getReflogReader(r); if (rlr != null) { size = Math.max(rlr.getReverseEntries().size(), size); } @@ -589,10 +589,7 @@ public class FileReftableDatabase extends RefDatabase { if (writeLogs) { for (Ref r : refs) { long idx = size; - ReflogReader reader = refDb.getReflogReader(r.getName()); - if (reader == null) { - continue; - } + ReflogReader reader = refDb.getReflogReader(r); for (ReflogEntry e : reader.getReverseEntries()) { w.writeLog(r.getName(), idx, e.getWho(), e.getOldId(), e.getNewId(), e.getComment()); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefDatabase.java index 11d1732c77..49d5224325 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefDatabase.java @@ -368,9 +368,7 @@ public abstract class RefDatabase { */ @Nullable public ReflogReader getReflogReader(String refName) throws IOException { - // TODO: use exactRef here, which offers more predictable and therefore - // preferable behavior. - Ref ref = findRef(refName); + Ref ref = exactRef(refName); if (ref == null) { return null; } -- cgit v1.2.3 From 32b9a9523c2a8100f9ac22b251a10c3afb7d1144 Mon Sep 17 00:00:00 2001 From: Simon Eder Date: Wed, 9 Oct 2024 17:16:57 +0200 Subject: Submodules: Update submodule with deleted worktree The current implementation throws an exception when a submodule is updated for which the 'gitdir' still exists but the 'worktree' is missing. The current implementation tries to clone the submodule but fails as the 'gitdir' is not empty. The commit adds: - a check if the 'gitdir' is empty - if the 'gitdir' is not empty: - create a new '.git' file linking to the existing 'gitdir' - fetch the submodule - checkout the submodule unconditionally (ignore any configured update mode) - if the submodule is cloned checkout the submodule unconditionally (ignore any configured update mode) This change mimics the behavior of cgit. Bug: jgit-79 Change-Id: Iffc8659d2a04d866a43815c78c7760c0f3d82640 --- .../jgit/submodule/SubmoduleUpdateTest.java | 210 +++++++++++++++++---- .../eclipse/jgit/api/SubmoduleUpdateCommand.java | 91 +++++++-- .../src/org/eclipse/jgit/lib/ConfigConstants.java | 7 + 3 files changed, 253 insertions(+), 55 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/submodule/SubmoduleUpdateTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/submodule/SubmoduleUpdateTest.java index c5e9c2deaa..d54117005d 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/submodule/SubmoduleUpdateTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/submodule/SubmoduleUpdateTest.java @@ -17,21 +17,25 @@ import java.io.File; import java.io.IOException; import java.util.Collection; +import org.eclipse.jgit.api.CheckoutCommand; import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.InitCommand; +import org.eclipse.jgit.api.SubmoduleAddCommand; import org.eclipse.jgit.api.SubmoduleUpdateCommand; import org.eclipse.jgit.api.errors.GitAPIException; import org.eclipse.jgit.dircache.DirCache; import org.eclipse.jgit.dircache.DirCacheEditor; import org.eclipse.jgit.dircache.DirCacheEditor.PathEdit; import org.eclipse.jgit.dircache.DirCacheEntry; +import org.eclipse.jgit.junit.JGitTestUtil; import org.eclipse.jgit.junit.RepositoryTestCase; +import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.FileMode; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.lib.StoredConfig; -import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.storage.file.FileBasedConfig; import org.junit.Test; @@ -40,6 +44,91 @@ import org.junit.Test; */ public class SubmoduleUpdateTest extends RepositoryTestCase { + private Repository submoduleRepo; + + private Git git; + + private AnyObjectId subRepoCommit2; + + private void createSubmoduleRepo() throws IOException, GitAPIException { + File directory = createTempDirectory("submodule_repo"); + InitCommand init = Git.init(); + init.setDirectory(directory); + init.call(); + submoduleRepo = Git.open(directory).getRepository(); + try (Git sub = Git.wrap(submoduleRepo)) { + // commit something + JGitTestUtil.writeTrashFile(submoduleRepo, "commit1.txt", + "commit 1"); + sub.add().addFilepattern("commit1.txt").call(); + sub.commit().setMessage("commit 1").call().getId(); + + JGitTestUtil.writeTrashFile(submoduleRepo, "commit2.txt", + "commit 2"); + sub.add().addFilepattern("commit2.txt").call(); + subRepoCommit2 = sub.commit().setMessage("commit 2").call().getId(); + } + } + + private void addSubmodule(String path) throws GitAPIException { + SubmoduleAddCommand command = new SubmoduleAddCommand(db); + command.setPath(path); + String uri = submoduleRepo.getDirectory().toURI().toString(); + command.setURI(uri); + try (Repository repo = command.call()) { + assertNotNull(repo); + } + git.add().addFilepattern(path).addFilepattern(Constants.DOT_GIT_MODULES) + .call(); + git.commit().setMessage("adding submodule").call(); + recursiveDelete(new File(git.getRepository().getWorkTree(), path)); + recursiveDelete( + new File(new File(git.getRepository().getCommonDirectory(), + Constants.MODULES), path)); + } + + @Override + public void setUp() throws Exception { + super.setUp(); + createSubmoduleRepo(); + + git = Git.wrap(db); + // commit something + writeTrashFile("initial.txt", "initial"); + git.add().addFilepattern("initial.txt").call(); + git.commit().setMessage("initial commit").call(); + } + + public void updateModeClonedRestoredSubmoduleTemplate(String mode) + throws Exception { + String path = "sub"; + addSubmodule(path); + + StoredConfig cfg = git.getRepository().getConfig(); + if (mode != null) { + cfg.load(); + cfg.setString(ConfigConstants.CONFIG_SUBMODULE_SECTION, path, + ConfigConstants.CONFIG_KEY_UPDATE, mode); + cfg.save(); + } + SubmoduleUpdateCommand update = new SubmoduleUpdateCommand(db); + update.call(); + try (Git subGit = Git.open(new File(db.getWorkTree(), path))) { + update.call(); + assertEquals(subRepoCommit2.getName(), + subGit.getRepository().getBranch()); + } + + recursiveDelete(new File(db.getWorkTree(), path)); + + update.call(); + try (Git subGit = Git.open(new File(db.getWorkTree(), path))) { + update.call(); + assertEquals(subRepoCommit2.getName(), + subGit.getRepository().getBranch()); + } + } + @Test public void repositoryWithNoSubmodules() throws GitAPIException { SubmoduleUpdateCommand command = new SubmoduleUpdateCommand(db); @@ -50,35 +139,9 @@ public class SubmoduleUpdateTest extends RepositoryTestCase { @Test public void repositoryWithSubmodule() throws Exception { - writeTrashFile("file.txt", "content"); - Git git = Git.wrap(db); - git.add().addFilepattern("file.txt").call(); - final RevCommit commit = git.commit().setMessage("create file").call(); final String path = "sub"; - DirCache cache = db.lockDirCache(); - DirCacheEditor editor = cache.editor(); - editor.add(new PathEdit(path) { - - @Override - public void apply(DirCacheEntry ent) { - ent.setFileMode(FileMode.GITLINK); - ent.setObjectId(commit); - } - }); - editor.commit(); - - StoredConfig config = db.getConfig(); - config.setString(ConfigConstants.CONFIG_SUBMODULE_SECTION, path, - ConfigConstants.CONFIG_KEY_URL, db.getDirectory().toURI() - .toString()); - config.save(); - - FileBasedConfig modulesConfig = new FileBasedConfig(new File( - db.getWorkTree(), Constants.DOT_GIT_MODULES), db.getFS()); - modulesConfig.setString(ConfigConstants.CONFIG_SUBMODULE_SECTION, path, - ConfigConstants.CONFIG_KEY_PATH, path); - modulesConfig.save(); + addSubmodule(path); SubmoduleUpdateCommand command = new SubmoduleUpdateCommand(db); Collection updated = command.call(); @@ -90,7 +153,7 @@ public class SubmoduleUpdateTest extends RepositoryTestCase { assertTrue(generator.next()); try (Repository subRepo = generator.getRepository()) { assertNotNull(subRepo); - assertEquals(commit, subRepo.resolve(Constants.HEAD)); + assertEquals(subRepoCommit2, subRepo.resolve(Constants.HEAD)); String worktreeDir = subRepo.getConfig().getString( ConfigConstants.CONFIG_CORE_SECTION, null, ConfigConstants.CONFIG_KEY_WORKTREE); @@ -104,8 +167,8 @@ public class SubmoduleUpdateTest extends RepositoryTestCase { } @Test - public void repositoryWithUnconfiguredSubmodule() throws IOException, - GitAPIException { + public void repositoryWithUnconfiguredSubmodule() + throws IOException, GitAPIException { final ObjectId id = ObjectId .fromString("abcd1234abcd1234abcd1234abcd1234abcd1234"); final String path = "sub"; @@ -121,16 +184,14 @@ public class SubmoduleUpdateTest extends RepositoryTestCase { }); editor.commit(); - FileBasedConfig modulesConfig = new FileBasedConfig(new File( - db.getWorkTree(), Constants.DOT_GIT_MODULES), db.getFS()); + FileBasedConfig modulesConfig = new FileBasedConfig( + new File(db.getWorkTree(), Constants.DOT_GIT_MODULES), + db.getFS()); modulesConfig.setString(ConfigConstants.CONFIG_SUBMODULE_SECTION, path, ConfigConstants.CONFIG_KEY_PATH, path); String url = "git://server/repo.git"; modulesConfig.setString(ConfigConstants.CONFIG_SUBMODULE_SECTION, path, ConfigConstants.CONFIG_KEY_URL, url); - String update = "rebase"; - modulesConfig.setString(ConfigConstants.CONFIG_SUBMODULE_SECTION, path, - ConfigConstants.CONFIG_KEY_UPDATE, update); modulesConfig.save(); SubmoduleUpdateCommand command = new SubmoduleUpdateCommand(db); @@ -140,8 +201,8 @@ public class SubmoduleUpdateTest extends RepositoryTestCase { } @Test - public void repositoryWithInitializedSubmodule() throws IOException, - GitAPIException { + public void repositoryWithInitializedSubmodule() + throws IOException, GitAPIException { final ObjectId id = ObjectId .fromString("abcd1234abcd1234abcd1234abcd1234abcd1234"); final String path = "sub"; @@ -168,4 +229,77 @@ public class SubmoduleUpdateTest extends RepositoryTestCase { assertNotNull(updated); assertTrue(updated.isEmpty()); } + + @Test + public void updateModeMergeClonedRestoredSubmodule() throws Exception { + updateModeClonedRestoredSubmoduleTemplate( + ConfigConstants.CONFIG_KEY_MERGE); + } + + @Test + public void updateModeRebaseClonedRestoredSubmodule() throws Exception { + updateModeClonedRestoredSubmoduleTemplate( + ConfigConstants.CONFIG_KEY_REBASE); + } + + @Test + public void updateModeCheckoutClonedRestoredSubmodule() throws Exception { + updateModeClonedRestoredSubmoduleTemplate( + ConfigConstants.CONFIG_KEY_CHECKOUT); + } + + @Test + public void updateModeMissingClonedRestoredSubmodule() throws Exception { + updateModeClonedRestoredSubmoduleTemplate(null); + } + + @Test + public void updateMode() throws Exception { + String path = "sub"; + addSubmodule(path); + + StoredConfig cfg = git.getRepository().getConfig(); + cfg.load(); + cfg.setString(ConfigConstants.CONFIG_SUBMODULE_SECTION, path, + ConfigConstants.CONFIG_KEY_UPDATE, + ConfigConstants.CONFIG_KEY_REBASE); + cfg.save(); + + SubmoduleUpdateCommand update = new SubmoduleUpdateCommand(db); + update.call(); + try (Git subGit = Git.open(new File(db.getWorkTree(), path))) { + CheckoutCommand checkout = subGit.checkout(); + checkout.setName("master"); + checkout.call(); + update.call(); + assertEquals("master", subGit.getRepository().getBranch()); + } + + cfg.load(); + cfg.setString(ConfigConstants.CONFIG_SUBMODULE_SECTION, path, + ConfigConstants.CONFIG_KEY_UPDATE, + ConfigConstants.CONFIG_KEY_CHECKOUT); + cfg.save(); + + update.call(); + try (Git subGit = Git.open(new File(db.getWorkTree(), path))) { + assertEquals(subRepoCommit2.getName(), + subGit.getRepository().getBranch()); + } + + cfg.load(); + cfg.setString(ConfigConstants.CONFIG_SUBMODULE_SECTION, path, + ConfigConstants.CONFIG_KEY_UPDATE, + ConfigConstants.CONFIG_KEY_MERGE); + cfg.save(); + + update.call(); + try (Git subGit = Git.open(new File(db.getWorkTree(), path))) { + CheckoutCommand checkout = subGit.checkout(); + checkout.setName("master"); + checkout.call(); + update.call(); + assertEquals("master", subGit.getRepository().getBranch()); + } + } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleUpdateCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleUpdateCommand.java index 3524984347..5e4b2ee0b7 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleUpdateCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/SubmoduleUpdateCommand.java @@ -28,6 +28,7 @@ import org.eclipse.jgit.api.errors.RefNotFoundException; import org.eclipse.jgit.api.errors.WrongRepositoryStateException; import org.eclipse.jgit.dircache.DirCacheCheckout; import org.eclipse.jgit.errors.ConfigInvalidException; +import org.eclipse.jgit.internal.storage.file.LockFile; import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.NullProgressMonitor; @@ -39,6 +40,7 @@ import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.submodule.SubmoduleWalk; import org.eclipse.jgit.treewalk.filter.PathFilterGroup; +import org.eclipse.jgit.util.FileUtils; /** * A class used to execute a submodule update command. @@ -62,6 +64,8 @@ public class SubmoduleUpdateCommand extends private boolean fetch = false; + private boolean clonedRestored; + /** *

* Constructor for SubmoduleUpdateCommand. @@ -116,26 +120,77 @@ public class SubmoduleUpdateCommand extends return this; } + private static boolean submoduleExists(File gitDir) { + if (gitDir != null && gitDir.isDirectory()) { + File[] files = gitDir.listFiles(); + return files != null && files.length != 0; + } + return false; + } + + private static void restoreSubmodule(File gitDir, File workingTree) + throws IOException { + LockFile dotGitLock = new LockFile( + new File(workingTree, Constants.DOT_GIT)); + if (dotGitLock.lock()) { + String content = Constants.GITDIR + + getRelativePath(gitDir, workingTree); + dotGitLock.write(Constants.encode(content)); + dotGitLock.commit(); + } + } + + private static String getRelativePath(File gitDir, File workingTree) { + File relPath; + try { + relPath = workingTree.toPath().relativize(gitDir.toPath()) + .toFile(); + } catch (IllegalArgumentException e) { + relPath = gitDir; + } + return FileUtils.pathToString(relPath); + } + + private String determineUpdateMode(String mode) { + if (clonedRestored) { + return ConfigConstants.CONFIG_KEY_CHECKOUT; + } + return mode; + } + private Repository getOrCloneSubmodule(SubmoduleWalk generator, String url) throws IOException, GitAPIException { Repository repository = generator.getRepository(); + boolean restored = false; + boolean cloned = false; if (repository == null) { - if (callback != null) { - callback.cloningSubmodule(generator.getPath()); - } - CloneCommand clone = Git.cloneRepository(); - configure(clone); - clone.setURI(url); - clone.setDirectory(generator.getDirectory()); - clone.setGitDir(new File( + File gitDir = new File( new File(repo.getCommonDirectory(), Constants.MODULES), - generator.getPath())); - clone.setRelativePaths(true); - if (monitor != null) { - clone.setProgressMonitor(monitor); + generator.getPath()); + if (submoduleExists(gitDir)) { + restoreSubmodule(gitDir, generator.getDirectory()); + restored = true; + clonedRestored = true; + repository = generator.getRepository(); + } else { + if (callback != null) { + callback.cloningSubmodule(generator.getPath()); + } + CloneCommand clone = Git.cloneRepository(); + configure(clone); + clone.setURI(url); + clone.setDirectory(generator.getDirectory()); + clone.setGitDir(gitDir); + clone.setRelativePaths(true); + if (monitor != null) { + clone.setProgressMonitor(monitor); + } + repository = clone.call().getRepository(); + cloned = true; + clonedRestored = true; } - repository = clone.call().getRepository(); - } else if (this.fetch) { + } + if ((this.fetch || restored) && !cloned) { if (fetchCallback != null) { fetchCallback.fetchingSubmodule(generator.getPath()); } @@ -172,15 +227,17 @@ public class SubmoduleUpdateCommand extends continue; // Skip submodules not registered in parent repository's config String url = generator.getConfigUrl(); - if (url == null) + if (url == null) { continue; - + } + clonedRestored = false; try (Repository submoduleRepo = getOrCloneSubmodule(generator, url); RevWalk walk = new RevWalk(submoduleRepo)) { RevCommit commit = walk .parseCommit(generator.getObjectId()); - String update = generator.getConfigUpdate(); + String update = determineUpdateMode( + generator.getConfigUpdate()); if (ConfigConstants.CONFIG_KEY_MERGE.equals(update)) { MergeCommand merge = new MergeCommand(submoduleRepo); merge.include(commit); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java index a57f1b714a..5068a6cce0 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java @@ -446,6 +446,13 @@ public final class ConfigConstants { /** The "rebase" key */ public static final String CONFIG_KEY_REBASE = "rebase"; + /** + * The "checkout" key + * + * @since 7.2 + */ + public static final String CONFIG_KEY_CHECKOUT = "checkout"; + /** The "url" key */ public static final String CONFIG_KEY_URL = "url"; -- cgit v1.2.3 From 0a3d13508aa9ef4dc6f5ad97014f2f0ea5568195 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Mon, 27 Jan 2025 22:49:09 +0100 Subject: LooseObjects: convert while loop into for loop to simplify the implementation and shorten variable names. Change-Id: I84c038e65263b7d0c38c4e90817ab8b07ad5c2da --- .../org/eclipse/jgit/internal/storage/file/LooseObjects.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LooseObjects.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LooseObjects.java index b4bb2a9293..ce565308be 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LooseObjects.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LooseObjects.java @@ -49,7 +49,7 @@ class LooseObjects { * Maximum number of attempts to read a loose object for which a stale file * handle exception is thrown */ - private final static int MAX_LOOSE_OBJECT_STALE_READ_ATTEMPTS = 5; + private final static int MAX_STALE_READ_RETRIES = 5; private final File directory; @@ -163,9 +163,7 @@ class LooseObjects { } ObjectLoader open(WindowCursor curs, AnyObjectId id) throws IOException { - int readAttempts = 0; - while (readAttempts < MAX_LOOSE_OBJECT_STALE_READ_ATTEMPTS) { - readAttempts++; + for (int retries = 0; retries < MAX_STALE_READ_RETRIES; retries++) { File path = fileFor(id); if (trustFolderStat && !path.exists()) { break; @@ -184,8 +182,8 @@ class LooseObjects { if (LOG.isDebugEnabled()) { LOG.debug(MessageFormat.format( JGitText.get().looseObjectHandleIsStale, id.name(), - Integer.valueOf(readAttempts), Integer.valueOf( - MAX_LOOSE_OBJECT_STALE_READ_ATTEMPTS))); + Integer.valueOf(retries), Integer.valueOf( + MAX_STALE_READ_RETRIES))); } } } -- cgit v1.2.3 From 12b89ab6dbc897123089492741270a762911a8f3 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Mon, 27 Jan 2025 23:23:49 +0100 Subject: LooseObjects: compute loose object path before retry loop It's unnecessary to recompute this for retries. Change-Id: I383ecfa25d795ba177dd5ce14d12b318e38daf9b --- .../src/org/eclipse/jgit/internal/storage/file/LooseObjects.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LooseObjects.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LooseObjects.java index ce565308be..74ef6e139c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LooseObjects.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LooseObjects.java @@ -163,8 +163,8 @@ class LooseObjects { } ObjectLoader open(WindowCursor curs, AnyObjectId id) throws IOException { + File path = fileFor(id); for (int retries = 0; retries < MAX_STALE_READ_RETRIES; retries++) { - File path = fileFor(id); if (trustFolderStat && !path.exists()) { break; } -- cgit v1.2.3 From 772ee0e5fb9fbfb4a82ceb872e187abf62a4b453 Mon Sep 17 00:00:00 2001 From: Luca Milanesio Date: Wed, 29 Jan 2025 10:17:17 +0000 Subject: Log pruned packfiles as debug and not warn logs In the regular lifecycle of a Git repository, the packfiles are constantly created and removed as part of repacking and pruning. Lower the logging of disappearing packfiles to debug level to reduce the SPAMing on logfile with warnings that should be ignored as part of the normal repository life. Change-Id: I57b24ffb0b32f7a64810cab47b5009f2bf89a6b8 --- .../src/org/eclipse/jgit/internal/storage/file/PackDirectory.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java index e31126f027..706dbd6281 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java @@ -314,6 +314,7 @@ class PackDirectory { private void handlePackError(IOException e, Pack p) { String warnTmpl = null; + String debugTmpl = null; int transientErrorCount = 0; String errTmpl = JGitText.get().exceptionWhileReadingPack; if ((e instanceof CorruptObjectException) @@ -328,7 +329,7 @@ class PackDirectory { errTmpl = JGitText.get().packInaccessible; transientErrorCount = p.incrementTransientErrorCount(); } else { - warnTmpl = JGitText.get().packWasDeleted; + debugTmpl = JGitText.get().packWasDeleted; remove(p); } } else if (FileUtils.isStaleFileHandleInCausalChain(e)) { @@ -340,6 +341,9 @@ class PackDirectory { if (warnTmpl != null) { LOG.warn(MessageFormat.format(warnTmpl, p.getPackFile().getAbsolutePath()), e); + } else if (debugTmpl != null) { + LOG.debug(MessageFormat.format(debugTmpl, + p.getPackFile().getAbsolutePath()), e); } else { if (doLogExponentialBackoff(transientErrorCount)) { // Don't remove the pack from the list, as the error may be -- cgit v1.2.3 From c3c7f680c6bee238617f25f6aa85d0b565fc8ecb Mon Sep 17 00:00:00 2001 From: Luca Milanesio Date: Thu, 30 Jan 2025 18:20:02 +0000 Subject: PackDirectory: make template variables names more readable Follow-up of I57b24ffb0b3 addressing a readability issue on the variable names of the PackDirectory.handlePackError() method. Change-Id: I877a45328e9b7026e66f300c363ea5c1625af157 --- .../jgit/internal/storage/file/PackDirectory.java | 26 +++++++++++----------- 1 file changed, 13 insertions(+), 13 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java index 706dbd6281..1ce14b3a18 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java @@ -313,42 +313,42 @@ class PackDirectory { } private void handlePackError(IOException e, Pack p) { - String warnTmpl = null; - String debugTmpl = null; + String warnTemplate = null; + String debugTemplate = null; int transientErrorCount = 0; - String errTmpl = JGitText.get().exceptionWhileReadingPack; + String errorTemplate = JGitText.get().exceptionWhileReadingPack; if ((e instanceof CorruptObjectException) || (e instanceof PackInvalidException)) { - warnTmpl = JGitText.get().corruptPack; - LOG.warn(MessageFormat.format(warnTmpl, + warnTemplate = JGitText.get().corruptPack; + LOG.warn(MessageFormat.format(warnTemplate, p.getPackFile().getAbsolutePath()), e); // Assume the pack is corrupted, and remove it from the list. remove(p); } else if (e instanceof FileNotFoundException) { if (p.getPackFile().exists()) { - errTmpl = JGitText.get().packInaccessible; + errorTemplate = JGitText.get().packInaccessible; transientErrorCount = p.incrementTransientErrorCount(); } else { - debugTmpl = JGitText.get().packWasDeleted; + debugTemplate = JGitText.get().packWasDeleted; remove(p); } } else if (FileUtils.isStaleFileHandleInCausalChain(e)) { - warnTmpl = JGitText.get().packHandleIsStale; + warnTemplate = JGitText.get().packHandleIsStale; remove(p); } else { transientErrorCount = p.incrementTransientErrorCount(); } - if (warnTmpl != null) { - LOG.warn(MessageFormat.format(warnTmpl, + if (warnTemplate != null) { + LOG.warn(MessageFormat.format(warnTemplate, p.getPackFile().getAbsolutePath()), e); - } else if (debugTmpl != null) { - LOG.debug(MessageFormat.format(debugTmpl, + } else if (debugTemplate != null) { + LOG.debug(MessageFormat.format(debugTemplate, p.getPackFile().getAbsolutePath()), e); } else { if (doLogExponentialBackoff(transientErrorCount)) { // Don't remove the pack from the list, as the error may be // transient. - LOG.error(MessageFormat.format(errTmpl, + LOG.error(MessageFormat.format(errorTemplate, p.getPackFile().getAbsolutePath(), Integer.valueOf(transientErrorCount)), e); } -- cgit v1.2.3 From a8efd046fa5f47351f91b5dddd13becb83b9bdc8 Mon Sep 17 00:00:00 2001 From: Martin Fick Date: Wed, 29 Jan 2025 09:47:01 -0800 Subject: RevWalk: Add an isMergedIntoAnyCommit() method RevWalk had a bulk isMergedIntoAny() method, however it worked only for Refs. Add a similar method which works for RevCommits instead. Unfortunately isMergedIntoAny() cannot be overloaded since java does not include the generic types in Collections of method signatures, so the method name needs to be more complicated to differentiate it from the existing method. Change-Id: I4f8f3a83058a186fafe3b37726e21c5074a6b8e1 Signed-off-by: Martin Fick --- .../jgit/revwalk/RevWalkUtilsReachableTest.java | 3 +- org.eclipse.jgit/.settings/.api_filters | 8 ++++ .../src/org/eclipse/jgit/revwalk/RevWalk.java | 56 ++++++++++++++++++---- 3 files changed, 58 insertions(+), 9 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/RevWalkUtilsReachableTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/RevWalkUtilsReachableTest.java index 0a045c917b..ffc7c96f69 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/RevWalkUtilsReachableTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/revwalk/RevWalkUtilsReachableTest.java @@ -14,6 +14,7 @@ import static java.util.Arrays.asList; import static org.junit.Assert.assertEquals; import java.util.Collection; +import java.util.HashSet; import java.util.List; import org.eclipse.jgit.api.Git; @@ -121,7 +122,7 @@ public class RevWalkUtilsReachableTest extends RevWalkTestCase { Collection sortedRefs = RefComparator.sort(allRefs); List actual = RevWalkUtils.findBranchesReachableFrom(commit, rw, sortedRefs); - assertEquals(refsThatShouldContainCommit, actual); + assertEquals(new HashSet<>(refsThatShouldContainCommit), new HashSet<>(actual)); } } diff --git a/org.eclipse.jgit/.settings/.api_filters b/org.eclipse.jgit/.settings/.api_filters index 94adb25f78..4d7d7d07b7 100644 --- a/org.eclipse.jgit/.settings/.api_filters +++ b/org.eclipse.jgit/.settings/.api_filters @@ -80,6 +80,14 @@ + + + + + + + + diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevWalk.java b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevWalk.java index 76c14e9c26..9714fab325 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevWalk.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/revwalk/RevWalk.java @@ -19,9 +19,14 @@ import java.text.MessageFormat; import java.util.ArrayList; import java.util.Collection; import java.util.EnumSet; +import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; -import java.util.Optional; +import java.util.Map; +import java.util. +Optional; +import java.util.Set; import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.annotations.Nullable; @@ -539,6 +544,27 @@ public class RevWalk implements Iterable, AutoCloseable { NullProgressMonitor.INSTANCE).size() > 0; } + /** + * Determine if a commit is merged into any of the given + * revs. + * + * @param commit + * commit the caller thinks is reachable from revs. + * @param revs + * commits to start iteration from, and which is most likely a + * descendant (child) of commit. + * @return true if commit is merged into any of the revs; false otherwise. + * @throws java.io.IOException + * a pack file or loose object could not be read. + * @since 6.10.1 + */ + public boolean isMergedIntoAnyCommit(RevCommit commit, Collection revs) + throws IOException { + return getCommitsMergedInto(commit, revs, + GetMergedIntoStrategy.RETURN_ON_FIRST_FOUND, + NullProgressMonitor.INSTANCE).size() > 0; + } + /** * Determine if a commit is merged into all of the given * refs. @@ -562,7 +588,26 @@ public class RevWalk implements Iterable, AutoCloseable { private List getMergedInto(RevCommit needle, Collection haystacks, Enum returnStrategy, ProgressMonitor monitor) throws IOException { + Map> refsByCommit = new HashMap<>(); + for (Ref r : haystacks) { + RevObject o = peel(parseAny(r.getObjectId())); + if (!(o instanceof RevCommit)) { + continue; + } + refsByCommit.computeIfAbsent((RevCommit) o, c -> new ArrayList<>()).add(r); + } + monitor.update(1); List result = new ArrayList<>(); + for (RevCommit c : getCommitsMergedInto(needle, refsByCommit.keySet(), + returnStrategy, monitor)) { + result.addAll(refsByCommit.get(c)); + } + return result; + } + + private Set getCommitsMergedInto(RevCommit needle, Collection haystacks, + Enum returnStrategy, ProgressMonitor monitor) throws IOException { + Set result = new HashSet<>(); List uninteresting = new ArrayList<>(); List marked = new ArrayList<>(); RevFilter oldRF = filter; @@ -578,16 +623,11 @@ public class RevWalk implements Iterable, AutoCloseable { needle.parseHeaders(this); } int cutoff = needle.getGeneration(); - for (Ref r : haystacks) { + for (RevCommit c : haystacks) { if (monitor.isCancelled()) { return result; } monitor.update(1); - RevObject o = peel(parseAny(r.getObjectId())); - if (!(o instanceof RevCommit)) { - continue; - } - RevCommit c = (RevCommit) o; reset(UNINTERESTING | TEMP_MARK); markStart(c); boolean commitFound = false; @@ -599,7 +639,7 @@ public class RevWalk implements Iterable, AutoCloseable { } if (References.isSameObject(next, needle) || (next.flags & TEMP_MARK) != 0) { - result.add(r); + result.add(c); if (returnStrategy == GetMergedIntoStrategy.RETURN_ON_FIRST_FOUND) { return result; } -- cgit v1.2.3 From ccef9d428c219d92a6300ec91af4f8245dc32cd4 Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Thu, 30 Jan 2025 13:38:02 -0800 Subject: PackIndex.MutableEntry: new methods to copy its contents To build the multipack index, we will iterate over N indexes merging the results in sha1 order. With the current MutableEntry API, we need to create an instance of ObjectId to either compare and/or copy the selected entry to the multi index iterator. Allow MutableEntries to compare to each other without creating any new object, and copy their object id directly to another MutableObjectId. This allows to build other iterators with mutable entries that copy the required memory (few ints) instead of creating short-lived instances of ObjectId. Change-Id: I0c0091420d7c0847c03afdc9e73b566bb4eeba40 --- .../internal/storage/file/PackIndexTestCase.java | 83 +++++++++++++++------- .../jgit/internal/storage/file/PackIndex.java | 36 ++++++++-- 2 files changed, 91 insertions(+), 28 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackIndexTestCase.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackIndexTestCase.java index 24bdc4a97a..1f934acced 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackIndexTestCase.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/PackIndexTestCase.java @@ -13,6 +13,7 @@ package org.eclipse.jgit.internal.storage.file; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.File; @@ -25,6 +26,7 @@ import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.file.PackIndex.MutableEntry; import org.eclipse.jgit.junit.RepositoryTestCase; +import org.eclipse.jgit.lib.MutableObjectId; import org.eclipse.jgit.lib.ObjectId; import org.junit.Test; @@ -99,6 +101,39 @@ public abstract class PackIndexTestCase extends RepositoryTestCase { } } + @Test + public void testIteratorMutableEntryCompareTo() { + Iterator iterA = smallIdx.iterator(); + Iterator iterB = smallIdx.iterator(); + + MutableEntry aEntry = iterA.next(); + iterB.next(); + MutableEntry bEntry = iterB.next(); + // b is one ahead + assertTrue(aEntry.compareBySha1To(bEntry) < 0); + assertTrue(bEntry.compareBySha1To(aEntry) > 0); + + // advance a, now should be equal + assertEquals(0, iterA.next().compareBySha1To(bEntry)); + } + + @Test + public void testIteratorMutableEntryCopyTo() { + Iterator it = smallIdx.iterator(); + + MutableObjectId firstOidCopy = new MutableObjectId(); + MutableEntry next = it.next(); + next.copyOidTo(firstOidCopy); + ObjectId firstImmutable = next.toObjectId(); + + MutableEntry second = it.next(); + + // The copy has the right value after "next" + assertTrue(firstImmutable.equals(firstOidCopy)); + assertFalse("iterator has moved", + second.toObjectId().equals(firstImmutable)); + } + /** * Test results of iterator comparing to content of well-known (prepared) * small index. @@ -106,22 +141,22 @@ public abstract class PackIndexTestCase extends RepositoryTestCase { @Test public void testIteratorReturnedValues1() { Iterator iter = smallIdx.iterator(); - assertEquals("4b825dc642cb6eb9a060e54bf8d69288fbee4904", iter.next() - .name()); - assertEquals("540a36d136cf413e4b064c2b0e0a4db60f77feab", iter.next() - .name()); - assertEquals("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259", iter.next() - .name()); - assertEquals("6ff87c4664981e4397625791c8ea3bbb5f2279a3", iter.next() - .name()); - assertEquals("82c6b885ff600be425b4ea96dee75dca255b69e7", iter.next() - .name()); - assertEquals("902d5476fa249b7abc9d84c611577a81381f0327", iter.next() - .name()); - assertEquals("aabf2ffaec9b497f0950352b3e582d73035c2035", iter.next() - .name()); - assertEquals("c59759f143fb1fe21c197981df75a7ee00290799", iter.next() - .name()); + assertEquals("4b825dc642cb6eb9a060e54bf8d69288fbee4904", + iter.next().name()); + assertEquals("540a36d136cf413e4b064c2b0e0a4db60f77feab", + iter.next().name()); + assertEquals("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259", + iter.next().name()); + assertEquals("6ff87c4664981e4397625791c8ea3bbb5f2279a3", + iter.next().name()); + assertEquals("82c6b885ff600be425b4ea96dee75dca255b69e7", + iter.next().name()); + assertEquals("902d5476fa249b7abc9d84c611577a81381f0327", + iter.next().name()); + assertEquals("aabf2ffaec9b497f0950352b3e582d73035c2035", + iter.next().name()); + assertEquals("c59759f143fb1fe21c197981df75a7ee00290799", + iter.next().name()); assertFalse(iter.hasNext()); } @@ -198,16 +233,16 @@ public abstract class PackIndexTestCase extends RepositoryTestCase { @Test public void testIteratorReturnedValues2() { Iterator iter = denseIdx.iterator(); - while (!iter.next().name().equals( - "0a3d7772488b6b106fb62813c4d6d627918d9181")) { + while (!iter.next().name() + .equals("0a3d7772488b6b106fb62813c4d6d627918d9181")) { // just iterating } - assertEquals("1004d0d7ac26fbf63050a234c9b88a46075719d3", iter.next() - .name()); // same level-1 - assertEquals("10da5895682013006950e7da534b705252b03be6", iter.next() - .name()); // same level-1 - assertEquals("1203b03dc816ccbb67773f28b3c19318654b0bc8", iter.next() - .name()); + assertEquals("1004d0d7ac26fbf63050a234c9b88a46075719d3", + iter.next().name()); // same level-1 + assertEquals("10da5895682013006950e7da534b705252b03be6", + iter.next().name()); // same level-1 + assertEquals("1203b03dc816ccbb67773f28b3c19318654b0bc8", + iter.next().name()); } @Test diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java index 7189ce20a6..b3e4efb4fc 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackIndex.java @@ -286,7 +286,7 @@ public interface PackIndex * the index cannot be read. */ void resolve(Set matches, AbbreviatedObjectId id, - int matchLimit) throws IOException; + int matchLimit) throws IOException; /** * Get pack checksum @@ -304,6 +304,7 @@ public interface PackIndex class MutableEntry { /** Buffer of the ObjectId visited by the EntriesIterator. */ final MutableObjectId idBuffer = new MutableObjectId(); + /** Offset into the packfile of the current object. */ long offset; @@ -345,6 +346,34 @@ public interface PackIndex r.offset = offset; return r; } + + /** + * Similar to {@link Comparable#compareTo(Object)}, using only the + * object id in the entry. + * + * @param other + * Another mutable entry (probably from another index) + * + * @return a negative integer, zero, or a positive integer as this + * object is less than, equal to, or greater than the specified + * object. + */ + public int compareBySha1To(MutableEntry other) { + return idBuffer.compareTo(other.idBuffer); + } + + /** + * Copy the current ObjectId to dest + *

+ * Like {@link #toObjectId()}, but reusing the destination instead of + * creating a new ObjectId instance. + * + * @param dest + * destination for the object id + */ + public void copyOidTo(MutableObjectId dest) { + dest.fromObjectId(idBuffer); + } } /** @@ -368,7 +397,6 @@ public interface PackIndex this.objectCount = objectCount; } - @Override public boolean hasNext() { return returnedNumber < objectCount; @@ -393,7 +421,6 @@ public interface PackIndex */ protected abstract void readNext(); - /** * Copies to the entry an {@link ObjectId} from the int buffer and * position idx @@ -423,7 +450,8 @@ public interface PackIndex /** * Sets the {@code offset} to the entry * - * @param offset the offset in the pack file + * @param offset + * the offset in the pack file */ protected void setOffset(long offset) { entry.offset = offset; -- cgit v1.2.3 From 29e014aad0399fe8ede7c101d01b6e440ac9966b Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Thu, 23 Jan 2025 15:33:37 -0800 Subject: midx.PackIndexMerger: Helper to iterate over n-indexes The multipack index can be build merging the individual pack indexes and handling correctly the duplicates. PackIndexMerger is a utility function that gathers n-indexes and allows to iterate as one with the correct sha1 order. It also precalculates some of the needed metadata (as count of objects without duplicates, or if offsets go over 32 bits). Change-Id: I515b70fffff4489d83834488daf975f85726ea96 --- org.eclipse.jgit.test/META-INF/MANIFEST.MF | 1 + .../internal/storage/midx/PackIndexMergerTest.java | 239 +++++++++++++++ .../storage/midx/PackIndexPeekIteratorTest.java | 71 +++++ org.eclipse.jgit/META-INF/MANIFEST.MF | 1 + .../internal/storage/midx/PackIndexMerger.java | 337 +++++++++++++++++++++ 5 files changed, 649 insertions(+) create mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/midx/PackIndexMergerTest.java create mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/midx/PackIndexPeekIteratorTest.java create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/PackIndexMerger.java (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/META-INF/MANIFEST.MF b/org.eclipse.jgit.test/META-INF/MANIFEST.MF index 5a5dd88587..af5fd1c53a 100644 --- a/org.eclipse.jgit.test/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit.test/META-INF/MANIFEST.MF @@ -48,6 +48,7 @@ Import-Package: com.googlecode.javaewah;version="[1.1.6,2.0.0)", org.eclipse.jgit.internal.storage.file;version="[7.2.0,7.3.0)", org.eclipse.jgit.internal.storage.io;version="[7.2.0,7.3.0)", org.eclipse.jgit.internal.storage.memory;version="[7.2.0,7.3.0)", + org.eclipse.jgit.internal.storage.midx;version="[7.2.0,7.3.0)", org.eclipse.jgit.internal.storage.pack;version="[7.2.0,7.3.0)", org.eclipse.jgit.internal.storage.reftable;version="[7.2.0,7.3.0)", org.eclipse.jgit.internal.transport.connectivity;version="[7.2.0,7.3.0)", diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/midx/PackIndexMergerTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/midx/PackIndexMergerTest.java new file mode 100644 index 0000000000..1d8bde0f44 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/midx/PackIndexMergerTest.java @@ -0,0 +1,239 @@ +/* + * Copyright (C) 2025, Google Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.internal.storage.midx; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.Arrays; +import java.util.Iterator; +import java.util.Map; + +import org.eclipse.jgit.internal.storage.file.PackIndex; +import org.eclipse.jgit.junit.FakeIndexFactory; +import org.eclipse.jgit.junit.FakeIndexFactory.IndexObject; +import org.junit.Test; + +public class PackIndexMergerTest { + + @Test + public void rawIterator_noDuplicates() { + PackIndex idxOne = indexOf( + oidOffset("0000000000000000000000000000000000000001", 500), + oidOffset("0000000000000000000000000000000000000005", 12), + oidOffset("0000000000000000000000000000000000000010", 1500)); + PackIndex idxTwo = indexOf( + oidOffset("0000000000000000000000000000000000000002", 501), + oidOffset("0000000000000000000000000000000000000003", 13), + oidOffset("0000000000000000000000000000000000000015", 1501)); + PackIndex idxThree = indexOf( + oidOffset("0000000000000000000000000000000000000004", 502), + oidOffset("0000000000000000000000000000000000000007", 14), + oidOffset("0000000000000000000000000000000000000012", 1502)); + PackIndexMerger merger = new PackIndexMerger( + Map.of("p1", idxOne, "p2", idxTwo, "p3", idxThree)); + assertEquals(9, merger.getUniqueObjectCount()); + assertEquals(3, merger.getPackCount()); + assertFalse(merger.needsLargeOffsetsChunk()); + Iterator it = merger.rawIterator(); + assertNextEntry(it, "0000000000000000000000000000000000000001", 0, 500); + assertNextEntry(it, "0000000000000000000000000000000000000002", 1, 501); + assertNextEntry(it, "0000000000000000000000000000000000000003", 1, 13); + assertNextEntry(it, "0000000000000000000000000000000000000004", 2, 502); + assertNextEntry(it, "0000000000000000000000000000000000000005", 0, 12); + assertNextEntry(it, "0000000000000000000000000000000000000007", 2, 14); + assertNextEntry(it, "0000000000000000000000000000000000000010", 0, + 1500); + assertNextEntry(it, "0000000000000000000000000000000000000012", 2, + 1502); + assertNextEntry(it, "0000000000000000000000000000000000000015", 1, + 1501); + assertFalse(it.hasNext()); + } + + @Test + public void rawIterator_allDuplicates() { + PackIndex idxOne = indexOf( + oidOffset("0000000000000000000000000000000000000001", 500), + oidOffset("0000000000000000000000000000000000000005", 12), + oidOffset("0000000000000000000000000000000000000010", 1500)); + PackIndexMerger merger = new PackIndexMerger( + Map.of("p1", idxOne, "p2", idxOne, "p3", idxOne)); + assertEquals(3, merger.getUniqueObjectCount()); + assertEquals(3, merger.getPackCount()); + assertFalse(merger.needsLargeOffsetsChunk()); + Iterator it = merger.rawIterator(); + assertNextEntry(it, "0000000000000000000000000000000000000001", 0, 500); + assertNextEntry(it, "0000000000000000000000000000000000000001", 1, 500); + assertNextEntry(it, "0000000000000000000000000000000000000001", 2, 500); + assertNextEntry(it, "0000000000000000000000000000000000000005", 0, 12); + assertNextEntry(it, "0000000000000000000000000000000000000005", 1, 12); + assertNextEntry(it, "0000000000000000000000000000000000000005", 2, 12); + assertNextEntry(it, "0000000000000000000000000000000000000010", 0, + 1500); + assertNextEntry(it, "0000000000000000000000000000000000000010", 1, + 1500); + assertNextEntry(it, "0000000000000000000000000000000000000010", 2, + 1500); + assertFalse(it.hasNext()); + } + + @Test + public void bySha1Iterator_noDuplicates() { + PackIndex idxOne = indexOf( + oidOffset("0000000000000000000000000000000000000001", 500), + oidOffset("0000000000000000000000000000000000000005", 12), + oidOffset("0000000000000000000000000000000000000010", 1500)); + PackIndex idxTwo = indexOf( + oidOffset("0000000000000000000000000000000000000002", 501), + oidOffset("0000000000000000000000000000000000000003", 13), + oidOffset("0000000000000000000000000000000000000015", 1501)); + PackIndex idxThree = indexOf( + oidOffset("0000000000000000000000000000000000000004", 502), + oidOffset("0000000000000000000000000000000000000007", 14), + oidOffset("0000000000000000000000000000000000000012", 1502)); + PackIndexMerger merger = new PackIndexMerger( + Map.of("p1", idxOne, "p2", idxTwo, "p3", idxThree)); + assertEquals(9, merger.getUniqueObjectCount()); + assertEquals(3, merger.getPackCount()); + assertFalse(merger.needsLargeOffsetsChunk()); + Iterator it = merger.bySha1Iterator(); + assertNextEntry(it, "0000000000000000000000000000000000000001", 0, 500); + assertNextEntry(it, "0000000000000000000000000000000000000002", 1, 501); + assertNextEntry(it, "0000000000000000000000000000000000000003", 1, 13); + assertNextEntry(it, "0000000000000000000000000000000000000004", 2, 502); + assertNextEntry(it, "0000000000000000000000000000000000000005", 0, 12); + assertNextEntry(it, "0000000000000000000000000000000000000007", 2, 14); + assertNextEntry(it, "0000000000000000000000000000000000000010", 0, + 1500); + assertNextEntry(it, "0000000000000000000000000000000000000012", 2, + 1502); + assertNextEntry(it, "0000000000000000000000000000000000000015", 1, + 1501); + assertFalse(it.hasNext()); + } + + @Test + public void bySha1Iterator_allDuplicates() { + PackIndex idxOne = indexOf( + oidOffset("0000000000000000000000000000000000000001", 500), + oidOffset("0000000000000000000000000000000000000005", 12), + oidOffset("0000000000000000000000000000000000000010", 1500)); + PackIndexMerger merger = new PackIndexMerger( + Map.of("p1", idxOne, "p2", idxOne, "p3", idxOne)); + assertEquals(3, merger.getUniqueObjectCount()); + assertEquals(3, merger.getPackCount()); + assertFalse(merger.needsLargeOffsetsChunk()); + Iterator it = merger.bySha1Iterator(); + assertNextEntry(it, "0000000000000000000000000000000000000001", 0, 500); + assertNextEntry(it, "0000000000000000000000000000000000000005", 0, 12); + assertNextEntry(it, "0000000000000000000000000000000000000010", 0, + 1500); + assertFalse(it.hasNext()); + } + + @Test + public void bySha1Iterator_differentIndexSizes() { + PackIndex idxOne = indexOf( + oidOffset("0000000000000000000000000000000000000010", 1500)); + PackIndex idxTwo = indexOf( + oidOffset("0000000000000000000000000000000000000002", 500), + oidOffset("0000000000000000000000000000000000000003", 12)); + PackIndex idxThree = indexOf( + oidOffset("0000000000000000000000000000000000000004", 500), + oidOffset("0000000000000000000000000000000000000007", 12), + oidOffset("0000000000000000000000000000000000000012", 1500)); + PackIndexMerger merger = new PackIndexMerger( + Map.of("p1", idxOne, "p2", idxTwo, "p3", idxThree)); + assertEquals(6, merger.getUniqueObjectCount()); + assertEquals(3, merger.getPackCount()); + assertFalse(merger.needsLargeOffsetsChunk()); + Iterator it = merger.bySha1Iterator(); + assertNextEntry(it, "0000000000000000000000000000000000000002", 1, 500); + assertNextEntry(it, "0000000000000000000000000000000000000003", 1, 12); + assertNextEntry(it, "0000000000000000000000000000000000000004", 2, 500); + assertNextEntry(it, "0000000000000000000000000000000000000007", 2, 12); + assertNextEntry(it, "0000000000000000000000000000000000000010", 0, + 1500); + assertNextEntry(it, "0000000000000000000000000000000000000012", 2, + 1500); + assertFalse(it.hasNext()); + } + + @Test + public void merger_noIndexes() { + PackIndexMerger merger = new PackIndexMerger(Map.of()); + assertEquals(0, merger.getUniqueObjectCount()); + assertFalse(merger.needsLargeOffsetsChunk()); + assertTrue(merger.getPackNames().isEmpty()); + assertEquals(0, merger.getPackCount()); + assertFalse(merger.bySha1Iterator().hasNext()); + } + + @Test + public void merger_emptyIndexes() { + PackIndexMerger merger = new PackIndexMerger( + Map.of("p1", indexOf(), "p2", indexOf())); + assertEquals(0, merger.getUniqueObjectCount()); + assertFalse(merger.needsLargeOffsetsChunk()); + assertEquals(2, merger.getPackNames().size()); + assertEquals(2, merger.getPackCount()); + assertFalse(merger.bySha1Iterator().hasNext()); + } + + @Test + public void bySha1Iterator_largeOffsets_needsChunk() { + PackIndex idx1 = indexOf( + oidOffset("0000000000000000000000000000000000000002", 1L << 32), + oidOffset("0000000000000000000000000000000000000004", 12)); + PackIndex idx2 = indexOf(oidOffset( + "0000000000000000000000000000000000000003", (1L << 31) + 10)); + PackIndexMerger merger = new PackIndexMerger( + Map.of("p1", idx1, "p2", idx2)); + assertTrue(merger.needsLargeOffsetsChunk()); + assertEquals(2, merger.getOffsetsOver31BitsCount()); + assertEquals(3, merger.getUniqueObjectCount()); + } + + @Test + public void bySha1Iterator_largeOffsets_noChunk() { + // If no value is over 2^32-1, then we don't need large offset + PackIndex idx1 = indexOf( + oidOffset("0000000000000000000000000000000000000002", + (1L << 31) + 15), + oidOffset("0000000000000000000000000000000000000004", 12)); + PackIndex idx2 = indexOf(oidOffset( + "0000000000000000000000000000000000000003", (1L << 31) + 10)); + PackIndexMerger merger = new PackIndexMerger( + Map.of("p1", idx1, "p2", idx2)); + assertFalse(merger.needsLargeOffsetsChunk()); + assertEquals(2, merger.getOffsetsOver31BitsCount()); + assertEquals(3, merger.getUniqueObjectCount()); + } + + private static void assertNextEntry( + Iterator it, String oid, + int packId, long offset) { + assertTrue(it.hasNext()); + PackIndexMerger.MidxMutableEntry e = it.next(); + assertEquals(oid, e.getObjectId().name()); + assertEquals(packId, e.getPackId()); + assertEquals(offset, e.getOffset()); + } + + private static IndexObject oidOffset(String oid, long offset) { + return new IndexObject(oid, offset); + } + + private static PackIndex indexOf(IndexObject... objs) { + return FakeIndexFactory.indexOf(Arrays.asList(objs)); + } +} diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/midx/PackIndexPeekIteratorTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/midx/PackIndexPeekIteratorTest.java new file mode 100644 index 0000000000..917288a899 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/midx/PackIndexPeekIteratorTest.java @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2025, Google Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.internal.storage.midx; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +import java.util.Arrays; + +import org.eclipse.jgit.internal.storage.file.PackIndex; +import org.eclipse.jgit.junit.FakeIndexFactory; +import org.junit.Test; + +public class PackIndexPeekIteratorTest { + @Test + public void next() { + PackIndex index1 = indexOf( + object("0000000000000000000000000000000000000001", 500), + object("0000000000000000000000000000000000000003", 1500), + object("0000000000000000000000000000000000000005", 3000)); + PackIndexMerger.PackIndexPeekIterator it = new PackIndexMerger.PackIndexPeekIterator(0, index1); + assertEquals("0000000000000000000000000000000000000001", it.next().name()); + assertEquals("0000000000000000000000000000000000000003", it.next().name()); + assertEquals("0000000000000000000000000000000000000005", it.next().name()); + assertNull(it.next()); + } + + @Test + public void peek_doesNotAdvance() { + PackIndex index1 = indexOf( + object("0000000000000000000000000000000000000001", 500), + object("0000000000000000000000000000000000000003", 1500), + object("0000000000000000000000000000000000000005", 3000)); + PackIndexMerger.PackIndexPeekIterator it = new PackIndexMerger.PackIndexPeekIterator(0, index1); + it.next(); + assertEquals("0000000000000000000000000000000000000001", it.peek().name()); + assertEquals("0000000000000000000000000000000000000001", it.peek().name()); + it.next(); + assertEquals("0000000000000000000000000000000000000003", it.peek().name()); + assertEquals("0000000000000000000000000000000000000003", it.peek().name()); + it.next(); + assertEquals("0000000000000000000000000000000000000005", it.peek().name()); + assertEquals("0000000000000000000000000000000000000005", it.peek().name()); + it.next(); + assertNull(it.peek()); + assertNull(it.peek()); + } + + @Test + public void empty() { + PackIndex index1 = indexOf(); + PackIndexMerger.PackIndexPeekIterator it = new PackIndexMerger.PackIndexPeekIterator(0, index1); + assertNull(it.next()); + assertNull(it.peek()); + } + + private static PackIndex indexOf(FakeIndexFactory.IndexObject... objs) { + return FakeIndexFactory.indexOf(Arrays.asList(objs)); + } + + private static FakeIndexFactory.IndexObject object(String name, long offset) { + return new FakeIndexFactory.IndexObject(name, offset); + } +} \ No newline at end of file diff --git a/org.eclipse.jgit/META-INF/MANIFEST.MF b/org.eclipse.jgit/META-INF/MANIFEST.MF index 659ad8549f..f5cc16cfe2 100644 --- a/org.eclipse.jgit/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit/META-INF/MANIFEST.MF @@ -105,6 +105,7 @@ Export-Package: org.eclipse.jgit.annotations;version="7.2.0", org.eclipse.jgit.pgm", org.eclipse.jgit.internal.storage.memory;version="7.2.0"; x-friends:="org.eclipse.jgit.test", + org.eclipse.jgit.internal.storage.midx;version="7.2.0", org.eclipse.jgit.internal.storage.pack;version="7.2.0"; x-friends:="org.eclipse.jgit.junit, org.eclipse.jgit.test, diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/PackIndexMerger.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/PackIndexMerger.java new file mode 100644 index 0000000000..89814af107 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/PackIndexMerger.java @@ -0,0 +1,337 @@ +/* + * Copyright (C) 2025, Google Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.internal.storage.midx; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.stream.Collectors; + +import org.eclipse.jgit.internal.storage.file.PackIndex; +import org.eclipse.jgit.lib.AnyObjectId; +import org.eclipse.jgit.lib.MutableObjectId; + +/** + * Collect the stats and offers an iterator over the union of n-pack indexes. + *

+ * The multipack index is a list of (sha1, packid, offset) ordered by sha1. We + * can build it from the individual pack indexes (sha1, offset) ordered by sha1, + * with a simple merge ignoring duplicates. + *

+ * This class encapsulates the merging logic and precalculates the stats that + * the index needs (like total count of objects). To limit memory consumption, + * it does the merge as it goes during the iteration and iterators use mutable + * entries. The stats of the combined index are calculated in an iteration at + * construction time. + */ +class PackIndexMerger { + + private static final int LIMIT_31_BITS = (1 << 31) - 1; + + private static final long LIMIT_32_BITS = (1L << 32) - 1; + + /** + * Object returned by the iterator. + *

+ * The iterator returns (on each next()) the same instance with different + * values, to avoid allocating many short-lived objects. Callers should not + * keep a reference to that returned value. + */ + static class MidxMutableEntry { + // The object id + private final MutableObjectId oid = new MutableObjectId(); + + // Position of the pack in the ordered list of pack in this merger + private int packId; + + // Offset in its pack + private long offset; + + public AnyObjectId getObjectId() { + return oid; + } + + public int getPackId() { + return packId; + } + + public long getOffset() { + return offset; + } + + /** + * Copy values from another mutable entry + * + * @param packId + * packId + * @param other + * another mutable entry + */ + private void fill(int packId, PackIndex.MutableEntry other) { + other.copyOidTo(oid); + this.packId = packId; + this.offset = other.getOffset(); + } + } + + private final List packNames; + + private final List indexes; + + private final boolean needsLargeOffsetsChunk; + + private final int offsetsOver31BitsCount; + + private final int uniqueObjectCount; + + PackIndexMerger(Map packs) { + this.packNames = packs.keySet().stream().sorted() + .collect(Collectors.toUnmodifiableList()); + + this.indexes = packNames.stream().map(packs::get) + .collect(Collectors.toUnmodifiableList()); + + // Iterate for duplicates + int objectCount = 0; + boolean hasLargeOffsets = false; + int over31bits = 0; + MutableObjectId lastSeen = new MutableObjectId(); + MultiIndexIterator it = new MultiIndexIterator(indexes); + while (it.hasNext()) { + MidxMutableEntry entry = it.next(); + if (lastSeen.equals(entry.oid)) { + continue; + } + // If there is at least one offset value larger than 2^32-1, then + // the large offset chunk must exist, and offsets larger than + // 2^31-1 must be stored in it instead + if (entry.offset > LIMIT_32_BITS) { + hasLargeOffsets = true; + } + if (entry.offset > LIMIT_31_BITS) { + over31bits++; + } + + lastSeen.fromObjectId(entry.oid); + objectCount++; + } + uniqueObjectCount = objectCount; + offsetsOver31BitsCount = over31bits; + needsLargeOffsetsChunk = hasLargeOffsets; + } + + /** + * Object count of the merged index (i.e. without duplicates) + * + * @return object count of the merged index + */ + int getUniqueObjectCount() { + return uniqueObjectCount; + } + + /** + * If any object in any of the indexes has an offset over 2^32-1 + * + * @return true if there is any object with offset > 2^32 -1 + */ + boolean needsLargeOffsetsChunk() { + return needsLargeOffsetsChunk; + } + + /** + * How many object have offsets over 2^31-1 + *

+ * Per multipack index spec, IF there is large offset chunk, all this + * offsets should be there. + * + * @return number of objects with offsets over 2^31-1 + */ + int getOffsetsOver31BitsCount() { + return offsetsOver31BitsCount; + } + + /** + * List of pack names in alphabetical order. + *

+ * Order matters: In case of duplicates, the multipack index prefers the + * first package with it. This is in the same order we are using to + * prioritize duplicates. + * + * @return List of pack names, in the order used by the merge. + */ + List getPackNames() { + return packNames; + } + + /** + * How many packs are being merged + * + * @return count of packs merged + */ + int getPackCount() { + return packNames.size(); + } + + /** + * Iterator over the merged indexes in sha1 order without duplicates + *

+ * The returned entry in the iterator is mutable, callers should NOT keep a + * reference to it. + * + * @return an iterator in sha1 order without duplicates. + */ + Iterator bySha1Iterator() { + return new DedupMultiIndexIterator(new MultiIndexIterator(indexes), + getUniqueObjectCount()); + } + + /** + * For testing. Iterate all entries, not skipping duplicates (stable order) + * + * @return an iterator of all objects in sha1 order, including duplicates. + */ + Iterator rawIterator() { + return new MultiIndexIterator(indexes); + } + + /** + * Iterator over n-indexes in ObjectId order. + *

+ * It returns duplicates if the same object id is in different indexes. Wrap + * it with {@link DedupMultiIndexIterator (Iterator, int)} to avoid + * duplicates. + */ + private static final class MultiIndexIterator + implements Iterator { + + private final List indexIterators; + + private final MidxMutableEntry mutableEntry = new MidxMutableEntry(); + + MultiIndexIterator(List indexes) { + this.indexIterators = new ArrayList<>(indexes.size()); + for (int i = 0; i < indexes.size(); i++) { + PackIndexPeekIterator it = new PackIndexPeekIterator(i, + indexes.get(i)); + // Position in the first element + if (it.next() != null) { + indexIterators.add(it); + } + } + } + + @Override + public boolean hasNext() { + return !indexIterators.isEmpty(); + } + + @Override + public MidxMutableEntry next() { + PackIndexPeekIterator winner = null; + for (int index = 0; index < indexIterators.size(); index++) { + PackIndexPeekIterator current = indexIterators.get(index); + if (winner == null + || current.peek().compareBySha1To(winner.peek()) < 0) { + winner = current; + } + } + + if (winner == null) { + throw new NoSuchElementException(); + } + + mutableEntry.fill(winner.getPackId(), winner.peek()); + if (winner.next() == null) { + indexIterators.remove(winner); + }; + return mutableEntry; + } + } + + private static class DedupMultiIndexIterator + implements Iterator { + private final MultiIndexIterator src; + + private int remaining; + + private final MutableObjectId lastOid = new MutableObjectId(); + + DedupMultiIndexIterator(MultiIndexIterator src, int totalCount) { + this.src = src; + this.remaining = totalCount; + } + + @Override + public boolean hasNext() { + return remaining > 0; + } + + @Override + public MidxMutableEntry next() { + MidxMutableEntry next = src.next(); + while (next != null && lastOid.equals(next.oid)) { + next = src.next(); + } + + if (next == null) { + throw new NoSuchElementException(); + } + + lastOid.fromObjectId(next.oid); + remaining--; + return next; + } + } + + /** + * Convenience around the PackIndex iterator to read the current value + * multiple times without consuming it. + *

+ * This is used to merge indexes in the multipack index, where we need to + * compare the current value between indexes multiple times to find the + * next. + *

+ * We could also implement this keeping the position (int) and + * MutableEntry#getObjectId, but that would create an ObjectId per entry. + * This implementation reuses the MutableEntry and avoid instantiations. + */ + // Visible for testing + static class PackIndexPeekIterator { + private final Iterator it; + + private final int packId; + + PackIndex.MutableEntry current; + + PackIndexPeekIterator(int packId, PackIndex index) { + it = index.iterator(); + this.packId = packId; + } + + PackIndex.MutableEntry next() { + if (it.hasNext()) { + current = it.next(); + } else { + current = null; + } + return current; + } + + PackIndex.MutableEntry peek() { + return current; + } + + int getPackId() { + return packId; + } + } +} -- cgit v1.2.3 From deae89f9e7ddda2b44492c47e9d6d027ed4e2cda Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Wed, 5 Feb 2025 23:55:14 +0100 Subject: CommitConfig: fix potential NPE The local variable `comment` is null if the option core.commentChar is unset. Change-Id: I907dabff395f75b3a6367446389df395b28f027a --- org.eclipse.jgit/src/org/eclipse/jgit/lib/CommitConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/CommitConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/CommitConfig.java index f701a41d67..b1ba5dfa28 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/CommitConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/CommitConfig.java @@ -119,7 +119,7 @@ public class CommitConfig { if (!StringUtils.isEmptyOrNull(comment)) { if ("auto".equalsIgnoreCase(comment)) { //$NON-NLS-1$ autoCommentChar = true; - } else { + } else if (comment != null) { char first = comment.charAt(0); if (first > ' ' && first < 127) { commentCharacter = first; -- cgit v1.2.3 From a86e2a7d74fbbbdaa57f21e48ea16019771bcbe9 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Tue, 4 Feb 2025 11:10:39 +0100 Subject: Config: add getters for primitive types without default value which return null if the option is not configured. Change-Id: I0d444b396458f49712e35ef32427dc45ee3f8ec8 --- .../tst/org/eclipse/jgit/lib/ConfigTest.java | 41 ++++ org.eclipse.jgit/.settings/.api_filters | 46 ++++ .../src/org/eclipse/jgit/lib/Config.java | 265 ++++++++++++++++++--- .../eclipse/jgit/lib/DefaultTypedConfigGetter.java | 79 ++++-- .../org/eclipse/jgit/lib/TypedConfigGetter.java | 140 ++++++++++- 5 files changed, 527 insertions(+), 44 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/ConfigTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/ConfigTest.java index 31940a16f7..06fee8ea71 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/ConfigTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/ConfigTest.java @@ -1636,6 +1636,47 @@ public class ConfigTest { assertFalse(config.get(CoreConfig.KEY).enableCommitGraph()); } + @Test + public void testGetNoDefaultBoolean() { + Config config = new Config(); + assertNull(config.getBoolean("foo", "bar")); + assertNull(config.getBoolean("foo", "bar", "baz")); + } + + @Test + public void testGetNoDefaultEnum() { + Config config = new Config(); + assertNull(config.getEnum(new TestEnum[] { TestEnum.ONE_TWO }, "foo", + "bar", "baz")); + } + + @Test + public void testGetNoDefaultInt() { + Config config = new Config(); + assertNull(config.getInt("foo", "bar")); + assertNull(config.getInt("foo", "bar", "baz")); + } + @Test + public void testGetNoDefaultIntInRange() { + Config config = new Config(); + assertNull(config.getIntInRange("foo", "bar", 1, 5)); + assertNull(config.getIntInRange("foo", "bar", "baz", 1, 5)); + } + + @Test + public void testGetNoDefaultLong() { + Config config = new Config(); + assertNull(config.getLong("foo", "bar")); + assertNull(config.getLong("foo", "bar", "baz")); + } + + @Test + public void testGetNoDefaultTimeUnit() { + Config config = new Config(); + assertNull(config.getTimeUnit("foo", "bar", "baz", + TimeUnit.SECONDS)); + } + private static void assertValueRoundTrip(String value) throws ConfigInvalidException { assertValueRoundTrip(value, value); diff --git a/org.eclipse.jgit/.settings/.api_filters b/org.eclipse.jgit/.settings/.api_filters index 61d66582e0..877a488a73 100644 --- a/org.eclipse.jgit/.settings/.api_filters +++ b/org.eclipse.jgit/.settings/.api_filters @@ -14,4 +14,50 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Config.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Config.java index 07c5fa4500..345cb22f80 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Config.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Config.java @@ -34,6 +34,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import org.eclipse.jgit.annotations.NonNull; +import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.errors.ConfigInvalidException; import org.eclipse.jgit.events.ConfigChangedEvent; import org.eclipse.jgit.events.ConfigChangedListener; @@ -254,11 +255,27 @@ public class Config { * default value to return if no value was present. * @return an integer value from the configuration, or defaultValue. */ - public int getInt(final String section, final String name, - final int defaultValue) { - return typedGetter.getInt(this, section, null, name, defaultValue); + public int getInt(String section, String name, int defaultValue) { + return getInt(section, null, name, defaultValue); } + /** + * Obtain an integer value from the configuration. + * + * @param section + * section the key is grouped within. + * @param name + * name of the key to get. + * @return an integer value from the configuration, or {@code null} if not + * set. + * @since 7.2 + */ + @Nullable + public Integer getInt(String section, String name) { + return getInt(section, null, name); + } + + /** * Obtain an integer value from the configuration. * @@ -272,10 +289,30 @@ public class Config { * default value to return if no value was present. * @return an integer value from the configuration, or defaultValue. */ - public int getInt(final String section, String subsection, - final String name, final int defaultValue) { + public int getInt(String section, String subsection, String name, + int defaultValue) { + Integer v = typedGetter.getInt(this, section, subsection, name, + Integer.valueOf(defaultValue)); + return v == null ? defaultValue : v.intValue(); + } + + /** + * Obtain an integer value from the configuration. + * + * @param section + * section the key is grouped within. + * @param subsection + * subsection name, such a remote or branch name. + * @param name + * name of the key to get. + * @return an integer value from the configuration, or {@code null} if not + * set. + * @since 7.2 + */ + @Nullable + public Integer getInt(String section, String subsection, String name) { return typedGetter.getInt(this, section, subsection, name, - defaultValue); + null); } /** @@ -297,8 +334,30 @@ public class Config { */ public int getIntInRange(String section, String name, int minValue, int maxValue, int defaultValue) { - return typedGetter.getIntInRange(this, section, null, name, minValue, - maxValue, defaultValue); + return getIntInRange(section, null, name, + minValue, maxValue, defaultValue); + } + + /** + * Obtain an integer value from the configuration which must be inside given + * range. + * + * @param section + * section the key is grouped within. + * @param name + * name of the key to get. + * @param minValue + * minimum value + * @param maxValue + * maximum value + * @return an integer value from the configuration, or {@code null} if not + * set. + * @since 7.2 + */ + @Nullable + public Integer getIntInRange(String section, String name, int minValue, + int maxValue) { + return getIntInRange(section, null, name, minValue, maxValue); } /** @@ -322,8 +381,34 @@ public class Config { */ public int getIntInRange(String section, String subsection, String name, int minValue, int maxValue, int defaultValue) { + Integer v = typedGetter.getIntInRange(this, section, subsection, name, + minValue, maxValue, Integer.valueOf(defaultValue)); + return v == null ? defaultValue : v.intValue(); + } + + /** + * Obtain an integer value from the configuration which must be inside given + * range. + * + * @param section + * section the key is grouped within. + * @param subsection + * subsection name, such a remote or branch name. + * @param name + * name of the key to get. + * @param minValue + * minimum value + * @param maxValue + * maximum value + * @return an integer value from the configuration, or {@code null} if not + * set. + * @since 7.2 + */ + @Nullable + public Integer getIntInRange(String section, String subsection, String name, + int minValue, int maxValue) { return typedGetter.getIntInRange(this, section, subsection, name, - minValue, maxValue, defaultValue); + minValue, maxValue, null); } /** @@ -338,7 +423,23 @@ public class Config { * @return an integer value from the configuration, or defaultValue. */ public long getLong(String section, String name, long defaultValue) { - return typedGetter.getLong(this, section, null, name, defaultValue); + return getLong(section, null, name, defaultValue); + } + + /** + * Obtain an integer value from the configuration. + * + * @param section + * section the key is grouped within. + * @param name + * name of the key to get. + * @return an integer value from the configuration, or {@code null} if not + * set. + * @since 7.2 + */ + @Nullable + public Long getLong(String section, String name) { + return getLong(section, null, name); } /** @@ -355,9 +456,28 @@ public class Config { * @return an integer value from the configuration, or defaultValue. */ public long getLong(final String section, String subsection, - final String name, final long defaultValue) { - return typedGetter.getLong(this, section, subsection, name, - defaultValue); + String name, long defaultValue) { + Long v = typedGetter.getLong(this, section, subsection, name, + Long.valueOf(defaultValue)); + return v == null ? defaultValue : v.longValue(); + } + + /** + * Obtain an integer value from the configuration. + * + * @param section + * section the key is grouped within. + * @param subsection + * subsection name, such a remote or branch name. + * @param name + * name of the key to get. + * @return an integer value from the configuration, or {@code null} if not + * set. + * @since 7.2 + */ + @Nullable + public Long getLong(String section, String subsection, String name) { + return typedGetter.getLong(this, section, subsection, name, null); } /** @@ -372,9 +492,26 @@ public class Config { * @return true if any value or defaultValue is true, false for missing or * explicit false */ - public boolean getBoolean(final String section, final String name, - final boolean defaultValue) { - return typedGetter.getBoolean(this, section, null, name, defaultValue); + public boolean getBoolean(String section, String name, + boolean defaultValue) { + Boolean v = typedGetter.getBoolean(this, section, null, name, + Boolean.valueOf(defaultValue)); + return v == null ? defaultValue : v.booleanValue(); + } + + /** + * Get a boolean value from the git config + * + * @param section + * section the key is grouped within. + * @param name + * name of the key to get. + * @return configured boolean value, or {@code null} if not set. + * @since 7.2 + */ + @Nullable + public Boolean getBoolean(String section, String name) { + return getBoolean(section, null, name); } /** @@ -391,10 +528,28 @@ public class Config { * @return true if any value or defaultValue is true, false for missing or * explicit false */ - public boolean getBoolean(final String section, String subsection, - final String name, final boolean defaultValue) { - return typedGetter.getBoolean(this, section, subsection, name, - defaultValue); + public boolean getBoolean(String section, String subsection, String name, + boolean defaultValue) { + Boolean v = typedGetter.getBoolean(this, section, subsection, name, + Boolean.valueOf(defaultValue)); + return v == null ? defaultValue : v.booleanValue(); + } + + /** + * Get a boolean value from the git config + * + * @param section + * section the key is grouped within. + * @param subsection + * subsection name, such a remote or branch name. + * @param name + * name of the key to get. + * @return configured boolean value, or {@code null} if not set. + * @since 7.2 + */ + @Nullable + public Boolean getBoolean(String section, String subsection, String name) { + return typedGetter.getBoolean(this, section, subsection, name, null); } /** @@ -412,8 +567,8 @@ public class Config { * default value to return if no value was present. * @return the selected enumeration value, or {@code defaultValue}. */ - public > T getEnum(final String section, - final String subsection, final String name, final T defaultValue) { + public > T getEnum(String section, String subsection, + String name, @NonNull T defaultValue) { final T[] all = allValuesOf(defaultValue); return typedGetter.getEnum(this, all, section, subsection, name, defaultValue); @@ -448,13 +603,40 @@ public class Config { * @param defaultValue * default value to return if no value was present. * @return the selected enumeration value, or {@code defaultValue}. + * @deprecated use {@link #getEnum(String, String, String, Enum)} or + * {{@link #getEnum(Enum[], String, String, String)}} instead. */ - public > T getEnum(final T[] all, final String section, - final String subsection, final String name, final T defaultValue) { + @Nullable + @Deprecated + public > T getEnum(T[] all, String section, + String subsection, String name, @Nullable T defaultValue) { return typedGetter.getEnum(this, all, section, subsection, name, defaultValue); } + /** + * Parse an enumeration from the configuration. + * + * @param + * type of the returned enum + * @param all + * all possible values in the enumeration which should be + * recognized. Typically {@code EnumType.values()}. + * @param section + * section the key is grouped within. + * @param subsection + * subsection name, such a remote or branch name. + * @param name + * name of the key to get. + * @return the selected enumeration value, or {@code null} if not set. + * @since 7.2 + */ + @Nullable + public > T getEnum(T[] all, String section, + String subsection, String name) { + return typedGetter.getEnum(this, all, section, subsection, name, null); + } + /** * Get string value or null if not found. * @@ -466,8 +648,8 @@ public class Config { * the key name * @return a String value from the config, null if not found */ - public String getString(final String section, String subsection, - final String name) { + @Nullable + public String getString(String section, String subsection, String name) { return getRawString(section, subsection, name); } @@ -526,8 +708,34 @@ public class Config { */ public long getTimeUnit(String section, String subsection, String name, long defaultValue, TimeUnit wantUnit) { + Long v = typedGetter.getTimeUnit(this, section, subsection, name, + Long.valueOf(defaultValue), wantUnit); + return v == null ? defaultValue : v.longValue(); + + } + + /** + * Parse a numerical time unit, such as "1 minute", from the configuration. + * + * @param section + * section the key is in. + * @param subsection + * subsection the key is in, or null if not in a subsection. + * @param name + * the key name. + * @param wantUnit + * the units of {@code defaultValue} and the return value, as + * well as the units to assume if the value does not contain an + * indication of the units. + * @return the value, or {@code null} if not set, expressed in + * {@code units}. + * @since 7.2 + */ + @Nullable + public Long getTimeUnit(String section, String subsection, String name, + TimeUnit wantUnit) { return typedGetter.getTimeUnit(this, section, subsection, name, - defaultValue, wantUnit); + null, wantUnit); } /** @@ -555,8 +763,9 @@ public class Config { * @return the {@link Path}, or {@code defaultValue} if not set * @since 5.10 */ + @Nullable public Path getPath(String section, String subsection, String name, - @NonNull FS fs, File resolveAgainst, Path defaultValue) { + @NonNull FS fs, File resolveAgainst, @Nullable Path defaultValue) { return typedGetter.getPath(this, section, subsection, name, fs, resolveAgainst, defaultValue); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/DefaultTypedConfigGetter.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/DefaultTypedConfigGetter.java index a71549c92e..65093987e8 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/DefaultTypedConfigGetter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/DefaultTypedConfigGetter.java @@ -18,6 +18,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import org.eclipse.jgit.annotations.NonNull; +import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.lib.Config.ConfigEnum; import org.eclipse.jgit.transport.RefSpec; @@ -34,24 +35,32 @@ public class DefaultTypedConfigGetter implements TypedConfigGetter { @Override public boolean getBoolean(Config config, String section, String subsection, String name, boolean defaultValue) { + return getBoolean(config, section, subsection, name, defaultValue); + } + + @Nullable + @Override + public Boolean getBoolean(Config config, String section, String subsection, + String name, @Nullable Boolean defaultValue) { String n = config.getString(section, subsection, name); if (n == null) { return defaultValue; } if (Config.isMissing(n)) { - return true; + return Boolean.TRUE; } try { - return StringUtils.toBoolean(n); + return Boolean.valueOf(StringUtils.toBoolean(n)); } catch (IllegalArgumentException err) { throw new IllegalArgumentException(MessageFormat.format( JGitText.get().invalidBooleanValue, section, name, n), err); } } + @Nullable @Override public > T getEnum(Config config, T[] all, String section, - String subsection, String name, T defaultValue) { + String subsection, String name, @Nullable T defaultValue) { String value = config.getString(section, subsection, name); if (value == null) { return defaultValue; @@ -107,9 +116,26 @@ public class DefaultTypedConfigGetter implements TypedConfigGetter { @Override public int getInt(Config config, String section, String subsection, String name, int defaultValue) { - long val = config.getLong(section, subsection, name, defaultValue); + return getInt(config, section, subsection, name, defaultValue); + } + + @Nullable + @Override + @SuppressWarnings("boxing") + public Integer getInt(Config config, String section, String subsection, + String name, @Nullable Integer defaultValue) { + Long longDefault = defaultValue != null + ? Long.valueOf(defaultValue.longValue()) + : null; + Long val = config.getLong(section, subsection, name); + if (val == null) { + val = longDefault; + } + if (val == null) { + return null; + } if (Integer.MIN_VALUE <= val && val <= Integer.MAX_VALUE) { - return (int) val; + return Integer.valueOf(Math.toIntExact(val)); } throw new IllegalArgumentException(MessageFormat .format(JGitText.get().integerValueOutOfRange, section, name)); @@ -118,31 +144,48 @@ public class DefaultTypedConfigGetter implements TypedConfigGetter { @Override public int getIntInRange(Config config, String section, String subsection, String name, int minValue, int maxValue, int defaultValue) { - int val = getInt(config, section, subsection, name, defaultValue); + return getIntInRange(config, section, subsection, name, minValue, + maxValue, defaultValue); + } + + @Override + @SuppressWarnings("boxing") + public Integer getIntInRange(Config config, String section, + String subsection, String name, int minValue, int maxValue, + Integer defaultValue) { + Integer val = getInt(config, section, subsection, name, defaultValue); + if (val == null) { + return null; + } if ((val >= minValue && val <= maxValue) || val == UNSET_INT) { return val; } if (subsection == null) { throw new IllegalArgumentException(MessageFormat.format( JGitText.get().integerValueNotInRange, section, name, - Integer.valueOf(val), Integer.valueOf(minValue), - Integer.valueOf(maxValue))); + val, minValue, maxValue)); } throw new IllegalArgumentException(MessageFormat.format( JGitText.get().integerValueNotInRangeSubSection, section, - subsection, name, Integer.valueOf(val), - Integer.valueOf(minValue), Integer.valueOf(maxValue))); + subsection, name, val, minValue, maxValue)); } @Override public long getLong(Config config, String section, String subsection, String name, long defaultValue) { - final String str = config.getString(section, subsection, name); + return getLong(config, section, subsection, name, defaultValue); + } + + @Nullable + @Override + public Long getLong(Config config, String section, String subsection, + String name, @Nullable Long defaultValue) { + String str = config.getString(section, subsection, name); if (str == null) { return defaultValue; } try { - return StringUtils.parseLongWithSuffix(str, false); + return Long.valueOf(StringUtils.parseLongWithSuffix(str, false)); } catch (StringIndexOutOfBoundsException e) { // Empty return defaultValue; @@ -156,6 +199,14 @@ public class DefaultTypedConfigGetter implements TypedConfigGetter { @Override public long getTimeUnit(Config config, String section, String subsection, String name, long defaultValue, TimeUnit wantUnit) { + Long v = getTimeUnit(config, section, subsection, name, + Long.valueOf(defaultValue), wantUnit); + return v == null ? defaultValue : v.longValue(); + } + + @Override + public Long getTimeUnit(Config config, String section, String subsection, + String name, @Nullable Long defaultValue, TimeUnit wantUnit) { String valueString = config.getString(section, subsection, name); if (valueString == null) { @@ -232,8 +283,8 @@ public class DefaultTypedConfigGetter implements TypedConfigGetter { } try { - return wantUnit.convert(Long.parseLong(digits) * inputMul, - inputUnit); + return Long.valueOf(wantUnit + .convert(Long.parseLong(digits) * inputMul, inputUnit)); } catch (NumberFormatException nfe) { IllegalArgumentException iae = notTimeUnit(section, subsection, unitName, valueString); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/TypedConfigGetter.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/TypedConfigGetter.java index 0c03adcab8..3d4e0d1f3c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/TypedConfigGetter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/TypedConfigGetter.java @@ -17,6 +17,7 @@ import java.util.List; import java.util.concurrent.TimeUnit; import org.eclipse.jgit.annotations.NonNull; +import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.transport.RefSpec; import org.eclipse.jgit.util.FS; @@ -50,10 +51,35 @@ public interface TypedConfigGetter { * default value to return if no value was present. * @return true if any value or defaultValue is true, false for missing or * explicit false + * @deprecated use + * {@link #getBoolean(Config, String, String, String, Boolean)} + * instead */ + @Deprecated boolean getBoolean(Config config, String section, String subsection, String name, boolean defaultValue); + /** + * Get a boolean value from a git {@link Config}. + * + * @param config + * to get the value from + * @param section + * section the key is grouped within. + * @param subsection + * subsection name, such a remote or branch name. + * @param name + * name of the key to get. + * @param defaultValue + * default value to return if no value was present. + * @return true if any value or defaultValue is true, false for missing or + * explicit false + * @since 7.2 + */ + @Nullable + Boolean getBoolean(Config config, String section, String subsection, + String name, @Nullable Boolean defaultValue); + /** * Parse an enumeration from a git {@link Config}. * @@ -74,8 +100,9 @@ public interface TypedConfigGetter { * default value to return if no value was present. * @return the selected enumeration value, or {@code defaultValue}. */ + @Nullable > T getEnum(Config config, T[] all, String section, - String subsection, String name, T defaultValue); + String subsection, String name, @Nullable T defaultValue); /** * Obtain an integer value from a git {@link Config}. @@ -91,10 +118,33 @@ public interface TypedConfigGetter { * @param defaultValue * default value to return if no value was present. * @return an integer value from the configuration, or defaultValue. + * @deprecated use {@link #getInt(Config, String, String, String, Integer)} + * instead */ + @Deprecated int getInt(Config config, String section, String subsection, String name, int defaultValue); + /** + * Obtain an integer value from a git {@link Config}. + * + * @param config + * to get the value from + * @param section + * section the key is grouped within. + * @param subsection + * subsection name, such a remote or branch name. + * @param name + * name of the key to get. + * @param defaultValue + * default value to return if no value was present. + * @return an integer value from the configuration, or defaultValue. + * @since 7.2 + */ + @Nullable + Integer getInt(Config config, String section, String subsection, + String name, @Nullable Integer defaultValue); + /** * Obtain an integer value from a git {@link Config} which must be in given * range. @@ -117,10 +167,42 @@ public interface TypedConfigGetter { * @return an integer value from the configuration, or defaultValue. * {@code #UNSET_INT} if unset. * @since 6.1 + * @deprecated use + * {@link #getIntInRange(Config, String, String, String, int, int, Integer)} + * instead */ + @Deprecated int getIntInRange(Config config, String section, String subsection, String name, int minValue, int maxValue, int defaultValue); + /** + * Obtain an integer value from a git {@link Config} which must be in given + * range. + * + * @param config + * to get the value from + * @param section + * section the key is grouped within. + * @param subsection + * subsection name, such a remote or branch name. + * @param name + * name of the key to get. + * @param minValue + * minimal value + * @param maxValue + * maximum value + * @param defaultValue + * default value to return if no value was present. Use + * {@code #UNSET_INT} to set the default to unset. + * @return an integer value from the configuration, or defaultValue. + * {@code #UNSET_INT} if unset. + * @since 7.2 + */ + @Nullable + Integer getIntInRange(Config config, String section, String subsection, + String name, int minValue, int maxValue, + @Nullable Integer defaultValue); + /** * Obtain a long value from a git {@link Config}. * @@ -135,10 +217,33 @@ public interface TypedConfigGetter { * @param defaultValue * default value to return if no value was present. * @return a long value from the configuration, or defaultValue. + * @deprecated use {@link #getLong(Config, String, String, String, Long)} + * instead */ + @Deprecated long getLong(Config config, String section, String subsection, String name, long defaultValue); + /** + * Obtain a long value from a git {@link Config}. + * + * @param config + * to get the value from + * @param section + * section the key is grouped within. + * @param subsection + * subsection name, such a remote or branch name. + * @param name + * name of the key to get. + * @param defaultValue + * default value to return if no value was present. + * @return a long value from the configuration, or defaultValue. + * @since 7.2 + */ + @Nullable + Long getLong(Config config, String section, String subsection, String name, + @Nullable Long defaultValue); + /** * Parse a numerical time unit, such as "1 minute", from a git * {@link Config}. @@ -159,10 +264,40 @@ public interface TypedConfigGetter { * indication of the units. * @return the value, or {@code defaultValue} if not set, expressed in * {@code units}. + * @deprecated use + * {@link #getTimeUnit(Config, String, String, String, Long, TimeUnit)} + * instead */ + @Deprecated long getTimeUnit(Config config, String section, String subsection, String name, long defaultValue, TimeUnit wantUnit); + /** + * Parse a numerical time unit, such as "1 minute", from a git + * {@link Config}. + * + * @param config + * to get the value from + * @param section + * section the key is in. + * @param subsection + * subsection the key is in, or null if not in a subsection. + * @param name + * the key name. + * @param defaultValue + * default value to return if no value was present. + * @param wantUnit + * the units of {@code defaultValue} and the return value, as + * well as the units to assume if the value does not contain an + * indication of the units. + * @return the value, or {@code defaultValue} if not set, expressed in + * {@code units}. + * @since 7.2 + */ + @Nullable + Long getTimeUnit(Config config, String section, String subsection, + String name, @Nullable Long defaultValue, TimeUnit wantUnit); + /** * Parse a string value from a git {@link Config} and treat it as a file * path, replacing a ~/ prefix by the user's home directory. @@ -189,9 +324,10 @@ public interface TypedConfigGetter { * @return the {@link Path}, or {@code defaultValue} if not set * @since 5.10 */ + @Nullable default Path getPath(Config config, String section, String subsection, String name, @NonNull FS fs, File resolveAgainst, - Path defaultValue) { + @Nullable Path defaultValue) { String value = config.getString(section, subsection, name); if (value == null) { return defaultValue; -- cgit v1.2.3 From ff3a150495ad466169a7fb0cfaed06aace50da7b Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Wed, 15 Jan 2025 13:15:28 +0100 Subject: Improve configuration of trusting file attributes in FileSnapshot FileSnapshot relies on File attributes (which can be retrieved by the stat() function on Unix) to quickly determine if a File was modified without reading file content or listing content of a directory. On NFS this doesn't work reliably due to NFS client caching behavior. Hence we introduced the option core.trustFolderStat to control if FileSnapshot can trust File attributes to ensure we don't miss modifications on NFS. Later more specific options for handling packed and loose refs were added which also support another config value AFTER_OPEN, in addition to ALWAYS and NEVER, which refreshes File attributes by opening a FileInputStream on the file instead of reading its content and then trusts the File attributes of the refreshed File. We discussed in jgit-127 how to extend these options for other scenarios where file attributes are used to detect modifications and came to the conclusion to improve the existing trustXXX config options in the following way: - replace the not well defined "trustFolderStat" option by a general option "trustStat" which allows to configure all these scenarios with a single option - introduce a new enum TrustStat and use it for all scenarios. It has the values - NEVER don't trust File attributes - ALWAYS always trust File attributes - AFTER_OPEN open a FileInputStream on the respective file or folder to ensure its File attributes are refreshed and then trust the refreshed File attributes - INHERIT only used for specific options to signal it should inherit its value from the "trustStat" option - deprecate the old, now unused enums "TrustPackedRefsStat" and "TrustLooseRefStat" - deprecate "trustFolderStat", if set, translate it to the corresponding value of the new option "trustStat" - if both "trustFolderStat" and "trustStat" are configured the value configured for "trustStat" takes precedence and "trustFolderStat" is ignored - add one specific option for each scenario which can override the global setting - add new options "trustLooseObjectStat" and "trustPackStat" which allow to override the global setting for handling of loose objects and pack files - implement option AFTER_OPEN for "trustLooseObjectStat" and "trustPackStat" Bug: jgit-127 Change-Id: I662982258bc4494f146805875e52838394673c8f --- Documentation/config-options.md | 9 +- .../internal/storage/file/ObjectDirectoryTest.java | 17 +- .../org/eclipse/jgit/internal/JGitText.properties | 3 + .../src/org/eclipse/jgit/internal/JGitText.java | 3 + .../jgit/internal/storage/file/LooseObjects.java | 46 ++++-- .../jgit/internal/storage/file/PackDirectory.java | 38 +++-- .../jgit/internal/storage/file/RefDirectory.java | 39 ++--- .../src/org/eclipse/jgit/lib/ConfigConstants.java | 23 +++ .../src/org/eclipse/jgit/lib/CoreConfig.java | 172 ++++++++++++++++++++- 9 files changed, 278 insertions(+), 72 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/Documentation/config-options.md b/Documentation/config-options.md index eeb78ff550..807d4a8002 100644 --- a/Documentation/config-options.md +++ b/Documentation/config-options.md @@ -55,9 +55,12 @@ For details on native git options see also the official [git config documentatio | `core.streamFileThreshold` | `50 MiB` | ⃞ | The size threshold beyond which objects must be streamed. | | `core.supportsAtomicFileCreation` | `true` | ⃞ | Whether the filesystem supports atomic file creation. | | `core.symlinks` | Auto detect if filesystem supports symlinks| ✅ | If false, symbolic links are checked out as small plain files that contain the link text. | -| `core.trustFolderStat` | `true` | ⃞ | Whether to trust the pack folder's, packed-refs file's and loose-objects folder's file attributes (Java equivalent of stat command on *nix). When looking for pack files, if `false` JGit will always scan the `.git/objects/pack` folder and if set to `true` it assumes that pack files are unchanged if the file attributes of the pack folder are unchanged. When getting the list of packed refs, if `false` JGit will always read the packed-refs file and if set to `true` it uses the file attributes of the packed-refs file and will only read it if a file attribute has changed. When looking for loose objects, if `false` and if a loose object is not found, JGit will open and close a stream to `.git/objects` folder (which can refresh its directory listing, at least on some NFS clients) and retry looking for that loose object. Setting this option to `false` can help to workaround caching issues on NFS, but reduces performance. | -| `core.trustPackedRefsStat` | `unset` | ⃞ | Whether to trust the file attributes (Java equivalent of stat command on *nix) of the packed-refs file. If `never` JGit will ignore the file attributes of the packed-refs file and always read it. If `always` JGit will trust the file attributes of the packed-refs file and will only read it if a file attribute has changed. `after_open` behaves the same as `always`, except that the packed-refs file is opened and closed before its file attributes are considered. An open/close of the packed-refs file is known to refresh its file attributes, at least on some NFS clients. If `unset`, JGit will use the behavior described in `trustFolderStat`. | -| `core.trustLooseRefStat` | `always` | ⃞ | Whether to trust the file attributes (Java equivalent of stat command on *nix) of the loose ref. If `always` JGit will trust the file attributes of the loose ref and its parent directories. `after_open` behaves similar to `always`, except that all parent directories of the loose ref up to the repository root are opened and closed before its file attributes are considered. An open/close of these parent directories is known to refresh the file attributes, at least on some NFS clients. | +| ~~`core.trustFolderStat`~~ | `true` | ⃞ | __Deprecated__, use `core.trustStat` instead. If set to `true` translated to `core.trustStat=always`, if `false` translated to `core.trustStat=never`, see below. If both `core.trustFolderStat` and `core.trustStat` are configured then `trustStat` takes precedence and `trustFolderStat` is ignored. | +| `core.trustLooseRefStat` | `inherit` | ⃞ | Whether to trust the file attributes of loose refs and its fan-out parent directory. See `core.trustStat` for possible values. If `inherit`, JGit will use the behavior configured in `trustStat`. | +| `core.trustPackedRefsStat` | `inherit` | ⃞ | Whether to trust the file attributes of the packed-refs file. See `core.trustStat` for possible values. If `inherit`, JGit will use the behavior configured in `core.trustStat`. | +| `core.trustLooseObjectStat` | `inherit` | ⃞ | Whether to trust the file attributes of the loose object file and its fan-out parent directory. See `core.trustStat` for possible values. If `inherit`, JGit will use the behavior configured in `core.trustStat`. | +| `core.trustPackStat` | `inherit` | ⃞ | Whether to trust the file attributes of the `objects/pack` directory. See `core.trustStat` for possible values. If `inherit`, JGit will use the behavior configured in `core.trustStat`. | +| `core.trustStat` | `always` | ⃞ | Global option to configure whether to trust file attributes (Java equivalent of stat command on Unix) of files storing git objects. Can be overridden for specific files by configuring `core.trustLooseRefStat, core.trustPackedRefsStat, core.trustLooseObjectStat, core.trustPackStat`. If `never` JGit will ignore the file attributes of the file and always read it. If `always` JGit will trust the file attributes and will only read it if a file attribute has changed. `after_open` behaves the same as `always`, but file attributes are only considered *after* the file itself and any transient parent directories have been opened and closed. An open/close of the file/directory is known to refresh its file attributes, at least on some NFS clients. | | `core.worktree` | Root directory of the working tree if it is not the parent directory of the `.git` directory | ✅ | The path to the root of the working tree. | ## __fetch__ options diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/ObjectDirectoryTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/ObjectDirectoryTest.java index 7d298ee6f1..33cbc868ca 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/ObjectDirectoryTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/ObjectDirectoryTest.java @@ -50,6 +50,7 @@ import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; @@ -210,17 +211,17 @@ public class ObjectDirectoryTest extends RepositoryTestCase { .fromString("873fb8d667d05436d728c52b1d7a09528e6eb59b"); WindowCursor curs = new WindowCursor(db.getObjectDatabase()); - LooseObjects mock = mock(LooseObjects.class); + Config config = new Config(); + config.setString("core", null, "trustLooseObjectStat", "ALWAYS"); + LooseObjects spy = Mockito.spy(new LooseObjects(config, trash)); UnpackedObjectCache unpackedObjectCacheMock = mock( UnpackedObjectCache.class); - Mockito.when(mock.getObjectLoader(any(), any(), any())) - .thenThrow(new IOException("Stale File Handle")); - Mockito.when(mock.open(curs, id)).thenCallRealMethod(); - Mockito.when(mock.unpackedObjectCache()) - .thenReturn(unpackedObjectCacheMock); + doThrow(new IOException("Stale File Handle")).when(spy) + .getObjectLoader(any(), any(), any()); + doReturn(unpackedObjectCacheMock).when(spy).unpackedObjectCache(); - assertNull(mock.open(curs, id)); + assertNull(spy.open(curs, id)); verify(unpackedObjectCacheMock).remove(id); } @@ -231,7 +232,7 @@ public class ObjectDirectoryTest extends RepositoryTestCase { WindowCursor curs = new WindowCursor(db.getObjectDatabase()); Config config = new Config(); - config.setString("core", null, "trustFolderStat", "false"); + config.setString("core", null, "trustLooseObjectStat", "NEVER"); LooseObjects spy = spy(new LooseObjects(config, db.getObjectDatabase().getDirectory())); diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties index acfe812a20..7c47d3a04f 100644 --- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties +++ b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties @@ -267,6 +267,7 @@ deletingBranches=Deleting branches... deletingNotSupported=Deleting {0} not supported. depthMustBeAt1=Depth must be >= 1 depthWithUnshallow=Depth and unshallow can\'t be used together +deprecatedTrustFolderStat=Option core.trustFolderStat is deprecated, replace it by core.trustStat. destinationIsNotAWildcard=Destination is not a wildcard. detachedHeadDetected=HEAD is detached diffToolNotGivenError=No diff tool provided and no defaults configured. @@ -464,6 +465,7 @@ invalidTimestamp=Invalid timestamp in {0} invalidTimeUnitValue2=Invalid time unit value: {0}.{1}={2} invalidTimeUnitValue3=Invalid time unit value: {0}.{1}.{2}={3} invalidTreeZeroLengthName=Cannot append a tree entry with zero-length name +invalidTrustStat=core.trustStat must not be set to TrustStat.INHERIT, falling back to TrustStat.ALWAYS. invalidURL=Invalid URL {0} invalidWildcards=Invalid wildcards {0} invalidRefSpec=Invalid refspec {0} @@ -615,6 +617,7 @@ peerDidNotSupplyACompleteObjectGraph=peer did not supply a complete object graph personIdentEmailNonNull=E-mail address of PersonIdent must not be null. personIdentNameNonNull=Name of PersonIdent must not be null. postCommitHookFailed=Execution of post-commit hook failed: {0}. +precedenceTrustConfig=Both core.trustFolderStat and core.trustStat are set, ignoring trustFolderStat since trustStat takes precedence. Remove core.trustFolderStat from your configuration. prefixRemote=remote: problemWithResolvingPushRefSpecsLocally=Problem with resolving push ref specs locally: {0} progressMonUploading=Uploading {0} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java index 2d9d2c527c..07e882498c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java @@ -295,6 +295,7 @@ public class JGitText extends TranslationBundle { /***/ public String deleteTagUnexpectedResult; /***/ public String deletingBranches; /***/ public String deletingNotSupported; + /***/ public String deprecatedTrustFolderStat; /***/ public String depthMustBeAt1; /***/ public String depthWithUnshallow; /***/ public String destinationIsNotAWildcard; @@ -493,6 +494,7 @@ public class JGitText extends TranslationBundle { /***/ public String invalidTimeUnitValue2; /***/ public String invalidTimeUnitValue3; /***/ public String invalidTreeZeroLengthName; + /***/ public String invalidTrustStat; /***/ public String invalidURL; /***/ public String invalidWildcards; /***/ public String invalidRefSpec; @@ -645,6 +647,7 @@ public class JGitText extends TranslationBundle { /***/ public String personIdentEmailNonNull; /***/ public String personIdentNameNonNull; /***/ public String postCommitHookFailed; + /***/ public String precedenceTrustConfig; /***/ public String prefixRemote; /***/ public String problemWithResolvingPushRefSpecsLocally; /***/ public String progressMonUploading; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LooseObjects.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LooseObjects.java index 74ef6e139c..909b3e3082 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LooseObjects.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LooseObjects.java @@ -26,8 +26,9 @@ import org.eclipse.jgit.internal.storage.file.FileObjectDatabase.InsertLooseObje import org.eclipse.jgit.lib.AbbreviatedObjectId; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.Config; -import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.CoreConfig; +import org.eclipse.jgit.lib.CoreConfig.TrustStat; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectLoader; import org.eclipse.jgit.util.FileUtils; @@ -55,7 +56,7 @@ class LooseObjects { private final UnpackedObjectCache unpackedObjectCache; - private final boolean trustFolderStat; + private final TrustStat trustLooseObjectStat; /** * Initialize a reference to an on-disk object directory. @@ -68,9 +69,8 @@ class LooseObjects { LooseObjects(Config config, File dir) { directory = dir; unpackedObjectCache = new UnpackedObjectCache(); - trustFolderStat = config.getBoolean( - ConfigConstants.CONFIG_CORE_SECTION, - ConfigConstants.CONFIG_KEY_TRUSTFOLDERSTAT, true); + trustLooseObjectStat = config.get(CoreConfig.KEY) + .getTrustLooseObjectStat(); } /** @@ -108,7 +108,8 @@ class LooseObjects { */ boolean has(AnyObjectId objectId) { boolean exists = hasWithoutRefresh(objectId); - if (trustFolderStat || exists) { + if (trustLooseObjectStat == TrustStat.ALWAYS + || exists) { return exists; } try (InputStream stream = Files.newInputStream(directory.toPath())) { @@ -165,9 +166,29 @@ class LooseObjects { ObjectLoader open(WindowCursor curs, AnyObjectId id) throws IOException { File path = fileFor(id); for (int retries = 0; retries < MAX_STALE_READ_RETRIES; retries++) { - if (trustFolderStat && !path.exists()) { + boolean reload = true; + switch (trustLooseObjectStat) { + case NEVER: break; + case AFTER_OPEN: + try (InputStream stream = Files + .newInputStream(path.getParentFile().toPath())) { + // open the loose object's fanout directory to refresh + // attributes (on some NFS clients) + } catch (FileNotFoundException | NoSuchFileException e) { + // ignore + } + //$FALL-THROUGH$ + case ALWAYS: + if (!path.exists()) { + reload = false; + } + break; + case INHERIT: + // only used in CoreConfig internally + throw new IllegalStateException(); } + if (reload) { try { return getObjectLoader(curs, path, id); } catch (FileNotFoundException noFile) { @@ -181,9 +202,10 @@ class LooseObjects { } if (LOG.isDebugEnabled()) { LOG.debug(MessageFormat.format( - JGitText.get().looseObjectHandleIsStale, id.name(), - Integer.valueOf(retries), Integer.valueOf( - MAX_STALE_READ_RETRIES))); + JGitText.get().looseObjectHandleIsStale, + id.name(), Integer.valueOf(retries), + Integer.valueOf(MAX_STALE_READ_RETRIES))); + } } } } @@ -209,7 +231,7 @@ class LooseObjects { try { return getObjectLoaderWithoutRefresh(curs, path, id); } catch (FileNotFoundException e) { - if (trustFolderStat) { + if (trustLooseObjectStat == TrustStat.ALWAYS) { throw e; } try (InputStream stream = Files @@ -246,7 +268,7 @@ class LooseObjects { return getSizeWithoutRefresh(curs, id); } catch (FileNotFoundException noFile) { try { - if (trustFolderStat) { + if (trustLooseObjectStat == TrustStat.ALWAYS) { throw noFile; } try (InputStream stream = Files diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java index 1ce14b3a18..ef877d8a48 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java @@ -17,6 +17,8 @@ import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; import java.text.MessageFormat; import java.util.ArrayList; import java.util.Arrays; @@ -42,7 +44,8 @@ import org.eclipse.jgit.internal.storage.pack.PackWriter; import org.eclipse.jgit.lib.AbbreviatedObjectId; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.Config; -import org.eclipse.jgit.lib.ConfigConstants; +import org.eclipse.jgit.lib.CoreConfig; +import org.eclipse.jgit.lib.CoreConfig.TrustStat; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectLoader; import org.eclipse.jgit.util.FileUtils; @@ -72,7 +75,7 @@ class PackDirectory { private final AtomicReference packList; - private final boolean trustFolderStat; + private final TrustStat trustPackStat; /** * Initialize a reference to an on-disk 'pack' directory. @@ -86,14 +89,7 @@ class PackDirectory { this.config = config; this.directory = directory; packList = new AtomicReference<>(NO_PACKS); - - // Whether to trust the pack folder's modification time. If set to false - // we will always scan the .git/objects/pack folder to check for new - // pack files. If set to true (default) we use the folder's size, - // modification time, and key (inode) and assume that no new pack files - // can be in this folder if these attributes have not changed. - trustFolderStat = config.getBoolean(ConfigConstants.CONFIG_CORE_SECTION, - ConfigConstants.CONFIG_KEY_TRUSTFOLDERSTAT, true); + trustPackStat = config.get(CoreConfig.KEY).getTrustPackStat(); } /** @@ -365,8 +361,26 @@ class PackDirectory { } boolean searchPacksAgain(PackList old) { - return (!trustFolderStat || old.snapshot.isModified(directory)) - && old != scanPacks(old); + switch (trustPackStat) { + case NEVER: + break; + case AFTER_OPEN: + try (InputStream stream = Files + .newInputStream(directory.toPath())) { + // open the pack directory to refresh attributes (on some NFS clients) + } catch (IOException e) { + // ignore + } + //$FALL-THROUGH$ + case ALWAYS: + if (!old.snapshot.isModified(directory)) { + return false; + } + break; + case INHERIT: + // only used in CoreConfig internally + } + return old != scanPacks(old); } void insert(Pack pack) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java index 33996519f7..05f1ef53a1 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java @@ -64,10 +64,9 @@ import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.errors.ObjectWritingException; import org.eclipse.jgit.events.RefsChangedEvent; import org.eclipse.jgit.internal.JGitText; -import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; -import org.eclipse.jgit.lib.CoreConfig.TrustLooseRefStat; -import org.eclipse.jgit.lib.CoreConfig.TrustPackedRefsStat; +import org.eclipse.jgit.lib.CoreConfig; +import org.eclipse.jgit.lib.CoreConfig.TrustStat; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectIdRef; import org.eclipse.jgit.lib.ProgressMonitor; @@ -185,11 +184,7 @@ public class RefDirectory extends RefDatabase { private List retrySleepMs = RETRY_SLEEP_MS; - private final boolean trustFolderStat; - - private final TrustPackedRefsStat trustPackedRefsStat; - - private final TrustLooseRefStat trustLooseRefStat; + private final CoreConfig coreConfig; RefDirectory(RefDirectory refDb) { parent = refDb.parent; @@ -201,9 +196,7 @@ public class RefDirectory extends RefDatabase { packedRefsFile = refDb.packedRefsFile; looseRefs.set(refDb.looseRefs.get()); packedRefs.set(refDb.packedRefs.get()); - trustFolderStat = refDb.trustFolderStat; - trustPackedRefsStat = refDb.trustPackedRefsStat; - trustLooseRefStat = refDb.trustLooseRefStat; + coreConfig = refDb.coreConfig; inProcessPackedRefsLock = refDb.inProcessPackedRefsLock; } @@ -219,17 +212,7 @@ public class RefDirectory extends RefDatabase { looseRefs.set(RefList. emptyList()); packedRefs.set(NO_PACKED_REFS); - trustFolderStat = db.getConfig() - .getBoolean(ConfigConstants.CONFIG_CORE_SECTION, - ConfigConstants.CONFIG_KEY_TRUSTFOLDERSTAT, true); - trustPackedRefsStat = db.getConfig() - .getEnum(ConfigConstants.CONFIG_CORE_SECTION, null, - ConfigConstants.CONFIG_KEY_TRUST_PACKED_REFS_STAT, - TrustPackedRefsStat.UNSET); - trustLooseRefStat = db.getConfig() - .getEnum(ConfigConstants.CONFIG_CORE_SECTION, null, - ConfigConstants.CONFIG_KEY_TRUST_LOOSE_REF_STAT, - TrustLooseRefStat.ALWAYS); + coreConfig = db.getConfig().get(CoreConfig.KEY); inProcessPackedRefsLock = new ReentrantLock(true); } @@ -979,7 +962,7 @@ public class RefDirectory extends RefDatabase { PackedRefList getPackedRefs() throws IOException { final PackedRefList curList = packedRefs.get(); - switch (trustPackedRefsStat) { + switch (coreConfig.getTrustPackedRefsStat()) { case NEVER: break; case AFTER_OPEN: @@ -995,12 +978,8 @@ public class RefDirectory extends RefDatabase { return curList; } break; - case UNSET: - if (trustFolderStat - && !curList.snapshot.isModified(packedRefsFile)) { - return curList; - } - break; + case INHERIT: + // only used in CoreConfig internally } return refreshPackedRefs(curList); @@ -1186,7 +1165,7 @@ public class RefDirectory extends RefDatabase { LooseRef scanRef(LooseRef ref, String name) throws IOException { final File path = fileFor(name); - if (trustLooseRefStat.equals(TrustLooseRefStat.AFTER_OPEN)) { + if (coreConfig.getTrustLooseRefStat() == TrustStat.AFTER_OPEN) { refreshPathToLooseRef(Paths.get(name)); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java index 5068a6cce0..30e7de47c3 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java @@ -600,10 +600,20 @@ public final class ConfigConstants { /** * The "trustfolderstat" key in the "core" section + * * @since 3.6 + * @deprecated use {CONFIG_KEY_TRUST_STAT} instead */ + @Deprecated(since = "7.2", forRemoval = true) public static final String CONFIG_KEY_TRUSTFOLDERSTAT = "trustfolderstat"; + /** + * The "trustfilestat" key in the "core"section + * + * @since 7.2 + */ + public static final String CONFIG_KEY_TRUST_STAT = "truststat"; + /** * The "supportsAtomicFileCreation" key in the "core" section * @@ -1022,6 +1032,19 @@ public final class ConfigConstants { */ public static final String CONFIG_KEY_TRUST_LOOSE_REF_STAT = "trustLooseRefStat"; + /** + * The "trustLooseRefStat" key + * + * @since 7.2 + */ + public static final String CONFIG_KEY_TRUST_PACK_STAT = "trustPackStat"; + + /** + * The "trustLooseObjectFileStat" key + * + * @since 7.2 + */ + public static final String CONFIG_KEY_TRUST_LOOSE_OBJECT_STAT = "trustLooseObjectStat"; /** * The "pack.preserveOldPacks" key * diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/CoreConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/CoreConfig.java index 49602a75eb..e43c9653dd 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/CoreConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/CoreConfig.java @@ -17,12 +17,16 @@ package org.eclipse.jgit.lib; import static java.util.zip.Deflater.DEFAULT_COMPRESSION; +import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.lib.Config.SectionParser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * This class keeps git repository core parameters. */ public class CoreConfig { + private static final Logger LOG = LoggerFactory.getLogger(CoreConfig.class); /** Key for {@link Config#get(SectionParser)}. */ public static final Config.SectionParser KEY = CoreConfig::new; @@ -127,7 +131,9 @@ public class CoreConfig { * Permissible values for {@code core.trustPackedRefsStat}. * * @since 6.1.1 + * @deprecated use {@link TrustStat} instead */ + @Deprecated(since = "7.2", forRemoval = true) public enum TrustPackedRefsStat { /** Do not trust file attributes of the packed-refs file. */ NEVER, @@ -135,12 +141,15 @@ public class CoreConfig { /** Trust file attributes of the packed-refs file. */ ALWAYS, - /** Open and close the packed-refs file to refresh its file attributes - * and then trust it. */ + /** + * Open and close the packed-refs file to refresh its file attributes + * and then trust it. + */ AFTER_OPEN, - /** {@code core.trustPackedRefsStat} defaults to this when it is - * not set */ + /** + * {@code core.trustPackedRefsStat} defaults to this when it is not set + */ UNSET } @@ -148,17 +157,44 @@ public class CoreConfig { * Permissible values for {@code core.trustLooseRefStat}. * * @since 6.9 + * @deprecated use {@link TrustStat} instead */ + @Deprecated(since = "7.2", forRemoval = true) public enum TrustLooseRefStat { /** Trust file attributes of the loose ref. */ ALWAYS, - /** Open and close parent directories of the loose ref file until the - * repository root to refresh its file attributes and then trust it. */ + /** + * Open and close parent directories of the loose ref file until the + * repository root to refresh its file attributes and then trust it. + */ AFTER_OPEN, } + /** + * Values for {@code core.trustXXX} options. + * + * @since 7.2 + */ + public enum TrustStat { + /** Do not trust file attributes of a File. */ + NEVER, + + /** Always trust file attributes of a File. */ + ALWAYS, + + /** Open and close the File to refresh its file attributes + * and then trust it. */ + AFTER_OPEN, + + /** + * Used for specific options to inherit value from value set for + * core.trustStat. + */ + INHERIT + } + private final int compression; private final int packIndexVersion; @@ -169,6 +205,16 @@ public class CoreConfig { private final boolean commitGraph; + private final TrustStat trustStat; + + private final TrustStat trustPackedRefsStat; + + private final TrustStat trustLooseRefStat; + + private final TrustStat trustPackStat; + + private final TrustStat trustLooseObjectStat; + /** * Options for symlink handling * @@ -198,7 +244,13 @@ public class CoreConfig { DOTGITONLY } - private CoreConfig(Config rc) { + /** + * Create a new core configuration from the passed configuration. + * + * @param rc + * git configuration + */ + CoreConfig(Config rc) { compression = rc.getInt(ConfigConstants.CONFIG_CORE_SECTION, ConfigConstants.CONFIG_KEY_COMPRESSION, DEFAULT_COMPRESSION); packIndexVersion = rc.getInt(ConfigConstants.CONFIG_PACK_SECTION, @@ -210,6 +262,60 @@ public class CoreConfig { commitGraph = rc.getBoolean(ConfigConstants.CONFIG_CORE_SECTION, ConfigConstants.CONFIG_COMMIT_GRAPH, DEFAULT_COMMIT_GRAPH_ENABLE); + + trustStat = parseTrustStat(rc); + trustPackedRefsStat = parseTrustPackedRefsStat(rc); + trustLooseRefStat = parseTrustLooseRefStat(rc); + trustPackStat = parseTrustPackFileStat(rc); + trustLooseObjectStat = parseTrustLooseObjectFileStat(rc); + } + + private static TrustStat parseTrustStat(Config rc) { + Boolean tfs = rc.getBoolean(ConfigConstants.CONFIG_CORE_SECTION, + ConfigConstants.CONFIG_KEY_TRUSTFOLDERSTAT); + TrustStat ts = rc.getEnum(TrustStat.values(), + ConfigConstants.CONFIG_CORE_SECTION, null, + ConfigConstants.CONFIG_KEY_TRUST_STAT); + if (tfs != null) { + if (ts == null) { + LOG.warn(JGitText.get().deprecatedTrustFolderStat); + return tfs.booleanValue() ? TrustStat.ALWAYS : TrustStat.NEVER; + } + LOG.warn(JGitText.get().precedenceTrustConfig); + } + if (ts == null) { + ts = TrustStat.ALWAYS; + } else if (ts == TrustStat.INHERIT) { + LOG.warn(JGitText.get().invalidTrustStat); + ts = TrustStat.ALWAYS; + } + return ts; + } + + private TrustStat parseTrustPackedRefsStat(Config rc) { + return inheritParseTrustStat(rc, + ConfigConstants.CONFIG_KEY_TRUST_PACKED_REFS_STAT); + } + + private TrustStat parseTrustLooseRefStat(Config rc) { + return inheritParseTrustStat(rc, + ConfigConstants.CONFIG_KEY_TRUST_LOOSE_REF_STAT); + } + + private TrustStat parseTrustPackFileStat(Config rc) { + return inheritParseTrustStat(rc, + ConfigConstants.CONFIG_KEY_TRUST_PACK_STAT); + } + + private TrustStat parseTrustLooseObjectFileStat(Config rc) { + return inheritParseTrustStat(rc, + ConfigConstants.CONFIG_KEY_TRUST_LOOSE_OBJECT_STAT); + } + + private TrustStat inheritParseTrustStat(Config rc, String key) { + TrustStat t = rc.getEnum(ConfigConstants.CONFIG_CORE_SECTION, null, key, + TrustStat.INHERIT); + return t == TrustStat.INHERIT ? trustStat : t; } /** @@ -260,4 +366,56 @@ public class CoreConfig { public boolean enableCommitGraph() { return commitGraph; } + + /** + * Get how far we can trust file attributes of packed-refs file which is + * used to store {@link org.eclipse.jgit.lib.Ref}s in + * {@link org.eclipse.jgit.internal.storage.file.RefDirectory}. + * + * @return how far we can trust file attributes of packed-refs file. + * + * @since 7.2 + */ + public TrustStat getTrustPackedRefsStat() { + return trustPackedRefsStat; + } + + /** + * Get how far we can trust file attributes of loose ref files which are + * used to store {@link org.eclipse.jgit.lib.Ref}s in + * {@link org.eclipse.jgit.internal.storage.file.RefDirectory}. + * + * @return how far we can trust file attributes of loose ref files. + * + * @since 7.2 + */ + public TrustStat getTrustLooseRefStat() { + return trustLooseRefStat; + } + + /** + * Get how far we can trust file attributes of packed-refs file which is + * used to store {@link org.eclipse.jgit.lib.Ref}s in + * {@link org.eclipse.jgit.internal.storage.file.RefDirectory}. + * + * @return how far we can trust file attributes of packed-refs file. + * + * @since 7.2 + */ + public TrustStat getTrustPackStat() { + return trustPackStat; + } + + /** + * Get how far we can trust file attributes of loose ref files which are + * used to store {@link org.eclipse.jgit.lib.Ref}s in + * {@link org.eclipse.jgit.internal.storage.file.RefDirectory}. + * + * @return how far we can trust file attributes of loose ref files. + * + * @since 7.2 + */ + public TrustStat getTrustLooseObjectStat() { + return trustLooseObjectStat; + } } -- cgit v1.2.3 From f41253804068768c1d44a01808805e15a7b63eb6 Mon Sep 17 00:00:00 2001 From: Thomas Wolf Date: Fri, 7 Feb 2025 21:25:08 +0100 Subject: Merge: improve handling of case-variants Ensure that on a case-insensitive filesystem a merge that includes a rename of a file from one case variant to another does not delete the file. Basically make sure that we don't delete files that we had marked under a case variant as "keep" before, and ensure that when checking out a file, it is written to the file system with the exact casing recorded in the git tree. Bug: egit-76 Change-Id: Ibbc9ba97c70971ba3e83381b41364a5529f5a5dc --- .../tst/org/eclipse/jgit/api/MergeCommandTest.java | 92 ++++++++++++++++++++++ .../src/org/eclipse/jgit/dircache/Checkout.java | 12 ++- .../eclipse/jgit/dircache/DirCacheCheckout.java | 68 +++++++--------- .../src/org/eclipse/jgit/lib/Repository.java | 40 ++++++++++ 4 files changed, 170 insertions(+), 42 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/MergeCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/MergeCommandTest.java index 503fef9916..1ec506798c 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/MergeCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/MergeCommandTest.java @@ -21,6 +21,9 @@ import static org.junit.Assert.fail; import static org.junit.Assume.assumeTrue; import java.io.File; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Iterator; import java.util.regex.Pattern; @@ -46,6 +49,7 @@ import org.eclipse.jgit.util.FS; import org.eclipse.jgit.util.FileUtils; import org.eclipse.jgit.util.GitDateFormatter; import org.eclipse.jgit.util.GitDateFormatter.Format; +import org.junit.Assume; import org.junit.Before; import org.junit.Test; import org.junit.experimental.theories.DataPoints; @@ -2101,6 +2105,94 @@ public class MergeCommandTest extends RepositoryTestCase { } } + @Test + public void testMergeCaseInsensitiveRename() throws Exception { + Assume.assumeTrue( + "Test makes only sense on a case-insensitive file system", + db.isWorkTreeCaseInsensitive()); + try (Git git = new Git(db)) { + writeTrashFile("a", "aaa"); + git.add().addFilepattern("a").call(); + RevCommit initialCommit = git.commit().setMessage("initial").call(); + // "Rename" "a" to "A" + git.rm().addFilepattern("a").call(); + writeTrashFile("A", "aaa"); + git.add().addFilepattern("A").call(); + RevCommit master = git.commit().setMessage("rename to A").call(); + + createBranch(initialCommit, "refs/heads/side"); + checkoutBranch("refs/heads/side"); + + writeTrashFile("b", "bbb"); + git.add().addFilepattern("b").call(); + git.commit().setMessage("side").call(); + + // Merge master into side + MergeResult result = git.merge().include(master) + .setStrategy(MergeStrategy.RECURSIVE).call(); + assertEquals(MergeStatus.MERGED, result.getMergeStatus()); + assertTrue(new File(db.getWorkTree(), "A").isFile()); + // Double check + boolean found = true; + try (DirectoryStream dir = Files + .newDirectoryStream(db.getWorkTree().toPath())) { + for (Path p : dir) { + found = "A".equals(p.getFileName().toString()); + if (found) { + break; + } + } + } + assertTrue(found); + } + } + + @Test + public void testMergeCaseInsensitiveRenameConflict() throws Exception { + Assume.assumeTrue( + "Test makes only sense on a case-insensitive file system", + db.isWorkTreeCaseInsensitive()); + try (Git git = new Git(db)) { + writeTrashFile("a", "aaa"); + git.add().addFilepattern("a").call(); + RevCommit initialCommit = git.commit().setMessage("initial").call(); + // "Rename" "a" to "A" and change it + git.rm().addFilepattern("a").call(); + writeTrashFile("A", "yyy"); + git.add().addFilepattern("A").call(); + RevCommit master = git.commit().setMessage("rename to A").call(); + + createBranch(initialCommit, "refs/heads/side"); + checkoutBranch("refs/heads/side"); + + writeTrashFile("a", "xxx"); + git.add().addFilepattern("a").call(); + git.commit().setMessage("side").call(); + + // Merge master into side + MergeResult result = git.merge().include(master) + .setStrategy(MergeStrategy.RECURSIVE).call(); + assertEquals(MergeStatus.CONFLICTING, result.getMergeStatus()); + File a = new File(db.getWorkTree(), "A"); + assertTrue(a.isFile()); + // Double check + boolean found = true; + try (DirectoryStream dir = Files + .newDirectoryStream(db.getWorkTree().toPath())) { + for (Path p : dir) { + found = "A".equals(p.getFileName().toString()); + if (found) { + break; + } + } + } + assertTrue(found); + assertEquals(1, result.getConflicts().size()); + assertTrue(result.getConflicts().containsKey("a")); + checkFile(a, "yyy"); + } + } + private static void setExecutable(Git git, String path, boolean executable) { FS.DETECTED.setExecute( new File(git.getRepository().getWorkTree(), path), executable); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/Checkout.java b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/Checkout.java index accf732dc7..de02aecdb9 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/Checkout.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/Checkout.java @@ -217,10 +217,18 @@ public class Checkout { } } try { - if (recursiveDelete && Files.isDirectory(f.toPath(), - LinkOption.NOFOLLOW_LINKS)) { + boolean isDir = Files.isDirectory(f.toPath(), + LinkOption.NOFOLLOW_LINKS); + if (recursiveDelete && isDir) { FileUtils.delete(f, FileUtils.RECURSIVE); } + if (cache.getRepository().isWorkTreeCaseInsensitive() && !isDir) { + // We cannot rely on rename via Files.move() to work correctly + // if the target exists in a case variant. For instance with JDK + // 17 on Mac OS, the existing case-variant name is kept. On + // Windows 11 it would work and use the name given in 'f'. + FileUtils.delete(f, FileUtils.SKIP_MISSING); + } FileUtils.rename(tmpFile, f, StandardCopyOption.ATOMIC_MOVE); cachedParent.remove(f.getName()); } catch (IOException e) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java index 4f78404f48..d61a453973 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java @@ -5,7 +5,7 @@ * Copyright (C) 2006, Shawn O. Pearce * Copyright (C) 2010, Chrisian Halstrick * Copyright (C) 2019, 2020, Andre Bossert - * Copyright (C) 2017, 2023, Thomas Wolf and others + * Copyright (C) 2017, 2025, Thomas Wolf and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -31,6 +31,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeSet; import org.eclipse.jgit.api.errors.CanceledException; import org.eclipse.jgit.api.errors.FilterFailedException; @@ -66,7 +67,6 @@ import org.eclipse.jgit.treewalk.WorkingTreeOptions; import org.eclipse.jgit.treewalk.filter.PathFilter; import org.eclipse.jgit.util.FS; import org.eclipse.jgit.util.FS.ExecutionResult; -import org.eclipse.jgit.util.IntList; import org.eclipse.jgit.util.SystemReader; import org.eclipse.jgit.util.io.EolStreamTypeUtil; import org.slf4j.Logger; @@ -113,9 +113,11 @@ public class DirCacheCheckout { private Map updated = new LinkedHashMap<>(); + private Set existing; + private ArrayList conflicts = new ArrayList<>(); - private ArrayList removed = new ArrayList<>(); + private TreeSet removed; private ArrayList kept = new ArrayList<>(); @@ -185,7 +187,7 @@ public class DirCacheCheckout { * @return a list of all files removed by this checkout */ public List getRemoved() { - return removed; + return new ArrayList<>(removed); } /** @@ -214,6 +216,14 @@ public class DirCacheCheckout { this.mergeCommitTree = mergeCommitTree; this.workingTree = workingTree; this.initialCheckout = !repo.isBare() && !repo.getIndexFile().exists(); + boolean caseInsensitive = !repo.isBare() + && repo.isWorkTreeCaseInsensitive(); + this.removed = caseInsensitive + ? new TreeSet<>(String::compareToIgnoreCase) + : new TreeSet<>(); + this.existing = caseInsensitive + ? new TreeSet<>(String::compareToIgnoreCase) + : null; } /** @@ -521,6 +531,13 @@ public class DirCacheCheckout { // update our index builder.finish(); + // On case-insensitive file systems we may have a case variant kept + // and another one removed. In that case, don't remove it. + if (existing != null) { + removed.removeAll(existing); + existing.clear(); + } + // init progress reporting int numTotal = removed.size() + updated.size() + conflicts.size(); monitor.beginTask(JGitText.get().checkingOutFiles, numTotal); @@ -531,9 +548,9 @@ public class DirCacheCheckout { // when deleting files process them in the opposite order as they have // been reported. This ensures the files are deleted before we delete // their parent folders - IntList nonDeleted = new IntList(); - for (int i = removed.size() - 1; i >= 0; i--) { - String r = removed.get(i); + Iterator iter = removed.descendingIterator(); + while (iter.hasNext()) { + String r = iter.next(); file = new File(repo.getWorkTree(), r); if (!file.delete() && repo.getFS().exists(file)) { // The list of stuff to delete comes from the index @@ -542,7 +559,7 @@ public class DirCacheCheckout { // to delete it. A submodule is not empty, so it // is safe to check this after a failed delete. if (!repo.getFS().isDirectory(file)) { - nonDeleted.add(i); + iter.remove(); toBeDeleted.add(r); } } else { @@ -560,8 +577,6 @@ public class DirCacheCheckout { if (file != null) { removeEmptyParents(file); } - removed = filterOut(removed, nonDeleted); - nonDeleted = null; Iterator> toUpdate = updated .entrySet().iterator(); Map.Entry e = null; @@ -633,36 +648,6 @@ public class DirCacheCheckout { return toBeDeleted.isEmpty(); } - private static ArrayList filterOut(ArrayList strings, - IntList indicesToRemove) { - int n = indicesToRemove.size(); - if (n == strings.size()) { - return new ArrayList<>(0); - } - switch (n) { - case 0: - return strings; - case 1: - strings.remove(indicesToRemove.get(0)); - return strings; - default: - int length = strings.size(); - ArrayList result = new ArrayList<>(length - n); - // Process indicesToRemove from the back; we know that it - // contains indices in descending order. - int j = n - 1; - int idx = indicesToRemove.get(j); - for (int i = 0; i < length; i++) { - if (i == idx) { - idx = (--j >= 0) ? indicesToRemove.get(j) : -1; - } else { - result.add(strings.get(i)); - } - } - return result; - } - } - private static boolean isSamePrefix(String a, String b) { int as = a.lastIndexOf('/'); int bs = b.lastIndexOf('/'); @@ -1233,6 +1218,9 @@ public class DirCacheCheckout { if (!FileMode.TREE.equals(e.getFileMode())) { builder.add(e); } + if (existing != null) { + existing.add(path); + } if (force) { if (f == null || f.isModified(e, true, walk.getObjectReader())) { kept.add(path); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java index 757473878b..c9dc6da4ba 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/Repository.java @@ -26,6 +26,8 @@ import java.io.IOException; import java.io.OutputStream; import java.io.UncheckedIOException; import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.LinkOption; import java.text.MessageFormat; import java.util.ArrayList; import java.util.Collection; @@ -33,10 +35,12 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; import java.util.regex.Pattern; import org.eclipse.jgit.annotations.NonNull; @@ -132,6 +136,8 @@ public abstract class Repository implements AutoCloseable { private final String initialBranch; + private final AtomicReference caseInsensitiveWorktree = new AtomicReference<>(); + /** * Initialize a new repository instance. * @@ -1576,6 +1582,40 @@ public abstract class Repository implements AutoCloseable { return workTree; } + /** + * Tells whether the work tree is on a case-insensitive file system. + * + * @return {@code true} if the work tree is case-insensitive; {@code false} + * otherwise + * @throws NoWorkTreeException + * if the repository is bare + * @since 7.2 + */ + public boolean isWorkTreeCaseInsensitive() throws NoWorkTreeException { + Boolean flag = caseInsensitiveWorktree.get(); + if (flag == null) { + File directory = getWorkTree(); + // See if we can find ".git" also as ".GIT". + File dotGit = new File(directory, Constants.DOT_GIT); + if (Files.exists(dotGit.toPath(), LinkOption.NOFOLLOW_LINKS)) { + dotGit = new File(directory, + Constants.DOT_GIT.toUpperCase(Locale.ROOT)); + flag = Boolean.valueOf(Files.exists(dotGit.toPath(), + LinkOption.NOFOLLOW_LINKS)); + } else { + // Fall back to a mostly sane default. On Mac, HFS+ and APFS + // partitions are case-insensitive by default but can be + // configured to be case-sensitive. + SystemReader system = SystemReader.getInstance(); + flag = Boolean.valueOf(system.isWindows() || system.isMacOS()); + } + if (!caseInsensitiveWorktree.compareAndSet(null, flag)) { + flag = caseInsensitiveWorktree.get(); + } + } + return flag.booleanValue(); + } + /** * Force a scan for changed refs. Fires an IndexChangedEvent(false) if * changes are detected. -- cgit v1.2.3 From 4c4bef885c59662c22cbaa940ac7bec0dfee2d21 Mon Sep 17 00:00:00 2001 From: Thomas Wolf Date: Sat, 8 Feb 2025 14:09:39 +0100 Subject: DirCacheCheckout.preScanOneTree: consider mode bits If only the file mode is changed, it's still a change and we must check out the entry from the commit. Bug: jgit-138 Change-Id: I83adebe563fcdb4cbe330edb44884d55ed463c2c --- .../tst/org/eclipse/jgit/api/ResetCommandTest.java | 26 ++++++++++++++++++++++ .../eclipse/jgit/dircache/DirCacheCheckout.java | 13 ++++++----- 2 files changed, 34 insertions(+), 5 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/ResetCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/ResetCommandTest.java index 4265806078..99873e1be1 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/ResetCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/ResetCommandTest.java @@ -42,6 +42,7 @@ import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.treewalk.TreeWalk; import org.eclipse.jgit.util.FileUtils; import org.junit.Assert; +import org.junit.Assume; import org.junit.Test; public class ResetCommandTest extends RepositoryTestCase { @@ -555,6 +556,31 @@ public class ResetCommandTest extends RepositoryTestCase { assertNull(db.resolve(Constants.HEAD)); } + @Test + public void testHardResetFileMode() throws Exception { + Assume.assumeTrue("Test must be able to set executable bit", + db.getFS().supportsExecute()); + git = new Git(db); + File a = writeTrashFile("a.txt", "aaa"); + File b = writeTrashFile("b.txt", "bbb"); + db.getFS().setExecute(b, true); + assertFalse(db.getFS().canExecute(a)); + assertTrue(db.getFS().canExecute(b)); + git.add().addFilepattern("a.txt").addFilepattern("b.txt").call(); + RevCommit commit = git.commit().setMessage("files created").call(); + db.getFS().setExecute(a, true); + db.getFS().setExecute(b, false); + assertTrue(db.getFS().canExecute(a)); + assertFalse(db.getFS().canExecute(b)); + git.add().addFilepattern("a.txt").addFilepattern("b.txt").call(); + git.commit().setMessage("change exe bits").call(); + Ref ref = git.reset().setRef(commit.getName()).setMode(HARD).call(); + assertSameAsHead(ref); + assertEquals(commit.getId(), ref.getObjectId()); + assertFalse(db.getFS().canExecute(a)); + assertTrue(db.getFS().canExecute(b)); + } + private void assertReflog(ObjectId prevHead, ObjectId head) throws IOException { // Check the reflog for HEAD diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java index d61a453973..18d77482e0 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCacheCheckout.java @@ -410,9 +410,11 @@ public class DirCacheCheckout { // content to be checked out. update(m); } - } else + } else { update(m); - } else if (f == null || !m.idEqual(i)) { + } + } else if (f == null || !m.idEqual(i) + || m.getEntryRawMode() != i.getEntryRawMode()) { // The working tree file is missing or the merge content differs // from index content update(m); @@ -420,11 +422,11 @@ public class DirCacheCheckout { // The index contains a file (and not a folder) if (f.isModified(i.getDirCacheEntry(), true, this.walk.getObjectReader()) - || i.getDirCacheEntry().getStage() != 0) + || i.getDirCacheEntry().getStage() != 0) { // The working tree file is dirty or the index contains a // conflict update(m); - else { + } else { // update the timestamp of the index with the one from the // file if not set, as we are sure to be in sync here. DirCacheEntry entry = i.getDirCacheEntry(); @@ -434,9 +436,10 @@ public class DirCacheCheckout { } keep(i.getEntryPathString(), entry, f); } - } else + } else { // The index contains a folder keep(i.getEntryPathString(), i.getDirCacheEntry(), f); + } } else { // There is no entry in the merge commit. Means: we want to delete // what's currently in the index and working tree -- cgit v1.2.3 From 072e93fded7229f00002232b33ed91a0ef614ddb Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Thu, 23 Jan 2025 15:49:30 -0800 Subject: midx.MultiPackIndexWriter: a writer for the multipack index format Fromg git documentation[1]: While the pack-indexes provide fast lookup per packfile, this performance degrades as the number of packfiles increases, because abbreviations need to inspect every packfile and we are more likely to have a miss on our most-recently-used packfile. For some large repositories, repacking into a single packfile is not feasible due to storage space or excessive repack times. (...) The multi-pack-index (MIDX for short) stores a list of objects and their offsets into multiple packfiles. (...) Thus, we can provide O(log N) lookup time for any number of packfiles. This is a writer of the multipack index format. The test only verifies the "shape" of the file, when we get a parser we can check also the values (specially the large offset handling). On the JGit repository, the multipack index generated by this writer passes the validation of `git multi-pack-index verify`. [1] https://git-scm.com/docs/pack-format#_multi_pack_index_midx_files_have_the_following_format Change-Id: I1fca599c4ebf28154f28f039c2c4cfe75b2dc79d --- .../storage/midx/CgitMidxCompatibilityTest.java | 210 ++++++++++ .../storage/midx/MultiPackIndexWriterTest.java | 161 ++++++++ .../org/eclipse/jgit/internal/JGitText.properties | 2 + .../src/org/eclipse/jgit/internal/JGitText.java | 2 + .../storage/midx/MultiPackIndexConstants.java | 55 +++ .../storage/midx/MultiPackIndexWriter.java | 422 +++++++++++++++++++++ 6 files changed, 852 insertions(+) create mode 100644 org.eclipse.jgit.test/exttst/org/eclipse/jgit/internal/storage/midx/CgitMidxCompatibilityTest.java create mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/midx/MultiPackIndexWriterTest.java create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexConstants.java create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexWriter.java (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/exttst/org/eclipse/jgit/internal/storage/midx/CgitMidxCompatibilityTest.java b/org.eclipse.jgit.test/exttst/org/eclipse/jgit/internal/storage/midx/CgitMidxCompatibilityTest.java new file mode 100644 index 0000000000..bbff7d433f --- /dev/null +++ b/org.eclipse.jgit.test/exttst/org/eclipse/jgit/internal/storage/midx/CgitMidxCompatibilityTest.java @@ -0,0 +1,210 @@ +/* + * Copyright (C) 2025, Google Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.internal.storage.midx; + +import static org.eclipse.jgit.internal.storage.midx.MultiPackIndexConstants.CHUNK_LOOKUP_WIDTH; +import static org.eclipse.jgit.internal.storage.midx.MultiPackIndexConstants.MIDX_CHUNKID_OBJECTOFFSETS; +import static org.eclipse.jgit.internal.storage.midx.MultiPackIndexConstants.MIDX_CHUNKID_OIDFANOUT; +import static org.eclipse.jgit.internal.storage.midx.MultiPackIndexConstants.MIDX_CHUNKID_OIDLOOKUP; +import static org.eclipse.jgit.internal.storage.midx.MultiPackIndexConstants.MIDX_CHUNKID_PACKNAMES; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.eclipse.jgit.internal.storage.file.ObjectDirectory; +import org.eclipse.jgit.internal.storage.file.Pack; +import org.eclipse.jgit.internal.storage.file.PackFile; +import org.eclipse.jgit.internal.storage.file.PackIndex; +import org.eclipse.jgit.internal.storage.pack.PackExt; +import org.eclipse.jgit.lib.NullProgressMonitor; +import org.eclipse.jgit.test.resources.SampleDataRepositoryTestCase; +import org.eclipse.jgit.util.NB; +import org.junit.Test; + +public class CgitMidxCompatibilityTest extends SampleDataRepositoryTestCase { + + @Test + public void jgitMidx_verifyByCgit() + throws IOException, InterruptedException { + byte[] jgitMidxBytes = generateJGitMidx(); + writeMidx(jgitMidxBytes); + assertEquals("cgit exit code", 0, run_cgit_multipackindex_verify()); + } + + @Test + public void compareBasicChunkSizes() + throws IOException, InterruptedException { + // We cannot compare byte-by-byte because there are optional chunks and + // it is not guaranteed what cgit and jgit will generate + byte[] jgitMidxBytes = generateJGitMidx(); + assertEquals("cgit exit code", 0, run_cgit_multipackindex_write()); + byte[] cgitMidxBytes = readCgitMidx(); + + RawMultiPackIndex jgitMidx = new RawMultiPackIndex(jgitMidxBytes); + RawMultiPackIndex cgitMidx = new RawMultiPackIndex(cgitMidxBytes); + + // This is a fixed sized chunk + assertEquals(256 * 4, cgitMidx.getChunkSize(MIDX_CHUNKID_OIDFANOUT)); + assertArrayEquals(cgitMidx.getRawChunk(MIDX_CHUNKID_OIDFANOUT), + jgitMidx.getRawChunk(MIDX_CHUNKID_OIDFANOUT)); + + assertArrayEquals(cgitMidx.getRawChunk(MIDX_CHUNKID_OIDLOOKUP), + jgitMidx.getRawChunk(MIDX_CHUNKID_OIDLOOKUP)); + + // The spec has changed from padding packnames to a multile of four, to + // move the packname chunk to the end of the file. + // git 2.48 pads the packs names to a multiple of 4 + // jgit puts the chunk at the end + byte[] cgitPacknames = trimPadding( + cgitMidx.getRawChunk(MIDX_CHUNKID_PACKNAMES)); + assertArrayEquals(cgitPacknames, + jgitMidx.getRawChunk(MIDX_CHUNKID_PACKNAMES)); + + assertArrayEquals(cgitMidx.getRawChunk(MIDX_CHUNKID_OBJECTOFFSETS), + jgitMidx.getRawChunk(MIDX_CHUNKID_OBJECTOFFSETS)); + + } + + private byte[] generateJGitMidx() throws IOException { + Map indexes = new HashMap<>(); + for (Pack pack : db.getObjectDatabase().getPacks()) { + PackFile packFile = pack.getPackFile().create(PackExt.INDEX); + indexes.put(packFile.getName(), pack.getIndex()); + } + + MultiPackIndexWriter writer = new MultiPackIndexWriter(); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + writer.write(NullProgressMonitor.INSTANCE, out, indexes); + return out.toByteArray(); + } + + private int run_cgit_multipackindex_write() + throws IOException, InterruptedException { + String[] command = new String[] { "git", "multi-pack-index", "write" }; + Process proc = Runtime.getRuntime().exec(command, new String[0], + db.getDirectory()); + return proc.waitFor(); + } + + private int run_cgit_multipackindex_verify() + throws IOException, InterruptedException { + String[] command = new String[] { "git", "multi-pack-index", "verify" }; + Process proc = Runtime.getRuntime().exec(command, new String[0], + db.getDirectory()); + return proc.waitFor(); + } + + private byte[] readCgitMidx() throws IOException { + File midx = getMIdxStandardLocation(); + assertTrue("cgit multi-pack-index exists", midx.exists()); + return Files.readAllBytes(midx.toPath()); + } + + private void writeMidx(byte[] midx) throws IOException { + File midxFile = getMIdxStandardLocation(); + Files.write(midxFile.toPath(), midx); + } + + private File getMIdxStandardLocation() { + return new File( + ((ObjectDirectory) db.getObjectDatabase()).getPackDirectory(), + "multi-pack-index"); + } + + private byte[] trimPadding(byte[] data) { + // Chunk MUST have one \0, we want to remove any extra \0 + int newEnd = data.length - 1; + while (newEnd - 1 >= 0 && data[newEnd - 1] == 0) { + newEnd--; + } + + if (newEnd == data.length - 1) { + return data; + } + return Arrays.copyOfRange(data, 0, newEnd + 1); + } + + private static class RawMultiPackIndex { + private final List chunks; + + private final byte[] midx; + + private RawMultiPackIndex(byte[] midx) { + this.chunks = readChunks(midx); + this.midx = midx; + } + + long getChunkSize(int chunkId) { + int chunkPos = findChunkPosition(chunks, chunkId); + return chunks.get(chunkPos + 1).offset + - chunks.get(chunkPos).offset; + } + + long getOffset(int chunkId) { + return chunks.get(findChunkPosition(chunks, chunkId)).offset; + } + + private long getNextOffset(int chunkId) { + return chunks.get(findChunkPosition(chunks, chunkId) + 1).offset; + } + + byte[] getRawChunk(int chunkId) { + int start = (int) getOffset(chunkId); + int end = (int) getNextOffset(chunkId); + return Arrays.copyOfRange(midx, start, end); + } + + private static int findChunkPosition(List chunks, + int id) { + int chunkPos = -1; + for (int i = 0; i < chunks.size(); i++) { + if (chunks.get(i).id() == id) { + chunkPos = i; + break; + } + } + if (chunkPos == -1) { + throw new IllegalStateException("Chunk doesn't exist"); + } + return chunkPos; + } + + private List readChunks(byte[] midx) { + // Read the number of "chunkOffsets" (1 byte) + int chunkCount = midx[6]; + byte[] lookupBuffer = new byte[CHUNK_LOOKUP_WIDTH + * (chunkCount + 1)]; + System.arraycopy(midx, 12, lookupBuffer, 0, lookupBuffer.length); + + List chunks = new ArrayList<>(chunkCount + 1); + for (int i = 0; i <= chunkCount; i++) { + // chunks[chunkCount] is just a marker, in order to record the + // length of the last chunk. + int id = NB.decodeInt32(lookupBuffer, i * 12); + long offset = NB.decodeInt64(lookupBuffer, i * 12 + 4); + chunks.add(new ChunkSegment(id, offset)); + } + return chunks; + } + } + + private record ChunkSegment(int id, long offset) { + } +} diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/midx/MultiPackIndexWriterTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/midx/MultiPackIndexWriterTest.java new file mode 100644 index 0000000000..82f3eb1e08 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/midx/MultiPackIndexWriterTest.java @@ -0,0 +1,161 @@ +/* + * Copyright (C) 2025, Google Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.internal.storage.midx; + +import static org.eclipse.jgit.internal.storage.midx.MultiPackIndexConstants.CHUNK_LOOKUP_WIDTH; +import static org.eclipse.jgit.internal.storage.midx.MultiPackIndexConstants.MIDX_CHUNKID_LARGEOFFSETS; +import static org.eclipse.jgit.internal.storage.midx.MultiPackIndexConstants.MIDX_CHUNKID_OBJECTOFFSETS; +import static org.eclipse.jgit.internal.storage.midx.MultiPackIndexConstants.MIDX_CHUNKID_OIDFANOUT; +import static org.eclipse.jgit.internal.storage.midx.MultiPackIndexConstants.MIDX_CHUNKID_OIDLOOKUP; +import static org.eclipse.jgit.internal.storage.midx.MultiPackIndexConstants.MIDX_CHUNKID_PACKNAMES; +import static org.eclipse.jgit.internal.storage.midx.MultiPackIndexConstants.MIDX_CHUNKID_REVINDEX; +import static org.junit.Assert.assertEquals; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import org.eclipse.jgit.internal.storage.file.PackIndex; +import org.eclipse.jgit.junit.FakeIndexFactory; +import org.eclipse.jgit.junit.FakeIndexFactory.IndexObject; +import org.eclipse.jgit.lib.NullProgressMonitor; +import org.eclipse.jgit.util.NB; +import org.junit.Test; + +public class MultiPackIndexWriterTest { + + @Test + public void write_allSmallOffsets() throws IOException { + PackIndex index1 = indexOf( + object("0000000000000000000000000000000000000001", 500), + object("0000000000000000000000000000000000000003", 1500), + object("0000000000000000000000000000000000000005", 3000)); + PackIndex index2 = indexOf( + object("0000000000000000000000000000000000000002", 500), + object("0000000000000000000000000000000000000004", 1500), + object("0000000000000000000000000000000000000006", 3000)); + + Map data = Map.of("packname1", index1, "packname2", + index2); + + MultiPackIndexWriter writer = new MultiPackIndexWriter(); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + writer.write(NullProgressMonitor.INSTANCE, out, data); + // header (12 bytes) + // + chunkHeader (6 * 12 bytes) + // + fanout table (256 * 4 bytes) + // + OIDs (6 * 20 bytes) + // + (pack, offset) pairs (6 * 8) + // + RIDX (6 * 4 bytes) + // + packfile names (2 * 10) + // + checksum (20) + assertEquals(1340, out.size()); + List chunkIds = readChunkIds(out); + assertEquals(5, chunkIds.size()); + assertEquals(0, chunkIds.indexOf(MIDX_CHUNKID_OIDFANOUT)); + assertEquals(1, chunkIds.indexOf(MIDX_CHUNKID_OIDLOOKUP)); + assertEquals(2, chunkIds.indexOf(MIDX_CHUNKID_OBJECTOFFSETS)); + assertEquals(3, chunkIds.indexOf(MIDX_CHUNKID_REVINDEX)); + assertEquals(4, chunkIds.indexOf(MIDX_CHUNKID_PACKNAMES)); + } + + @Test + public void write_smallOffset_limit() throws IOException { + PackIndex index1 = indexOf( + object("0000000000000000000000000000000000000001", 500), + object("0000000000000000000000000000000000000003", 1500), + object("0000000000000000000000000000000000000005", (1L << 32) -1)); + PackIndex index2 = indexOf( + object("0000000000000000000000000000000000000002", 500), + object("0000000000000000000000000000000000000004", 1500), + object("0000000000000000000000000000000000000006", 3000)); + Map data = + Map.of("packname1", index1, "packname2", index2); + + MultiPackIndexWriter writer = new MultiPackIndexWriter(); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + writer.write(NullProgressMonitor.INSTANCE, out, data); + // header (12 bytes) + // + chunkHeader (6 * 12 bytes) + // + fanout table (256 * 4 bytes) + // + OIDs (6 * 20 bytes) + // + (pack, offset) pairs (6 * 8) + // + RIDX (6 * 4 bytes) + // + packfile names (2 * 10) + // + checksum (20) + assertEquals(1340, out.size()); + List chunkIds = readChunkIds(out); + assertEquals(5, chunkIds.size()); + assertEquals(0, chunkIds.indexOf(MIDX_CHUNKID_OIDFANOUT)); + assertEquals(1, chunkIds.indexOf(MIDX_CHUNKID_OIDLOOKUP)); + assertEquals(2, chunkIds.indexOf(MIDX_CHUNKID_OBJECTOFFSETS)); + assertEquals(3, chunkIds.indexOf(MIDX_CHUNKID_REVINDEX)); + assertEquals(4, chunkIds.indexOf(MIDX_CHUNKID_PACKNAMES)); + } + + @Test + public void write_largeOffset() throws IOException { + PackIndex index1 = indexOf( + object("0000000000000000000000000000000000000001", 500), + object("0000000000000000000000000000000000000003", 1500), + object("0000000000000000000000000000000000000005", 1L << 32)); + PackIndex index2 = indexOf( + object("0000000000000000000000000000000000000002", 500), + object("0000000000000000000000000000000000000004", 1500), + object("0000000000000000000000000000000000000006", 3000)); + Map data = + Map.of("packname1", index1, "packname2", index2); + + MultiPackIndexWriter writer = new MultiPackIndexWriter(); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + writer.write(NullProgressMonitor.INSTANCE, out, data); + // header (12 bytes) + // + chunkHeader (7 * 12 bytes) + // + fanout table (256 * 4 bytes) + // + OIDs (6 * 20 bytes) + // + (pack, offset) pairs (6 * 8) + // + (large-offset) (1 * 8) + // + RIDX (6 * 4 bytes) + // + packfile names (2 * 10) + // + checksum (20) + assertEquals(1360, out.size()); + List chunkIds = readChunkIds(out); + assertEquals(6, chunkIds.size()); + assertEquals(0, chunkIds.indexOf(MIDX_CHUNKID_OIDFANOUT)); + assertEquals(1, chunkIds.indexOf(MIDX_CHUNKID_OIDLOOKUP)); + assertEquals(2, chunkIds.indexOf(MIDX_CHUNKID_OBJECTOFFSETS)); + assertEquals(3, chunkIds.indexOf(MIDX_CHUNKID_LARGEOFFSETS)); + assertEquals(4, chunkIds.indexOf(MIDX_CHUNKID_REVINDEX)); + assertEquals(5, chunkIds.indexOf(MIDX_CHUNKID_PACKNAMES)); + } + + private List readChunkIds(ByteArrayOutputStream out) { + List chunkIds = new ArrayList<>(); + byte[] raw = out.toByteArray(); + int numChunks = raw[6]; + int position = 12; + for (int i = 0; i < numChunks; i++) { + chunkIds.add(NB.decodeInt32(raw, position)); + position += CHUNK_LOOKUP_WIDTH; + } + return chunkIds; + } + + private static PackIndex indexOf(IndexObject... objs) { + return FakeIndexFactory.indexOf(Arrays.asList(objs)); + } + + private static IndexObject object(String name, long offset) { + return new IndexObject(name, offset); + } +} diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties index 7c47d3a04f..392f4e9954 100644 --- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties +++ b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties @@ -529,6 +529,8 @@ mkDirsFailed=Creating directories for {0} failed month=month months=months monthsAgo={0} months ago +multiPackIndexUnexpectedSize=MultiPack index: expected %d bytes but out has %d bytes +multiPackIndexWritingCancelled=Multipack index writing was canceled multipleMergeBasesFor=Multiple merge bases for:\n {0}\n {1} found:\n {2}\n {3} nameMustNotBeNullOrEmpty=Ref name must not be null or empty. need2Arguments=Need 2 arguments diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java index 07e882498c..d7fe456946 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java @@ -559,6 +559,8 @@ public class JGitText extends TranslationBundle { /***/ public String month; /***/ public String months; /***/ public String monthsAgo; + /***/ public String multiPackIndexUnexpectedSize; + /***/ public String multiPackIndexWritingCancelled; /***/ public String multipleMergeBasesFor; /***/ public String nameMustNotBeNullOrEmpty; /***/ public String need2Arguments; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexConstants.java new file mode 100644 index 0000000000..6122a9a143 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexConstants.java @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2025, Google Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.internal.storage.midx; + +class MultiPackIndexConstants { + static final int MIDX_SIGNATURE = 0x4d494458; /* MIDX */ + + static final byte MIDX_VERSION = 1; + + /** + * We infer the length of object IDs (OIDs) from this value: + * + * 

+	 * 1 => SHA-1
+	 * 2 => SHA-256
+	 *
+ */ + static final byte OID_HASH_VERSION = 1; + + static final int MULTIPACK_INDEX_FANOUT_SIZE = 4 * 256; + + /** + * First 4 bytes describe the chunk id. Value 0 is a terminating label. + * Other 8 bytes provide the byte-offset in current file for chunk to start. + */ + static final int CHUNK_LOOKUP_WIDTH = 12; + + /** "PNAM" chunk */ + static final int MIDX_CHUNKID_PACKNAMES = 0x504e414d; + + /** "OIDF" chunk */ + static final int MIDX_CHUNKID_OIDFANOUT = 0x4f494446; + + /** "OIDL" chunk */ + static final int MIDX_CHUNKID_OIDLOOKUP = 0x4f49444c; + + /** "OOFF" chunk */ + static final int MIDX_CHUNKID_OBJECTOFFSETS = 0x4f4f4646; + + /** "LOFF" chunk */ + static final int MIDX_CHUNKID_LARGEOFFSETS = 0x4c4f4646; + + /** "RIDX" chunk */ + static final int MIDX_CHUNKID_REVINDEX = 0x52494458; + + private MultiPackIndexConstants() { + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexWriter.java new file mode 100644 index 0000000000..28d2fb2334 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexWriter.java @@ -0,0 +1,422 @@ +/* + * Copyright (C) 2025, Google Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.internal.storage.midx; + +import static org.eclipse.jgit.internal.storage.midx.MultiPackIndexConstants.CHUNK_LOOKUP_WIDTH; +import static org.eclipse.jgit.internal.storage.midx.MultiPackIndexConstants.MIDX_CHUNKID_LARGEOFFSETS; +import static org.eclipse.jgit.internal.storage.midx.MultiPackIndexConstants.MIDX_CHUNKID_OBJECTOFFSETS; +import static org.eclipse.jgit.internal.storage.midx.MultiPackIndexConstants.MIDX_CHUNKID_OIDFANOUT; +import static org.eclipse.jgit.internal.storage.midx.MultiPackIndexConstants.MIDX_CHUNKID_OIDLOOKUP; +import static org.eclipse.jgit.internal.storage.midx.MultiPackIndexConstants.MIDX_CHUNKID_PACKNAMES; +import static org.eclipse.jgit.internal.storage.midx.MultiPackIndexConstants.MIDX_CHUNKID_REVINDEX; +import static org.eclipse.jgit.internal.storage.midx.MultiPackIndexConstants.MIDX_SIGNATURE; +import static org.eclipse.jgit.internal.storage.midx.MultiPackIndexConstants.MIDX_VERSION; +import static org.eclipse.jgit.internal.storage.midx.MultiPackIndexConstants.MULTIPACK_INDEX_FANOUT_SIZE; +import static org.eclipse.jgit.internal.storage.midx.MultiPackIndexConstants.OID_HASH_VERSION; +import static org.eclipse.jgit.lib.Constants.OBJECT_ID_LENGTH; + +import java.io.IOException; +import java.io.InterruptedIOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.internal.storage.file.PackIndex; +import org.eclipse.jgit.internal.storage.io.CancellableDigestOutputStream; +import org.eclipse.jgit.internal.storage.midx.PackIndexMerger.MidxMutableEntry; +import org.eclipse.jgit.lib.ProgressMonitor; +import org.eclipse.jgit.util.NB; + +/** + * Writes a collection of indexes as a multipack index. + *

+ * See multipack + * index format spec + */ +public class MultiPackIndexWriter { + + private static final int LIMIT_31_BITS = (1 << 31) - 1; + + private static final int MIDX_HEADER_SIZE = 12; + + /** + * Writes the inputs in the multipack index format in the outputStream. + * + * @param monitor + * progress monitor + * @param outputStream + * stream to write the multipack index file + * @param inputs + * pairs of name and index for each pack to include in the + * multipack index. + * @throws IOException + * Error writing to the stream + */ + public void write(ProgressMonitor monitor, OutputStream outputStream, + Map inputs) throws IOException { + PackIndexMerger data = new PackIndexMerger(inputs); + + // List of chunks in the order they need to be written + List chunkHeaders = createChunkHeaders(data); + long expectedSize = calculateExpectedSize(chunkHeaders); + try (CancellableDigestOutputStream out = new CancellableDigestOutputStream( + monitor, outputStream)) { + writeHeader(out, chunkHeaders.size(), data.getPackCount()); + writeChunkLookup(out, chunkHeaders); + + WriteContext ctx = new WriteContext(out, data); + for (ChunkHeader chunk : chunkHeaders) { + chunk.writerFn.write(ctx); + } + writeCheckSum(out); + if (expectedSize != out.length()) { + throw new IllegalStateException(String.format( + JGitText.get().multiPackIndexUnexpectedSize, + expectedSize, out.length())); + } + } catch (InterruptedIOException e) { + throw new IOException(JGitText.get().multiPackIndexWritingCancelled, + e); + } + } + + private static long calculateExpectedSize(List chunks) { + int chunkLookup = (chunks.size() + 1) * CHUNK_LOOKUP_WIDTH; + long chunkContent = chunks.stream().mapToLong(c -> c.size).sum(); + return /* header */ 12 + chunkLookup + chunkContent + /* CRC */ 20; + } + + private List createChunkHeaders(PackIndexMerger data) { + List chunkHeaders = new ArrayList<>(); + chunkHeaders.add(new ChunkHeader(MIDX_CHUNKID_OIDFANOUT, + MULTIPACK_INDEX_FANOUT_SIZE, this::writeFanoutTable)); + chunkHeaders.add(new ChunkHeader(MIDX_CHUNKID_OIDLOOKUP, + (long) data.getUniqueObjectCount() * OBJECT_ID_LENGTH, + this::writeOidLookUp)); + chunkHeaders.add(new ChunkHeader(MIDX_CHUNKID_OBJECTOFFSETS, + 8L * data.getUniqueObjectCount(), this::writeObjectOffsets)); + if (data.needsLargeOffsetsChunk()) { + chunkHeaders.add(new ChunkHeader(MIDX_CHUNKID_LARGEOFFSETS, + 8L * data.getOffsetsOver31BitsCount(), + this::writeObjectLargeOffsets)); + } + chunkHeaders.add(new ChunkHeader(MIDX_CHUNKID_REVINDEX, + 4L * data.getUniqueObjectCount(), this::writeRidx)); + + int packNamesSize = data.getPackNames().stream() + .mapToInt(String::length).map(i -> i + 1 /* null at the end */) + .sum(); + chunkHeaders.add(new ChunkHeader(MIDX_CHUNKID_PACKNAMES, packNamesSize, + this::writePackfileNames)); + return chunkHeaders; + } + + /** + * Write the first 12 bytes of the multipack index. + *

+ * These bytes include things like magic number, version, number of + * chunks... + * + * @param out + * output stream to write + * @param numChunks + * number of chunks this multipack index is going to have + * @param packCount + * number of packs covered by this multipack index + * @throws IOException + * error writing to the output stream + */ + private void writeHeader(CancellableDigestOutputStream out, int numChunks, + int packCount) throws IOException { + byte[] headerBuffer = new byte[MIDX_HEADER_SIZE]; + NB.encodeInt32(headerBuffer, 0, MIDX_SIGNATURE); + byte[] buff = { MIDX_VERSION, OID_HASH_VERSION, (byte) numChunks, + (byte) 0 }; + System.arraycopy(buff, 0, headerBuffer, 4, 4); + NB.encodeInt32(headerBuffer, 8, packCount); + out.write(headerBuffer, 0, headerBuffer.length); + out.flush(); + } + + /** + * Write a table of "chunkId, start-offset", with a special value "0, + * end-of-previous_chunk", to mark the end. + * + * @param out + * output stream to write + * @param chunkHeaders + * list of chunks in the order they are expected to be written + * @throws IOException + * error writing to the output stream + */ + private void writeChunkLookup(CancellableDigestOutputStream out, + List chunkHeaders) throws IOException { + + // first chunk will start at header + this lookup block + long chunkStart = MIDX_HEADER_SIZE + + (long) (chunkHeaders.size() + 1) * CHUNK_LOOKUP_WIDTH; + byte[] chunkEntry = new byte[CHUNK_LOOKUP_WIDTH]; + for (ChunkHeader chunkHeader : chunkHeaders) { + NB.encodeInt32(chunkEntry, 0, chunkHeader.chunkId); + NB.encodeInt64(chunkEntry, 4, chunkStart); + out.write(chunkEntry); + chunkStart += chunkHeader.size; + } + // Terminating label for the block + // (chunkid 0, offset where the next block would start) + NB.encodeInt32(chunkEntry, 0, 0); + NB.encodeInt64(chunkEntry, 4, chunkStart); + out.write(chunkEntry); + } + + /** + * Write the fanout table for the object ids + *

+ * Table with 256 entries (one byte), where the ith entry, F[i], stores the + * number of OIDs with first byte at most i. Thus, F[255] stores the total + * number of objects. + * + * @param ctx + * write context + * @throws IOException + * error writing to the output stream + */ + + private void writeFanoutTable(WriteContext ctx) throws IOException { + byte[] tmp = new byte[4]; + int[] fanout = new int[256]; + Iterator iterator = ctx.data.bySha1Iterator(); + while (iterator.hasNext()) { + MidxMutableEntry e = iterator.next(); + fanout[e.getObjectId().getFirstByte() & 0xff]++; + } + for (int i = 1; i < fanout.length; i++) { + fanout[i] += fanout[i - 1]; + } + for (int n : fanout) { + NB.encodeInt32(tmp, 0, n); + ctx.out.write(tmp, 0, 4); + } + } + + /** + * Write the OID lookup chunk + *

+ * A list of OIDs in sha1 order. + * + * @param ctx + * write context + * @throws IOException + * error writing to the output stream + */ + private void writeOidLookUp(WriteContext ctx) throws IOException { + byte[] tmp = new byte[OBJECT_ID_LENGTH]; + + Iterator iterator = ctx.data.bySha1Iterator(); + while (iterator.hasNext()) { + MidxMutableEntry e = iterator.next(); + e.getObjectId().copyRawTo(tmp, 0); + ctx.out.write(tmp, 0, OBJECT_ID_LENGTH); + } + } + + /** + * Write the object offsets chunk + *

+ * A list of offsets, parallel to the list of OIDs. If the offset is too + * large (see {@link #fitsIn31bits(long)}), this contains the position in + * the large offsets list (marked with a 1 in the most significant bit). + * + * @param ctx + * write context + * @throws IOException + * error writing to the output stream + */ + private void writeObjectOffsets(WriteContext ctx) throws IOException { + byte[] entry = new byte[8]; + Iterator iterator = ctx.data.bySha1Iterator(); + while (iterator.hasNext()) { + MidxMutableEntry e = iterator.next(); + NB.encodeInt32(entry, 0, e.getPackId()); + if (!ctx.data.needsLargeOffsetsChunk() + || fitsIn31bits(e.getOffset())) { + NB.encodeInt32(entry, 4, (int) e.getOffset()); + } else { + int offloadedPosition = ctx.largeOffsets.append(e.getOffset()); + NB.encodeInt32(entry, 4, offloadedPosition | (1 << 31)); + } + ctx.out.write(entry); + } + } + + /** + * Writes the reverse index chunk + *

+ * This stores the position of the objects in the main index, ordered first + * by pack and then by offset + * + * @param ctx + * write context + * @throws IOException + * erorr writing to the output stream + */ + private void writeRidx(WriteContext ctx) throws IOException { + Map> packOffsets = new HashMap<>( + ctx.data.getPackCount()); + // TODO(ifrade): Brute force solution loading all offsets/packs in + // memory. We could also iterate reverse indexes looking up + // their position in the midx (and discarding if the pack doesn't + // match). + Iterator iterator = ctx.data.bySha1Iterator(); + int midxPosition = 0; + while (iterator.hasNext()) { + MidxMutableEntry e = iterator.next(); + OffsetPosition op = new OffsetPosition(e.getOffset(), midxPosition); + midxPosition++; + packOffsets.computeIfAbsent(e.getPackId(), k -> new ArrayList<>()) + .add(op); + } + + for (int i = 0; i < ctx.data.getPackCount(); i++) { + List offsetsForPack = packOffsets.get(i); + if (offsetsForPack.isEmpty()) { + continue; + } + offsetsForPack.sort(Comparator.comparing(OffsetPosition::offset)); + byte[] ridxForPack = new byte[4 * offsetsForPack.size()]; + for (int j = 0; j < offsetsForPack.size(); j++) { + NB.encodeInt32(ridxForPack, j * 4, + offsetsForPack.get(j).position); + } + ctx.out.write(ridxForPack); + } + } + + /** + * Write the large offset chunk + *

+ * A list of large offsets (long). The regular offset chunk will point to a + * position here. + * + * @param ctx + * writer context + * @throws IOException + * error writing to the output stream + */ + private void writeObjectLargeOffsets(WriteContext ctx) throws IOException { + ctx.out.write(ctx.largeOffsets.offsets, 0, + ctx.largeOffsets.bytePosition); + } + + /** + * Write the list of packfiles chunk + *

+ * List of packfiles (in lexicographical order) with an \0 at the end + * + * @param ctx + * writer context + * @throws IOException + * error writing to the output stream + */ + private void writePackfileNames(WriteContext ctx) throws IOException { + for (String packName : ctx.data.getPackNames()) { + // Spec doesn't talk about encoding. + ctx.out.write(packName.getBytes(StandardCharsets.UTF_8)); + ctx.out.write(0); + } + } + + /** + * Write final checksum of the data written to the stream + * + * @param out + * output stream used to write + * @throws IOException + * error writing to the output stream + */ + private void writeCheckSum(CancellableDigestOutputStream out) + throws IOException { + out.write(out.getDigest()); + out.flush(); + } + + + private record OffsetPosition(long offset, int position) { + } + + /** + * If there is at least one offset value larger than 2^32-1, then the large + * offset chunk must exist, and offsets larger than 2^31-1 must be stored in + * it instead + * + * @param offset object offset + * + * @return true if the offset fits in 31 bits + */ + private static boolean fitsIn31bits(long offset) { + return offset <= LIMIT_31_BITS; + } + + private static class LargeOffsets { + private final byte[] offsets; + + private int bytePosition; + + LargeOffsets(int largeOffsetsCount) { + offsets = new byte[largeOffsetsCount * 8]; + bytePosition = 0; + } + + /** + * Add an offset to the large offset chunk + * + * @param largeOffset + * a large offset + * @return the position of the just inserted offset (as in number of + * offsets, NOT in bytes) + */ + int append(long largeOffset) { + int at = bytePosition; + NB.encodeInt64(offsets, at, largeOffset); + bytePosition += 8; + return at / 8; + } + } + + private record ChunkHeader(int chunkId, long size, ChunkWriter writerFn) { + } + + @FunctionalInterface + private interface ChunkWriter { + void write(WriteContext ctx) throws IOException; + } + + private static class WriteContext { + final CancellableDigestOutputStream out; + + final PackIndexMerger data; + + final LargeOffsets largeOffsets; + + WriteContext(CancellableDigestOutputStream out, PackIndexMerger data) { + this.out = out; + this.data = data; + this.largeOffsets = new LargeOffsets( + data.getOffsetsOver31BitsCount()); + } + } +} -- cgit v1.2.3 From e9f43b6c1f9069a0a66c8a513b130c4df421dd6e Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Thu, 23 Jan 2025 16:03:04 -0800 Subject: midx.MultiPackIndexPrettyPrinter: pretty printer to debug multi pack index Prints the multipack index file in a human readable format. This helps to debug/inspect multipack indexes generated by jgit or git. Change-Id: I04f477b3763b0ecfde6f4379f267de8a609a54e7 --- .../storage/midx/MultiPackIndexPrettyPrinter.java | 154 +++++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexPrettyPrinter.java (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexPrettyPrinter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexPrettyPrinter.java new file mode 100644 index 0000000000..e8428b8fca --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexPrettyPrinter.java @@ -0,0 +1,154 @@ +/* + * Copyright (C) 2025, Google Inc. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.internal.storage.midx; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.eclipse.jgit.internal.storage.midx.MultiPackIndexConstants.CHUNK_LOOKUP_WIDTH; + +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.List; + +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.util.NB; + +/** + * Prints a multipack index file in a human-readable format. + */ +@SuppressWarnings("nls") +public class MultiPackIndexPrettyPrinter { + + /** + * Writes to out, in human-readable format, the multipack index in rawMidx + * + * @param rawMidx the bytes of a multipack index + * @param out a writer + */ + public static void prettyPrint(byte[] rawMidx, PrintWriter out) { + // Header (12 bytes) + out.println("[ 0] Magic: " + new String(rawMidx, 0, 4, UTF_8)); + out.println("[ 4] Version number: " + (int) rawMidx[4]); + out.println("[ 5] OID version: " + (int) rawMidx[5]); + int chunkCount = rawMidx[6]; + out.println("[ 6] # of chunks: " + chunkCount); + out.println("[ 7] # of bases: " + (int) rawMidx[7]); + int numberOfPacks = NB.decodeInt32(rawMidx, 8); + out.println("[ 8] # of packs: " + numberOfPacks); + + // Chunk lookup table + List chunkSegments = new ArrayList<>(); + int current = printChunkLookup(out, rawMidx, chunkCount, chunkSegments); + + for (int i = 0; i < chunkSegments.size() - 1; i++) { + ChunkSegment segment = chunkSegments.get(i); + if (current != segment.startOffset()) { + throw new IllegalStateException(String.format( + "We are at byte %d, but segment should start at %d", + current, segment.startOffset())); + } + out.printf("Starting chunk: %s @ %d%n", segment.chunkName(), + segment.startOffset()); + switch (segment.chunkName()) { + case "OIDF" -> current = printOIDF(out, rawMidx, current); + case "OIDL" -> current = printOIDL(out, rawMidx, current, + chunkSegments.get(i + 1).startOffset); + case "OOFF" -> current = printOOFF(out, rawMidx, current, + chunkSegments.get(i + 1).startOffset); + case "PNAM" -> current = printPNAM(out, rawMidx, current, + chunkSegments.get(i + 1).startOffset); + case "RIDX" -> current = printRIDX(out, rawMidx, current, + chunkSegments.get(i + 1).startOffset); + default -> { + out.printf( + "Skipping %s (don't know how to print it yet)%n", + segment.chunkName()); + current = (int) chunkSegments.get(i + 1).startOffset(); + } + } + } + // Checksum is a SHA-1, use ObjectId to parse it + out.printf("[ %d] Checksum %s%n", current, + ObjectId.fromRaw(rawMidx, current).name()); + out.printf("Total size: " + (current + 20)); + } + + private static int printChunkLookup(PrintWriter out, byte[] rawMidx, int chunkCount, + List chunkSegments) { + out.println("Starting chunk lookup @ 12"); + int current = 12; + for (int i = 0; i < chunkCount; i++) { + String chunkName = new String(rawMidx, current, 4, UTF_8); + long offset = NB.decodeInt64(rawMidx, current + 4); + out.printf("[ %d] |%8s|%8d|%n", current, chunkName, offset); + current += CHUNK_LOOKUP_WIDTH; + chunkSegments.add(new ChunkSegment(chunkName, offset)); + } + String chunkName = "0000"; + long offset = NB.decodeInt64(rawMidx, current + 4); + out.printf("[ %d] |%8s|%8d|%n", current, chunkName, offset); + current += CHUNK_LOOKUP_WIDTH; + chunkSegments.add(new ChunkSegment(chunkName, offset)); + return current; + } + + private static int printOIDF(PrintWriter out, byte[] rawMidx, int start) { + int current = start; + for (short i = 0; i < 256; i++) { + out.printf("[ %d] (%02X) %d%n", current, i, + NB.decodeInt32(rawMidx, current)); + current += 4; + } + return current; + } + + private static int printOIDL(PrintWriter out, byte[] rawMidx, int start, long end) { + int i = start; + while (i < end) { + out.printf("[ %d] %s%n", i, + ObjectId.fromRaw(rawMidx, i).name()); + i += 20; + } + return i; + } + + private static int printOOFF(PrintWriter out, byte[] rawMidx, int start, long end) { + int i = start; + while (i < end) { + out.printf("[ %d] %d %d%n", i, NB.decodeInt32(rawMidx, i), + NB.decodeInt32(rawMidx, i + 4)); + i += 8; + } + return i; + } + + private static int printRIDX(PrintWriter out, byte[] rawMidx, int start, long end) { + int i = start; + while (i < end) { + out.printf("[ %d] %d%n", i, NB.decodeInt32(rawMidx, i)); + i += 4; + } + return (int) end; + } + + private static int printPNAM(PrintWriter out, byte[] rawMidx, int start, long end) { + int nameStart = start; + for (int i = start; i < end; i++) { + if (rawMidx[i] == 0) { + out + .println(new String(rawMidx, nameStart, i - nameStart)); + nameStart = i + 1; + } + } + return (int) end; + } + + private record ChunkSegment(String chunkName, long startOffset) { + } +} -- cgit v1.2.3 From 097b158c3b03c275ccef27bdb35c91ccf8372a02 Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Tue, 11 Feb 2025 09:57:55 -0800 Subject: DefaultTypedConfigGetter: Box values to avoid infinite recursion Errorprone says: DefaultTypedConfigGetter.java:176: error: [InfiniteRecursion] This method always recurses, and will cause a StackOverflowError return getLong(config, section, subsection, name, defaultValue); [1] introduced new getters with boxed types to return a null when the config is not set. The getters of unboxed types should call to the boxed version, but, as the values are not explicitely boxed, they are calling to themselves. [1] https://gerrithub.io/c/eclipse-jgit/jgit/+/1207895 Change-Id: Ied45a199c8ef905e3774a17a04d91a656aa0e42b --- .../eclipse/jgit/lib/DefaultTypedConfigGetter.java | 40 +++++++++++++++------- 1 file changed, 27 insertions(+), 13 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/DefaultTypedConfigGetter.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/DefaultTypedConfigGetter.java index 65093987e8..3059f283fe 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/DefaultTypedConfigGetter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/DefaultTypedConfigGetter.java @@ -32,10 +32,12 @@ import org.eclipse.jgit.util.StringUtils; */ public class DefaultTypedConfigGetter implements TypedConfigGetter { + @SuppressWarnings("boxed") @Override public boolean getBoolean(Config config, String section, String subsection, String name, boolean defaultValue) { - return getBoolean(config, section, subsection, name, defaultValue); + return neverNull(getBoolean(config, section, subsection, name, + Boolean.valueOf(defaultValue))); } @Nullable @@ -116,7 +118,8 @@ public class DefaultTypedConfigGetter implements TypedConfigGetter { @Override public int getInt(Config config, String section, String subsection, String name, int defaultValue) { - return getInt(config, section, subsection, name, defaultValue); + return neverNull(getInt(config, section, subsection, name, + Integer.valueOf(defaultValue))); } @Nullable @@ -144,8 +147,8 @@ public class DefaultTypedConfigGetter implements TypedConfigGetter { @Override public int getIntInRange(Config config, String section, String subsection, String name, int minValue, int maxValue, int defaultValue) { - return getIntInRange(config, section, subsection, name, minValue, - maxValue, defaultValue); + return neverNull(getIntInRange(config, section, subsection, name, + minValue, maxValue, Integer.valueOf(defaultValue))); } @Override @@ -161,9 +164,9 @@ public class DefaultTypedConfigGetter implements TypedConfigGetter { return val; } if (subsection == null) { - throw new IllegalArgumentException(MessageFormat.format( - JGitText.get().integerValueNotInRange, section, name, - val, minValue, maxValue)); + throw new IllegalArgumentException( + MessageFormat.format(JGitText.get().integerValueNotInRange, + section, name, val, minValue, maxValue)); } throw new IllegalArgumentException(MessageFormat.format( JGitText.get().integerValueNotInRangeSubSection, section, @@ -173,7 +176,8 @@ public class DefaultTypedConfigGetter implements TypedConfigGetter { @Override public long getLong(Config config, String section, String subsection, String name, long defaultValue) { - return getLong(config, section, subsection, name, defaultValue); + return neverNull(getLong(config, section, subsection, name, + Long.valueOf(defaultValue))); } @Nullable @@ -190,8 +194,9 @@ public class DefaultTypedConfigGetter implements TypedConfigGetter { // Empty return defaultValue; } catch (NumberFormatException nfe) { - throw new IllegalArgumentException(MessageFormat.format( - JGitText.get().invalidIntegerValue, section, name, str), + throw new IllegalArgumentException( + MessageFormat.format(JGitText.get().invalidIntegerValue, + section, name, str), nfe); } } @@ -199,9 +204,8 @@ public class DefaultTypedConfigGetter implements TypedConfigGetter { @Override public long getTimeUnit(Config config, String section, String subsection, String name, long defaultValue, TimeUnit wantUnit) { - Long v = getTimeUnit(config, section, subsection, name, - Long.valueOf(defaultValue), wantUnit); - return v == null ? defaultValue : v.longValue(); + return neverNull(getTimeUnit(config, section, subsection, name, + Long.valueOf(defaultValue), wantUnit)); } @Override @@ -325,4 +329,14 @@ public class DefaultTypedConfigGetter implements TypedConfigGetter { } return result; } + + // Trick for the checkers. When we use this, one is never null, but + // they don't know. + @NonNull + private static T neverNull(T one) { + if (one == null) { + throw new IllegalArgumentException(); + } + return one; + } } -- cgit v1.2.3 From 5320df833b91eb82f82bdafa9b48397c1012c400 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 14 Feb 2025 02:10:19 +0100 Subject: MultiPackIndexWriter: add missing @since for new API class and fix boxing warnings. Change-Id: Ia9b4deba7892256639c53bac5d7b62f1fbb01389 --- .../jgit/internal/storage/midx/MultiPackIndexWriter.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexWriter.java index 28d2fb2334..bddf3ac4ad 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexWriter.java @@ -46,6 +46,8 @@ import org.eclipse.jgit.util.NB; * See multipack * index format spec + * + * @since 7.2 */ public class MultiPackIndexWriter { @@ -86,7 +88,8 @@ public class MultiPackIndexWriter { if (expectedSize != out.length()) { throw new IllegalStateException(String.format( JGitText.get().multiPackIndexUnexpectedSize, - expectedSize, out.length())); + Long.valueOf(expectedSize), + Long.valueOf(out.length()))); } } catch (InterruptedIOException e) { throw new IOException(JGitText.get().multiPackIndexWritingCancelled, @@ -287,12 +290,13 @@ public class MultiPackIndexWriter { MidxMutableEntry e = iterator.next(); OffsetPosition op = new OffsetPosition(e.getOffset(), midxPosition); midxPosition++; - packOffsets.computeIfAbsent(e.getPackId(), k -> new ArrayList<>()) - .add(op); + packOffsets.computeIfAbsent(Integer.valueOf(e.getPackId()), + k -> new ArrayList<>()).add(op); } for (int i = 0; i < ctx.data.getPackCount(); i++) { - List offsetsForPack = packOffsets.get(i); + List offsetsForPack = packOffsets + .get(Integer.valueOf(i)); if (offsetsForPack.isEmpty()) { continue; } -- cgit v1.2.3 From 5dae4a8351287526b323547f942da72b9e00f456 Mon Sep 17 00:00:00 2001 From: Martin Fick Date: Wed, 12 Feb 2025 19:07:42 -0800 Subject: Pack: no longer set invalid in openFail() The intention of the 'invalidate' argument in openFail() is to invalidate the Pack in certain situations. However, after moving doOpen() to a lock instead of using synchronized, the invalidation approach could also incorrectly mark an already invalid Pack valid, which was never the intention since previously invalid would only ever get set to false if it already was false. Fix this by never setting invalid in openFail(), instead set invalid explicitly before calling openFail when needed. This makes the intent clearer, and aligns better with all the existing comments already trying to explain the boolean (and some of them become obvious enough now that the comment is deleted or shortened). This is also likely faster than adding a conditional in openFail() to make 'invalidate' work properly. Change-Id: Ie6182103ee2994724cb5cb0b64030fedba84b637 Signed-off-by: Martin Fick --- .../eclipse/jgit/internal/storage/file/Pack.java | 30 ++++++++++++---------- 1 file changed, 16 insertions(+), 14 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java index 1a7b5de1d7..122d800582 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java @@ -694,7 +694,7 @@ public class Pack implements Iterable { private void doOpen() throws IOException { if (invalid) { - openFail(true, invalidatingCause); + openFail(invalidatingCause); throw new PackInvalidException(packFile, invalidatingCause); } try { @@ -705,39 +705,41 @@ public class Pack implements Iterable { } } catch (InterruptedIOException e) { // don't invalidate the pack, we are interrupted from another thread - openFail(false, e); + openFail(e); throw e; } catch (FileNotFoundException fn) { - // don't invalidate the pack if opening an existing file failed - // since it may be related to a temporary lack of resources (e.g. - // max open files) - openFail(!packFile.exists(), fn); + if (!packFile.exists()) { + // Failure to open an existing file may be related to a temporary lack of resources + // (e.g. max open files) + invalid = true; + } + openFail(fn); throw fn; } catch (EOFException | AccessDeniedException | NoSuchFileException | CorruptObjectException | NoPackSignatureException | PackMismatchException | UnpackException | UnsupportedPackIndexVersionException | UnsupportedPackVersionException pe) { - // exceptions signaling permanent problems with a pack - openFail(true, pe); + invalid = true; // exceptions signaling permanent problems with a pack + openFail(pe); throw pe; } catch (IOException ioe) { - // mark this packfile as invalid when NFS stale file handle error - // occur - openFail(FileUtils.isStaleFileHandleInCausalChain(ioe), ioe); + if (FileUtils.isStaleFileHandleInCausalChain(ioe)) { + invalid = true; + } + openFail(ioe); throw ioe; } catch (RuntimeException ge) { // generic exceptions could be transient so we should not mark the // pack invalid to avoid false MissingObjectExceptions - openFail(false, ge); + openFail(ge); throw ge; } } - private void openFail(boolean invalidate, Exception cause) { + private void openFail(Exception cause) { activeWindows = 0; activeCopyRawData = 0; - invalid = invalidate; invalidatingCause = cause; doClose(); } -- cgit v1.2.3 From c625d67f8d7c25b5ce728ecfcc7c894d0ad03938 Mon Sep 17 00:00:00 2001 From: Thomas Wolf Date: Sun, 16 Feb 2025 11:50:44 +0100 Subject: URIish: fix stack overflow in regex matching The regex for a relative path used greedy matches, which could cause excessive backtracking. Simplify the regex and use a possessive quantifier to avoid backtracking at all: a relative path is a sequence (AB)*A?, where A and B are disjunct: once (AB)* has been matched, there is no need for any backtracking in the relative path. Bug: egit-80 Change-Id: Ic7865f20767d85ec1db2d0b92adcd548099eb643 --- .../tst/org/eclipse/jgit/transport/URIishTest.java | 37 ++++++++++++++++++++++ .../src/org/eclipse/jgit/transport/URIish.java | 4 +-- 2 files changed, 39 insertions(+), 2 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/URIishTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/URIishTest.java index d403624b71..67920029d4 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/URIishTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/transport/URIishTest.java @@ -81,6 +81,43 @@ public class URIishTest { assertEquals(u, new URIish(str)); } + @Test + public void testBrokenFilePath() throws Exception { + String str = "D:\\\\my\\\\x"; + URIish u = new URIish(str); + assertNull(u.getScheme()); + assertFalse(u.isRemote()); + assertEquals(str, u.getPath()); + assertEquals(u, new URIish(str)); + } + + @Test + public void testStackOverflow() throws Exception { + StringBuilder b = new StringBuilder("D:\\"); + for (int i = 0; i < 4000; i++) { + b.append("x\\"); + } + String str = b.toString(); + URIish u = new URIish(str); + assertNull(u.getScheme()); + assertFalse(u.isRemote()); + assertEquals(str, u.getPath()); + } + + @Test + public void testStackOverflow2() throws Exception { + StringBuilder b = new StringBuilder("D:\\"); + for (int i = 0; i < 4000; i++) { + b.append("x\\"); + } + b.append('y'); + String str = b.toString(); + URIish u = new URIish(str); + assertNull(u.getScheme()); + assertFalse(u.isRemote()); + assertEquals(str, u.getPath()); + } + @Test public void testRelativePath() throws Exception { final String str = "../../foo/bar"; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/URIish.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/URIish.java index 4de6ff825f..7b5842b712 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/URIish.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/URIish.java @@ -82,7 +82,7 @@ public class URIish implements Serializable { * Part of a pattern which matches a relative path. Relative paths don't * start with slash or drive letters. Defines no capturing group. */ - private static final String RELATIVE_PATH_P = "(?:(?:[^\\\\/]+[\\\\/]+)*[^\\\\/]+[\\\\/]*)"; //$NON-NLS-1$ + private static final String RELATIVE_PATH_P = "(?:(?:[^\\\\/]+[\\\\/]+)*+[^\\\\/]*)"; //$NON-NLS-1$ /** * Part of a pattern which matches a relative or absolute path. Defines no @@ -120,7 +120,7 @@ public class URIish implements Serializable { * path (maybe even containing windows drive-letters) or a relative path. */ private static final Pattern LOCAL_FILE = Pattern.compile("^" // //$NON-NLS-1$ - + "([\\\\/]?" + PATH_P + ")" // //$NON-NLS-1$ //$NON-NLS-2$ + + "([\\\\/]?+" + PATH_P + ")" // //$NON-NLS-1$ //$NON-NLS-2$ + "$"); //$NON-NLS-1$ /** -- cgit v1.2.3 From 24238a35739971cbd86c852f13a2f73f2cad3246 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Tue, 11 Feb 2025 00:56:23 +0100 Subject: Replace usage of deprecated Config#getEnum method Need to make DirCacheVersions public otherwise Config#allValuesOf cannot invoke its #values method via introspection. Change-Id: Id11a6fdbe7ce3d84f04bf47e98746424dcc761b4 --- .../tst/org/eclipse/jgit/api/CloneCommandTest.java | 4 +--- .../eclipse/jgit/api/RenameBranchCommandTest.java | 24 +++++++++------------- .../src/org/eclipse/jgit/api/FetchCommand.java | 4 ++-- .../src/org/eclipse/jgit/api/PullCommand.java | 6 +++--- .../src/org/eclipse/jgit/dircache/DirCache.java | 22 ++++++++++++++++---- .../src/org/eclipse/jgit/lib/BranchConfig.java | 3 +-- .../src/org/eclipse/jgit/lib/GpgConfig.java | 3 +-- .../org/eclipse/jgit/submodule/SubmoduleWalk.java | 8 ++++---- .../src/org/eclipse/jgit/transport/HttpConfig.java | 7 +++---- 9 files changed, 43 insertions(+), 38 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CloneCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CloneCommandTest.java index f5fd73ba38..661878fa07 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CloneCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/CloneCommandTest.java @@ -797,7 +797,7 @@ public class CloneCommandTest extends RepositoryTestCase { assertNull(git2.getRepository().getConfig().getEnum( BranchRebaseMode.values(), ConfigConstants.CONFIG_BRANCH_SECTION, "test", - ConfigConstants.CONFIG_KEY_REBASE, null)); + ConfigConstants.CONFIG_KEY_REBASE)); StoredConfig userConfig = SystemReader.getInstance() .getUserConfig(); @@ -813,7 +813,6 @@ public class CloneCommandTest extends RepositoryTestCase { addRepoToClose(git2.getRepository()); assertEquals(BranchRebaseMode.REBASE, git2.getRepository().getConfig().getEnum( - BranchRebaseMode.values(), ConfigConstants.CONFIG_BRANCH_SECTION, "test", ConfigConstants.CONFIG_KEY_REBASE, BranchRebaseMode.NONE)); @@ -830,7 +829,6 @@ public class CloneCommandTest extends RepositoryTestCase { addRepoToClose(git2.getRepository()); assertEquals(BranchRebaseMode.REBASE, git2.getRepository().getConfig().getEnum( - BranchRebaseMode.values(), ConfigConstants.CONFIG_BRANCH_SECTION, "test", ConfigConstants.CONFIG_KEY_REBASE, BranchRebaseMode.NONE)); diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/RenameBranchCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/RenameBranchCommandTest.java index 534ebd9c61..add5886c2d 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/RenameBranchCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/RenameBranchCommandTest.java @@ -118,23 +118,21 @@ public class RenameBranchCommandTest extends RepositoryTestCase { String branch = "b1"; assertEquals(BranchRebaseMode.REBASE, - config.getEnum(BranchRebaseMode.values(), - ConfigConstants.CONFIG_BRANCH_SECTION, Constants.MASTER, - ConfigConstants.CONFIG_KEY_REBASE, + config.getEnum(ConfigConstants.CONFIG_BRANCH_SECTION, + Constants.MASTER, ConfigConstants.CONFIG_KEY_REBASE, BranchRebaseMode.NONE)); assertNull(config.getEnum(BranchRebaseMode.values(), ConfigConstants.CONFIG_BRANCH_SECTION, branch, - ConfigConstants.CONFIG_KEY_REBASE, null)); + ConfigConstants.CONFIG_KEY_REBASE)); assertNotNull(git.branchRename().setNewName(branch).call()); config = git.getRepository().getConfig(); assertNull(config.getEnum(BranchRebaseMode.values(), ConfigConstants.CONFIG_BRANCH_SECTION, Constants.MASTER, - ConfigConstants.CONFIG_KEY_REBASE, null)); + ConfigConstants.CONFIG_KEY_REBASE)); assertEquals(BranchRebaseMode.REBASE, - config.getEnum(BranchRebaseMode.values(), - ConfigConstants.CONFIG_BRANCH_SECTION, branch, + config.getEnum(ConfigConstants.CONFIG_BRANCH_SECTION, branch, ConfigConstants.CONFIG_KEY_REBASE, BranchRebaseMode.NONE)); } @@ -170,13 +168,12 @@ public class RenameBranchCommandTest extends RepositoryTestCase { String branch = "b1"; assertEquals(BranchRebaseMode.REBASE, - config.getEnum(BranchRebaseMode.values(), - ConfigConstants.CONFIG_BRANCH_SECTION, Constants.MASTER, - ConfigConstants.CONFIG_KEY_REBASE, + config.getEnum(ConfigConstants.CONFIG_BRANCH_SECTION, + Constants.MASTER, ConfigConstants.CONFIG_KEY_REBASE, BranchRebaseMode.NONE)); assertNull(config.getEnum(BranchRebaseMode.values(), ConfigConstants.CONFIG_BRANCH_SECTION, branch, - ConfigConstants.CONFIG_KEY_REBASE, null)); + ConfigConstants.CONFIG_KEY_REBASE)); assertTrue(config.getBoolean(ConfigConstants.CONFIG_BRANCH_SECTION, Constants.MASTER, ConfigConstants.CONFIG_KEY_MERGE, true)); assertFalse(config.getBoolean(ConfigConstants.CONFIG_BRANCH_SECTION, @@ -187,10 +184,9 @@ public class RenameBranchCommandTest extends RepositoryTestCase { config = git.getRepository().getConfig(); assertNull(config.getEnum(BranchRebaseMode.values(), ConfigConstants.CONFIG_BRANCH_SECTION, Constants.MASTER, - ConfigConstants.CONFIG_KEY_REBASE, null)); + ConfigConstants.CONFIG_KEY_REBASE)); assertEquals(BranchRebaseMode.REBASE, - config.getEnum(BranchRebaseMode.values(), - ConfigConstants.CONFIG_BRANCH_SECTION, branch, + config.getEnum(ConfigConstants.CONFIG_BRANCH_SECTION, branch, ConfigConstants.CONFIG_KEY_REBASE, BranchRebaseMode.NONE)); assertFalse(config.getBoolean(ConfigConstants.CONFIG_BRANCH_SECTION, diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/FetchCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/FetchCommand.java index 0713c38931..f24127bd51 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/FetchCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/FetchCommand.java @@ -124,7 +124,7 @@ public class FetchCommand extends TransportCommand { FetchRecurseSubmodulesMode mode = repo.getConfig().getEnum( FetchRecurseSubmodulesMode.values(), ConfigConstants.CONFIG_SUBMODULE_SECTION, path, - ConfigConstants.CONFIG_KEY_FETCH_RECURSE_SUBMODULES, null); + ConfigConstants.CONFIG_KEY_FETCH_RECURSE_SUBMODULES); if (mode != null) { return mode; } @@ -132,7 +132,7 @@ public class FetchCommand extends TransportCommand { // Fall back to fetch.recurseSubmodules, if set mode = repo.getConfig().getEnum(FetchRecurseSubmodulesMode.values(), ConfigConstants.CONFIG_FETCH_SECTION, null, - ConfigConstants.CONFIG_KEY_RECURSE_SUBMODULES, null); + ConfigConstants.CONFIG_KEY_RECURSE_SUBMODULES); if (mode != null) { return mode; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/PullCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/PullCommand.java index 83ae0fc9d4..4b2cee45c2 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/PullCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/PullCommand.java @@ -533,9 +533,9 @@ public class PullCommand extends TransportCommand { Config config) { BranchRebaseMode mode = config.getEnum(BranchRebaseMode.values(), ConfigConstants.CONFIG_BRANCH_SECTION, - branchName, ConfigConstants.CONFIG_KEY_REBASE, null); + branchName, ConfigConstants.CONFIG_KEY_REBASE); if (mode == null) { - mode = config.getEnum(BranchRebaseMode.values(), + mode = config.getEnum( ConfigConstants.CONFIG_PULL_SECTION, null, ConfigConstants.CONFIG_KEY_REBASE, BranchRebaseMode.NONE); } @@ -549,7 +549,7 @@ public class PullCommand extends TransportCommand { Config config = repo.getConfig(); Merge ffMode = config.getEnum(Merge.values(), ConfigConstants.CONFIG_PULL_SECTION, null, - ConfigConstants.CONFIG_KEY_FF, null); + ConfigConstants.CONFIG_KEY_FF); return ffMode != null ? FastForwardMode.valueOf(ffMode) : null; } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCache.java b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCache.java index 34dba0b5be..c650d6e8e7 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCache.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/dircache/DirCache.java @@ -1037,7 +1037,12 @@ public class DirCache { } } - enum DirCacheVersion implements ConfigEnum { + /** + * DirCache versions + * + * @since 7.2 + */ + public enum DirCacheVersion implements ConfigEnum { /** Minimum index version on-disk format that we support. */ DIRC_VERSION_MINIMUM(2), @@ -1060,6 +1065,9 @@ public class DirCache { this.version = versionCode; } + /** + * @return the version code for this version + */ public int getVersionCode() { return version; } @@ -1078,6 +1086,13 @@ public class DirCache { } } + /** + * Create DirCacheVersion from integer value of the version code. + * + * @param val + * integer value of the version code. + * @return the DirCacheVersion instance of the version code. + */ public static DirCacheVersion fromInt(int val) { for (DirCacheVersion v : DirCacheVersion.values()) { if (val == v.getVersionCode()) { @@ -1098,9 +1113,8 @@ public class DirCache { boolean manyFiles = cfg.getBoolean( ConfigConstants.CONFIG_FEATURE_SECTION, ConfigConstants.CONFIG_KEY_MANYFILES, false); - indexVersion = cfg.getEnum(DirCacheVersion.values(), - ConfigConstants.CONFIG_INDEX_SECTION, null, - ConfigConstants.CONFIG_KEY_VERSION, + indexVersion = cfg.getEnum(ConfigConstants.CONFIG_INDEX_SECTION, + null, ConfigConstants.CONFIG_KEY_VERSION, manyFiles ? DirCacheVersion.DIRC_VERSION_PATHCOMPRESS : DirCacheVersion.DIRC_VERSION_EXTENDED); skipHash = cfg.getBoolean(ConfigConstants.CONFIG_INDEX_SECTION, diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/BranchConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/BranchConfig.java index e15c7af932..7921052aaa 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/BranchConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/BranchConfig.java @@ -187,8 +187,7 @@ public class BranchConfig { * @since 4.5 */ public BranchRebaseMode getRebaseMode() { - return config.getEnum(BranchRebaseMode.values(), - ConfigConstants.CONFIG_BRANCH_SECTION, branchName, + return config.getEnum(ConfigConstants.CONFIG_BRANCH_SECTION, branchName, ConfigConstants.CONFIG_KEY_REBASE, BranchRebaseMode.NONE); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgConfig.java index 76ed36a6e5..23d16db39f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/GpgConfig.java @@ -74,8 +74,7 @@ public class GpgConfig { * the config to read from */ public GpgConfig(Config config) { - keyFormat = config.getEnum(GpgFormat.values(), - ConfigConstants.CONFIG_GPG_SECTION, null, + keyFormat = config.getEnum(ConfigConstants.CONFIG_GPG_SECTION, null, ConfigConstants.CONFIG_KEY_FORMAT, GpgFormat.OPENPGP); signingKey = config.getString(ConfigConstants.CONFIG_USER_SECTION, null, ConfigConstants.CONFIG_KEY_SIGNINGKEY); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/submodule/SubmoduleWalk.java b/org.eclipse.jgit/src/org/eclipse/jgit/submodule/SubmoduleWalk.java index becc8082ba..105cba7d28 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/submodule/SubmoduleWalk.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/submodule/SubmoduleWalk.java @@ -787,14 +787,14 @@ public class SubmoduleWalk implements AutoCloseable { IgnoreSubmoduleMode mode = repoConfig.getEnum( IgnoreSubmoduleMode.values(), ConfigConstants.CONFIG_SUBMODULE_SECTION, getModuleName(), - ConfigConstants.CONFIG_KEY_IGNORE, null); + ConfigConstants.CONFIG_KEY_IGNORE); if (mode != null) { return mode; } lazyLoadModulesConfig(); - return modulesConfig.getEnum(IgnoreSubmoduleMode.values(), - ConfigConstants.CONFIG_SUBMODULE_SECTION, getModuleName(), - ConfigConstants.CONFIG_KEY_IGNORE, IgnoreSubmoduleMode.NONE); + return modulesConfig.getEnum(ConfigConstants.CONFIG_SUBMODULE_SECTION, + getModuleName(), ConfigConstants.CONFIG_KEY_IGNORE, + IgnoreSubmoduleMode.NONE); } /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/HttpConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/HttpConfig.java index 73eddb8e21..f10b7bf452 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/HttpConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/HttpConfig.java @@ -302,8 +302,7 @@ public class HttpConfig { int postBufferSize = config.getInt(HTTP, POST_BUFFER_KEY, 1 * 1024 * 1024); boolean sslVerifyFlag = config.getBoolean(HTTP, SSL_VERIFY_KEY, true); - HttpRedirectMode followRedirectsMode = config.getEnum( - HttpRedirectMode.values(), HTTP, null, + HttpRedirectMode followRedirectsMode = config.getEnum(HTTP, null, FOLLOW_REDIRECTS_KEY, HttpRedirectMode.INITIAL); int redirectLimit = config.getInt(HTTP, MAX_REDIRECTS_KEY, MAX_REDIRECTS); @@ -335,8 +334,8 @@ public class HttpConfig { postBufferSize); sslVerifyFlag = config.getBoolean(HTTP, match, SSL_VERIFY_KEY, sslVerifyFlag); - followRedirectsMode = config.getEnum(HttpRedirectMode.values(), - HTTP, match, FOLLOW_REDIRECTS_KEY, followRedirectsMode); + followRedirectsMode = config.getEnum(HTTP, match, + FOLLOW_REDIRECTS_KEY, followRedirectsMode); int newMaxRedirects = config.getInt(HTTP, match, MAX_REDIRECTS_KEY, redirectLimit); if (newMaxRedirects >= 0) { -- cgit v1.2.3 From e295f7bd9eeccfd039393d58574461fc9d924e8a Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Wed, 12 Feb 2025 16:25:05 -0800 Subject: blame.cache: Define interface and POJO to cache blame results The blame generator goes into the history of a file until it blames all lines. If the caller keeps a cache of the results, the generator could use it to shorten that walk, using the blame information of an older revision of the file to build the new blame view. Define an interface and POJO for the blame cache. Change-Id: Ib6b033ef46089bbc5a5b32e8e060d4ab0f74b871 --- .../org/eclipse/jgit/blame/cache/BlameCache.java | 46 ++++++++ .../org/eclipse/jgit/blame/cache/CacheRegion.java | 121 +++++++++++++++++++++ 2 files changed, 167 insertions(+) create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/blame/cache/BlameCache.java create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/blame/cache/CacheRegion.java (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/blame/cache/BlameCache.java b/org.eclipse.jgit/src/org/eclipse/jgit/blame/cache/BlameCache.java new file mode 100644 index 0000000000..d44fb5f62b --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/blame/cache/BlameCache.java @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2025, Google LLC. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.blame.cache; + +import java.io.IOException; +import java.util.List; + +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.Repository; + +/** + * Keeps the blame information for a path at certain commit. + *

+ * If there is a result, it covers the whole file at that revision + * + * @since 7.2 + */ +public interface BlameCache { + /** + * Gets the blame of a path at a given commit if available. + *

+ * Since this cache is used in blame calculation, this get() method should + * only retrieve the cache value, and not re-trigger blame calculation. In + * other words, this acts as "getIfPresent", and not "computeIfAbsent". + * + * @param repo + * repository containing the commit + * @param commitId + * we are looking at the file in this revision + * @param path + * path a file in the repo + * + * @return the blame of a path at a given commit or null if not in cache + * @throws IOException + * error retrieving/parsing values from storage + */ + List get(Repository repo, ObjectId commitId, String path) + throws IOException; +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/blame/cache/CacheRegion.java b/org.eclipse.jgit/src/org/eclipse/jgit/blame/cache/CacheRegion.java new file mode 100644 index 0000000000..6aa4eef2ca --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/blame/cache/CacheRegion.java @@ -0,0 +1,121 @@ +/* + * Copyright (C) 2025, Google LLC. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.blame.cache; + +import org.eclipse.jgit.lib.ObjectId; + +/** + * Region of the blame of a file. + *

+ * Usually all parameters are non-null, except when the Region was created + * to fill an unblamed gap (to cover for bugs in the calculation). In that + * case, path, commit and author will be null. + * + * @since 7.2 + **/ +public class CacheRegion implements Comparable { + private final String sourcePath; + + private final ObjectId sourceCommit; + + private final int end; + + private final int start; + + /** + * A blamed portion of a file + * + * @param path + * location of the file + * @param commit + * commit that is modifying this region + * @param start + * first line of this region (inclusive) + * @param end + * last line of this region (non-inclusive!) + */ + public CacheRegion(String path, ObjectId commit, + int start, int end) { + allOrNoneNull(path, commit); + this.sourcePath = path; + this.sourceCommit = commit; + this.start = start; + this.end = end; + } + + /** + * First line of this region. Starting by 0, inclusive + * + * @return first line of this region. + */ + public int getStart() { + return start; + } + + /** + * One after last line in this region (or: last line non-inclusive) + * + * @return one after last line in this region. + */ + public int getEnd() { + return end; + } + + + /** + * Path of the file this region belongs to + * + * @return path in the repo/commit + */ + public String getSourcePath() { + return sourcePath; + } + + /** + * Commit this region belongs to + * + * @return commit for this region + */ + public ObjectId getSourceCommit() { + return sourceCommit; + } + + @Override + public int compareTo(CacheRegion o) { + return start - o.start; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + if (sourceCommit != null) { + sb.append(sourceCommit.name(), 0, 7).append(' ') + .append(" (") + .append(sourcePath).append(')'); + } else { + sb.append(""); + } + sb.append(' ').append("start=").append(start).append(", count=") + .append(end - start); + return sb.toString(); + } + + private static void allOrNoneNull(String path, ObjectId commit) { + if (path != null && commit != null) { + return; + } + + if (path == null && commit == null) { + return; + } + throw new IllegalArgumentException(String.format( + "expected all null or none: %s, %s", path, commit)); + } +} -- cgit v1.2.3 From 871c5e17781cdba1991051fbf015e43a29d1b31f Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Thu, 6 Feb 2025 14:07:47 -0800 Subject: BlameGenerator: cache and reuse blame results Blame goes into the history of a file until it blames all lines. If we have a blamed version of the file in cache, we can use it to resolve the lines not blamed yet and cut the calculation earlier. When processing a candidate, check if it is in the cache and if so fill the blame from the pending regions using the information of the cached (and fully blamed) copy. The generator doesn't populate the cache itself. Callers must take the final results and put them in the cache. Change-Id: Ia99b09d6d825e96940ca4200967958923bf647c7 --- .../jgit/blame/BlameGeneratorCacheTest.java | 398 +++++++++++++++++++++ .../eclipse/jgit/blame/BlameRegionMergerTest.java | 320 +++++++++++++++++ .../src/org/eclipse/jgit/blame/BlameGenerator.java | 121 ++++++- .../org/eclipse/jgit/blame/BlameRegionMerger.java | 158 ++++++++ .../src/org/eclipse/jgit/blame/BlameResult.java | 1 + 5 files changed, 993 insertions(+), 5 deletions(-) create mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/blame/BlameGeneratorCacheTest.java create mode 100644 org.eclipse.jgit.test/tst/org/eclipse/jgit/blame/BlameRegionMergerTest.java create mode 100644 org.eclipse.jgit/src/org/eclipse/jgit/blame/BlameRegionMerger.java (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/blame/BlameGeneratorCacheTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/blame/BlameGeneratorCacheTest.java new file mode 100644 index 0000000000..65cac11982 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/blame/BlameGeneratorCacheTest.java @@ -0,0 +1,398 @@ +/* + * Copyright (C) 2025, Google LLC. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.blame; + +import static java.lang.String.join; +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.eclipse.jgit.blame.cache.BlameCache; +import org.eclipse.jgit.blame.cache.CacheRegion; +import org.eclipse.jgit.internal.storage.file.FileRepository; +import org.eclipse.jgit.junit.RepositoryTestCase; +import org.eclipse.jgit.junit.TestRepository; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.revwalk.RevCommit; +import org.junit.Test; + +public class BlameGeneratorCacheTest extends RepositoryTestCase { + private static final String FILE = "file.txt"; + + /** + * Simple history: + * + * 

+	 *          C1    C2    C3    C4   C4 blame
+	 * lines ----------------------------------
+	 * L1    |  C1    C1    C1    C1     C1
+	 * L2    |  C1    C1   *C3   *C4     C4
+	 * L3    |  C1    C1   *C3    C3     C3
+	 * L4    |       *C2    C2   *C4     C4
+	 *
+ * + * @throws Exception any error + */ + @Test + public void blame_simple_correctRegions() throws Exception { + RevCommit c1, c2, c3, c4; + try (TestRepository r = new TestRepository<>(db)) { + c1 = commit(r, lines("L1C1", "L2C1", "L3C1")); + c2 = commit(r, lines("L1C1", "L2C1", "L3C1", "L4C2"), c1); + c3 = commit(r, lines("L1C1", "L2C3", "L3C3", "L4C2"), c2); + c4 = commit(r, lines("L1C1", "L2C4", "L3C3", "L4C4"), c3); + } + + List expectedRegions = Arrays.asList( + new EmittedRegion(c1, 0, 1), + new EmittedRegion(c4, 1, 2), + new EmittedRegion(c3, 2, 3), + new EmittedRegion(c4, 3, 4)); + + assertRegions(c4, null, expectedRegions, 4); + assertRegions(c4, emptyCache(), expectedRegions, 4); + assertRegions(c4, blameAndCache(c4), expectedRegions, 4); + assertRegions(c4, blameAndCache(c3), expectedRegions, 4); + assertRegions(c4, blameAndCache(c2), expectedRegions, 4); + assertRegions(c4, blameAndCache(c1), expectedRegions, 4); + } + + @Test + public void blame_simple_cacheUsage() throws Exception { + RevCommit c1, c2, c3, c4; + try (TestRepository r = new TestRepository<>(db)) { + c1 = commit(r, lines("L1C1", "L2C1", "L3C1")); + c2 = commit(r, lines("L1C1", "L2C1", "L3C1", "L4C2"), c1); + c3 = commit(r, lines("L1C1", "L2C3", "L3C3", "L4C2"), c2); + c4 = commit(r, lines("L1C1", "L2C4", "L3C3", "L4C4"), c3); + } + + assertCacheUsage(c4, null, false, 4); + assertCacheUsage(c4, emptyCache(), false, 4); + assertCacheUsage(c4, blameAndCache(c4), true, 1); + assertCacheUsage(c4, blameAndCache(c3), true, 2); + assertCacheUsage(c4, blameAndCache(c2), true, 3); + assertCacheUsage(c4, blameAndCache(c1), true, 4); + } + + /** + * Overwrite: + * + * 
+	 *          C1    C2    C3    C3 blame
+	 * lines ----------------------------------
+	 * L1    |  C1    C1   *C3      C3
+	 * L2    |  C1    C1   *C3      C3
+	 * L3    |  C1    C1   *C3      C3
+	 * L4    |       *C2
+	 *
+ * + * @throws Exception any error + */ + @Test + public void blame_ovewrite_correctRegions() throws Exception { + RevCommit c1, c2, c3; + try (TestRepository r = new TestRepository<>(db)) { + c1 = commit(r, lines("L1C1", "L2C1", "L3C1")); + c2 = commit(r, lines("L1C1", "L2C1", "L3C1", "L4C2"), c1); + c3 = commit(r, lines("L1C3", "L2C3", "L3C3"), c2); + } + + List expectedRegions = Arrays.asList( + new EmittedRegion(c3, 0, 3)); + + assertRegions(c3, null, expectedRegions, 3); + assertRegions(c3, emptyCache(), expectedRegions, 3); + assertRegions(c3, blameAndCache(c3), expectedRegions, 3); + assertRegions(c3, blameAndCache(c2), expectedRegions, 3); + assertRegions(c3, blameAndCache(c1), expectedRegions, 3); + } + + @Test + public void blame_overwrite_cacheUsage() throws Exception { + RevCommit c1, c2, c3; + try (TestRepository r = new TestRepository<>(db)) { + c1 = commit(r, lines("L1C1", "L2C1", "L3C1")); + c2 = commit(r, lines("L1C1", "L2C1", "L3C1", "L4C2"), c1); + c3 = commit(r, lines("L1C3", "L2C3", "L3C3"), c2); + } + + assertCacheUsage(c3, null, false, 1); + assertCacheUsage(c3, emptyCache(), false, 1); + assertCacheUsage(c3, blameAndCache(c3), true, 1); + assertCacheUsage(c3, blameAndCache(c2), false, 1); + assertCacheUsage(c3, blameAndCache(c1), false, 1); + } + + /** + * Merge: + * + * 
+	 *                 root
+	 *                 ----
+	 *                 L1  -
+	 *                 L2  -
+	 *                 L3  -
+	 *               /     \
+	 *           sideA     sideB
+	 *           -----     -----
+	 *           *L1 a      L1 -
+	 *           *L2 a      L2 -
+	 *           *L3 a      L3 -
+	 *           *L4 a     *L4 b
+	 *            L5 -     *L5 b
+	 *            L6 -     *L6 b
+	 *            L7 -     *L7 b
+	 *              \       /
+	 *                merge
+	 *                -----
+	 *              L1-L4 a (from sideA)
+	 *              L5-L7 - (common, from root)
+	 *              L8-L11 b (from sideB)
+	 *
+ * + * @throws Exception any error + */ + @Test + public void blame_merge_correctRegions() throws Exception { + RevCommit root, sideA, sideB, mergedTip; + try (TestRepository r = new TestRepository<>(db)) { + root = commitAsLines(r, "---"); + sideA = commitAsLines(r, "aaaa---", root); + sideB = commitAsLines(r, "---bbbb", root); + mergedTip = commitAsLines(r, "aaaa---bbbb", sideA, sideB); + } + + List expectedRegions = Arrays.asList( + new EmittedRegion(sideA, 0, 4), + new EmittedRegion(root, 4, 7), + new EmittedRegion(sideB, 7, 11)); + + assertRegions(mergedTip, null, expectedRegions, 11); + assertRegions(mergedTip, emptyCache(), expectedRegions, 11); + assertRegions(mergedTip, blameAndCache(root), expectedRegions, 11); + assertRegions(mergedTip, blameAndCache(sideA), expectedRegions, 11); + assertRegions(mergedTip, blameAndCache(sideB), expectedRegions, 11); + assertRegions(mergedTip, blameAndCache(mergedTip), expectedRegions, 11); + } + + @Test + public void blame_merge_cacheUsage() throws Exception { + RevCommit root, sideA, sideB, mergedTip; + try (TestRepository r = new TestRepository<>(db)) { + root = commitAsLines(r, "---"); + sideA = commitAsLines(r, "aaaa---", root); + sideB = commitAsLines(r, "---bbbb", root); + mergedTip = commitAsLines(r, "aaaa---bbbb", sideA, sideB); + } + + assertCacheUsage(mergedTip, null, /* cacheUsed */ false, + /* candidates */ 4); + assertCacheUsage(mergedTip, emptyCache(), false, 4); + assertCacheUsage(mergedTip, blameAndCache(mergedTip), true, 1); + + // While splitting unblamed regions to parents, sideA comes first + // and gets "aaaa----". Processing is by commit time, so sideB is + // explored first + assertCacheUsage(mergedTip, blameAndCache(sideA), true, 3); + assertCacheUsage(mergedTip, blameAndCache(sideB), true, 4); + assertCacheUsage(mergedTip, blameAndCache(root), true, 4); + } + + /** + * Moving block (insertion) + * + * 
+	 *          C1    C2    C3    C3 blame
+	 * lines ----------------------------------
+	 * L1    |  C1    C1    C1      C1
+	 * L2    |  C1   *C2    C2      C2
+	 * L3    |        C1   *C3      C3
+	 * L4    |              C1      C1
+	 *
+ * + * @throws Exception any error + */ + @Test + public void blame_movingBlock_correctRegions() throws Exception { + RevCommit c1, c2, c3; + try (TestRepository r = new TestRepository<>(db)) { + c1 = commit(r, lines("L1C1", "L2C1")); + c2 = commit(r, lines("L1C1", "middle", "L2C1"), c1); + c3 = commit(r, lines("L1C1", "middle", "extra", "L2C1"), c2); + } + + List expectedRegions = Arrays.asList( + new EmittedRegion(c1, 0, 1), + new EmittedRegion(c2, 1, 2), + new EmittedRegion(c3, 2, 3), + new EmittedRegion(c1, 3, 4)); + + assertRegions(c3, null, expectedRegions, 4); + assertRegions(c3, emptyCache(), expectedRegions, 4); + assertRegions(c3, blameAndCache(c3), expectedRegions, 4); + assertRegions(c3, blameAndCache(c2), expectedRegions, 4); + assertRegions(c3, blameAndCache(c1), expectedRegions, 4); + } + + @Test + public void blame_movingBlock_cacheUsage() throws Exception { + RevCommit c1, c2, c3; + try (TestRepository r = new TestRepository<>(db)) { + c1 = commitAsLines(r, "root---"); + c2 = commitAsLines(r, "rootXXX---", c1); + c3 = commitAsLines(r, "rootYYYXXX---", c2); + } + + assertCacheUsage(c3, null, false, 3); + assertCacheUsage(c3, emptyCache(), false, 3); + assertCacheUsage(c3, blameAndCache(c3), true, 1); + assertCacheUsage(c3, blameAndCache(c2), true, 2); + assertCacheUsage(c3, blameAndCache(c1), true, 3); + } + + private void assertRegions(RevCommit commit, InMemoryBlameCache cache, + List expectedRegions, int resultLineCount) + throws IOException { + try (BlameGenerator gen = new BlameGenerator(db, FILE, cache)) { + gen.push(null, db.parseCommit(commit)); + List regions = consume(gen); + assertRegionsEquals(expectedRegions, regions); + assertAllLinesCovered(/* lines= */ resultLineCount, regions); + } + } + + private void assertCacheUsage(RevCommit commit, InMemoryBlameCache cache, + boolean useCache, int candidatesVisited) throws IOException { + try (BlameGenerator gen = new BlameGenerator(db, FILE, cache)) { + gen.push(null, db.parseCommit(commit)); + consume(gen); + assertEquals(useCache, gen.getStats().isCacheHit()); + assertEquals(candidatesVisited, + gen.getStats().getCandidatesVisited()); + } + } + + private static void assertAllLinesCovered(int lines, + List regions) { + Collections.sort(regions); + assertEquals("Starts in first line", 0, regions.get(0).resultStart()); + for (int i = 1; i < regions.size(); i++) { + assertEquals("No gaps", regions.get(i).resultStart(), + regions.get(i - 1).resultEnd()); + } + assertEquals("Ends in last line", lines, + regions.get(regions.size() - 1).resultEnd()); + } + + private static void assertRegionsEquals( + List expected, List actual) { + assertEquals(expected.size(), actual.size()); + Collections.sort(actual); + for (int i = 0; i < expected.size(); i++) { + assertEquals(String.format("List differ in element %d", i), + expected.get(i), actual.get(i)); + } + } + + private static InMemoryBlameCache emptyCache() { + return new InMemoryBlameCache(""); + } + + private List consume(BlameGenerator generator) + throws IOException { + List result = new ArrayList<>(); + while (generator.next()) { + EmittedRegion genRegion = new EmittedRegion( + generator.getSourceCommit().toObjectId(), + generator.getResultStart(), generator.getResultEnd()); + result.add(genRegion); + } + return result; + } + + private InMemoryBlameCache blameAndCache(RevCommit commit) + throws IOException { + List regions; + try (BlameGenerator generator = new BlameGenerator(db, FILE)) { + generator.push(null, commit); + regions = consume(generator).stream() + .map(EmittedRegion::asCacheRegion) + .collect(Collectors.toUnmodifiableList()); + } + InMemoryBlameCache cache = new InMemoryBlameCache(""); + cache.put(commit, FILE, regions); + return cache; + } + + private static RevCommit commitAsLines(TestRepository r, + String charPerLine, RevCommit... parents) throws Exception { + return commit(r, charPerLine.replaceAll("\\S", "$0\n"), parents); + } + + private static RevCommit commit(TestRepository r, String contents, + RevCommit... parents) throws Exception { + return r.commit(r.tree(r.file(FILE, r.blob(contents))), parents); + } + + private static String lines(String... l) { + return join("\n", l); + } + + private record EmittedRegion(ObjectId oid, int resultStart, int resultEnd) + implements Comparable { + @Override + public int compareTo(EmittedRegion o) { + return resultStart - o.resultStart; + } + + CacheRegion asCacheRegion() { + return new CacheRegion(FILE, oid, resultStart, resultEnd); + } + } + + private static class InMemoryBlameCache implements BlameCache { + + private final Map> cache = new HashMap<>(); + + private final String description; + + public InMemoryBlameCache(String description) { + this.description = description; + } + + @Override + public List get(Repository repo, ObjectId commitId, + String path) throws IOException { + return cache.get(new Key(commitId.name(), path)); + } + + public void put(ObjectId commitId, String path, + List cachedRegions) { + cache.put(new Key(commitId.name(), path), cachedRegions); + } + + @Override + public String toString() { + return "InMemoryCache: " + description; + } + + record Key(String commitId, String path) { + } + } +} diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/blame/BlameRegionMergerTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/blame/BlameRegionMergerTest.java new file mode 100644 index 0000000000..cf0dc452e7 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/blame/BlameRegionMergerTest.java @@ -0,0 +1,320 @@ +/* + * Copyright (C) 2025, Google LLC. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.blame; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +import org.eclipse.jgit.blame.cache.CacheRegion; +import org.eclipse.jgit.junit.RepositoryTestCase; +import org.eclipse.jgit.lib.AnyObjectId; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.revwalk.RevCommit; +import org.junit.Test; + +public class BlameRegionMergerTest extends RepositoryTestCase { + + private static final ObjectId O1 = ObjectId + .fromString("ff6dd8db6edc9aa0ac58fea1d14a55be46c3eb14"); + + private static final ObjectId O2 = ObjectId + .fromString("c3c7f680c6bee238617f25f6aa85d0b565fc8ecb"); + + private static final ObjectId O3 = ObjectId + .fromString("29e014aad0399fe8ede7c101d01b6e440ac9966b"); + + List fakeCommits = List.of(new FakeRevCommit(O1), + new FakeRevCommit(O2), new FakeRevCommit(O3)); + + // In reverse order, so the code doesn't assume a sorted list + List cachedRegions = List.of( + new CacheRegion("README", O3, 20, 30), + new CacheRegion("README", O2, 10, 20), + new CacheRegion("README", O1, 0, 10)); + + BlameRegionMerger blamer = new BlameRegionMergerFakeCommits(fakeCommits, + cachedRegions); + + @Test + public void intersectRegions_allInside() { + Region unblamed = new Region(15, 18, 10); + CacheRegion blamed = new CacheRegion("README", O1, 10, 90); + + Region result = BlameRegionMerger.intersectRegions(unblamed, blamed); + // Same lines in result and source + assertEquals(15, result.resultStart); + assertEquals(18, result.sourceStart); + assertEquals(10, result.length); + assertNull(result.next); + } + + @Test + public void intersectRegions_startsBefore() { + // Intesecting [4, 14) with [10, 90) + Region unblamed = new Region(30, 4, 10); + CacheRegion blamed = new CacheRegion("README", O1, 10, 90); + + Region result = BlameRegionMerger.intersectRegions(unblamed, blamed); + + // The unblamed region starting at 4 (sourceStart), starts at 30 in the + // original file (resultStart). e.g. some commit introduced + // lines. If we take the second portion of the region, we need to move + // the result start accordingly. + assertEquals(36, result.resultStart); + assertEquals(10, result.sourceStart); + assertEquals(4, result.length); + assertNull(result.next); + } + + @Test + public void intersectRegions_endsAfter() { + // Intesecting [85, 95) with [10, 90) + Region unblamed = new Region(30, 85, 10); + CacheRegion blamed = new CacheRegion("README", O1, 10, 90); + + Region result = BlameRegionMerger.intersectRegions(unblamed, blamed); + + assertEquals(30, result.resultStart); + assertEquals(85, result.sourceStart); + assertEquals(5, result.length); + assertNull(result.next); + } + + @Test + public void intersectRegions_spillOverBothSides() { + // Intesecting [5, 100) with [10, 90) + Region unblamed = new Region(30, 5, 95); + CacheRegion blamed = new CacheRegion("README", O1, 10, 90); + + Region result = BlameRegionMerger.intersectRegions(unblamed, blamed); + + assertEquals(35, result.resultStart); + assertEquals(10, result.sourceStart); + assertEquals(80, result.length); + assertNull(result.next); + } + + @Test + public void intersectRegions_exactMatch() { + // Intesecting [5, 100) with [10, 90) + Region unblamed = new Region(30, 10, 80); + CacheRegion blamed = new CacheRegion("README", O1, 10, 90); + + Region result = BlameRegionMerger.intersectRegions(unblamed, blamed); + + assertEquals(30, result.resultStart); + assertEquals(10, result.sourceStart); + assertEquals(80, result.length); + assertNull(result.next); + } + + @Test + public void findOverlaps_allInside() { + Region unblamed = new Region(0, 11, 4); + List overlaps = blamer.findOverlaps(unblamed); + assertEquals(1, overlaps.size()); + assertEquals(10, overlaps.get(0).getStart()); + assertEquals(20, overlaps.get(0).getEnd()); + } + + @Test + public void findOverlaps_overTwoRegions() { + Region unblamed = new Region(0, 8, 4); + List overlaps = blamer.findOverlaps(unblamed); + assertEquals(2, overlaps.size()); + assertEquals(0, overlaps.get(0).getStart()); + assertEquals(10, overlaps.get(0).getEnd()); + assertEquals(10, overlaps.get(1).getStart()); + assertEquals(20, overlaps.get(1).getEnd()); + } + + @Test + public void findOverlaps_overThreeRegions() { + Region unblamed = new Region(0, 8, 15); + List overlaps = blamer.findOverlaps(unblamed); + assertEquals(3, overlaps.size()); + assertEquals(0, overlaps.get(0).getStart()); + assertEquals(10, overlaps.get(0).getEnd()); + assertEquals(10, overlaps.get(1).getStart()); + assertEquals(20, overlaps.get(1).getEnd()); + assertEquals(20, overlaps.get(2).getStart()); + assertEquals(30, overlaps.get(2).getEnd()); + } + + @Test + public void blame_exactOverlap() throws IOException { + Region unblamed = new Region(0, 10, 10); + List blamed = blamer.mergeOneRegion(unblamed); + + assertEquals(1, blamed.size()); + Candidate c = blamed.get(0); + assertEquals(c.sourceCommit.name(), O2.name()); + assertEquals(c.regionList.resultStart, unblamed.resultStart); + assertEquals(c.regionList.sourceStart, unblamed.sourceStart); + assertEquals(10, c.regionList.length); + assertNull(c.regionList.next); + } + + @Test + public void blame_corruptedIndex() { + Region outOfRange = new Region(0, 43, 4); + // This region is out of the blamed area + assertThrows(IllegalStateException.class, + () -> blamer.mergeOneRegion(outOfRange)); + } + + @Test + public void blame_allInsideOneBlamedRegion() throws IOException { + Region unblamed = new Region(0, 5, 3); + // This region if fully blamed to O1 + List blamed = blamer.mergeOneRegion(unblamed); + assertEquals(1, blamed.size()); + Candidate c = blamed.get(0); + assertEquals(c.sourceCommit.name(), O1.name()); + assertEquals(c.regionList.resultStart, unblamed.resultStart); + assertEquals(c.regionList.sourceStart, unblamed.sourceStart); + assertEquals(3, c.regionList.length); + assertNull(c.regionList.next); + } + + @Test + public void blame_overTwoBlamedRegions() throws IOException { + Region unblamed = new Region(0, 8, 5); + // (8, 10) belongs go C1, (10, 13) to C2 + List blamed = blamer.mergeOneRegion(unblamed); + assertEquals(2, blamed.size()); + Candidate c = blamed.get(0); + assertEquals(c.sourceCommit.name(), O1.name()); + assertEquals(unblamed.resultStart, c.regionList.resultStart); + assertEquals(unblamed.sourceStart, c.regionList.sourceStart); + assertEquals(2, c.regionList.length); + assertNull(c.regionList.next); + + c = blamed.get(1); + assertEquals(c.sourceCommit.name(), O2.name()); + assertEquals(2, c.regionList.resultStart); + assertEquals(10, c.regionList.sourceStart); + assertEquals(3, c.regionList.length); + assertNull(c.regionList.next); + } + + @Test + public void blame_all() throws IOException { + Region unblamed = new Region(0, 0, 30); + List blamed = blamer.mergeOneRegion(unblamed); + assertEquals(3, blamed.size()); + Candidate c = blamed.get(0); + assertEquals(c.sourceCommit.name(), O1.name()); + assertEquals(unblamed.resultStart, c.regionList.resultStart); + assertEquals(unblamed.sourceStart, c.regionList.sourceStart); + assertEquals(10, c.regionList.length); + assertNull(c.regionList.next); + + c = blamed.get(1); + assertEquals(c.sourceCommit.name(), O2.name()); + assertEquals(10, c.regionList.resultStart); + assertEquals(10, c.regionList.sourceStart); + assertEquals(10, c.regionList.length); + assertNull(c.regionList.next); + + c = blamed.get(2); + assertEquals(c.sourceCommit.name(), O3.name()); + assertEquals(20, c.regionList.resultStart); + assertEquals(20, c.regionList.sourceStart); + assertEquals(10, c.regionList.length); + assertNull(c.regionList.next); + } + + @Test + public void blame_fromCandidate() { + // We don't use anything from the candidate besides the + // regionList + Candidate c = new Candidate(null, null, null); + c.regionList = new Region(0, 8, 5); + c.regionList.next = new Region(22, 22, 4); + + Candidate blamed = blamer.mergeCandidate(c); + // Three candidates + assertNotNull(blamed); + assertNotNull(blamed.queueNext); + assertNotNull(blamed.queueNext.queueNext); + assertNull(blamed.queueNext.queueNext.queueNext); + + assertEquals(O1.name(), blamed.sourceCommit.name()); + + Candidate second = blamed.queueNext; + assertEquals(O2.name(), second.sourceCommit.name()); + + Candidate third = blamed.queueNext.queueNext; + assertEquals(O3.name(), third.sourceCommit.name()); + } + + @Test + public void blame_fromCandidate_twiceCandidateInOutput() { + Candidate c = new Candidate(null, null, null); + // This produces O1 and O2 + c.regionList = new Region(0, 8, 5); + // This produces O2 and O3 + c.regionList.next = new Region(20, 15, 7); + + Candidate blamed = blamer.mergeCandidate(c); + assertCandidateSingleRegion(O1, 2, blamed); + blamed = blamed.queueNext; + assertCandidateSingleRegion(O2, 3, blamed); + // We do not merge candidates afterwards, so these are + // two different candidates to the same source + blamed = blamed.queueNext; + assertCandidateSingleRegion(O2, 5, blamed); + blamed = blamed.queueNext; + assertCandidateSingleRegion(O3, 2, blamed); + assertNull(blamed.queueNext); + } + + private static void assertCandidateSingleRegion(ObjectId expectedOid, + int expectedLength, Candidate actual) { + assertNotNull("candidate", actual); + assertNotNull("region list not empty", actual.regionList); + assertNull("region list has only one element", actual.regionList.next); + assertEquals(expectedOid, actual.sourceCommit); + assertEquals(expectedLength, actual.regionList.length); + } + + private static final class BlameRegionMergerFakeCommits + extends BlameRegionMerger { + + private final Map cache; + + BlameRegionMergerFakeCommits(List commits, + List blamedRegions) { + super(null, null, blamedRegions); + cache = commits.stream().collect(Collectors + .toMap(RevCommit::toObjectId, Function.identity())); + } + + @Override + protected RevCommit parse(ObjectId oid) { + return cache.get(oid); + } + } + + private static final class FakeRevCommit extends RevCommit { + FakeRevCommit(AnyObjectId id) { + super(id); + } + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/blame/BlameGenerator.java b/org.eclipse.jgit/src/org/eclipse/jgit/blame/BlameGenerator.java index 77967df2e5..b2296e0123 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/blame/BlameGenerator.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/blame/BlameGenerator.java @@ -28,6 +28,8 @@ import org.eclipse.jgit.blame.Candidate.BlobCandidate; import org.eclipse.jgit.blame.Candidate.HeadCandidate; import org.eclipse.jgit.blame.Candidate.ReverseCandidate; import org.eclipse.jgit.blame.ReverseWalk.ReverseCommit; +import org.eclipse.jgit.blame.cache.BlameCache; +import org.eclipse.jgit.blame.cache.CacheRegion; import org.eclipse.jgit.diff.DiffAlgorithm; import org.eclipse.jgit.diff.DiffEntry; import org.eclipse.jgit.diff.DiffEntry.ChangeType; @@ -129,8 +131,19 @@ public class BlameGenerator implements AutoCloseable { /** Blame is currently assigned to this source. */ private Candidate outCandidate; + private Region outRegion; + private final BlameCache blameCache; + + /** + * Blame in reverse order needs the source lines, but we don't have them in + * the cache. We need to ignore the cache in that case. + */ + private boolean useCache = true; + + private final Stats stats = new Stats(); + /** * Create a blame generator for the repository and path (relative to * repository) @@ -142,6 +155,24 @@ public class BlameGenerator implements AutoCloseable { * repository). */ public BlameGenerator(Repository repository, String path) { + this(repository, path, null); + } + + /** + * Create a blame generator for the repository and path (relative to + * repository) + * + * @param repository + * repository to access revision data from. + * @param path + * initial path of the file to start scanning (relative to the + * repository). + * @param blameCache + * previously calculated blames. This generator will *not* + * populate it, just consume it. + */ + public BlameGenerator(Repository repository, String path, + @Nullable BlameCache blameCache) { this.repository = repository; this.resultPath = PathFilter.create(path); @@ -150,6 +181,7 @@ public class BlameGenerator implements AutoCloseable { initRevPool(false); remaining = -1; + this.blameCache = blameCache; } private void initRevPool(boolean reverse) { @@ -159,10 +191,12 @@ public class BlameGenerator implements AutoCloseable { if (revPool != null) revPool.close(); - if (reverse) + if (reverse) { + useCache = false; revPool = new ReverseWalk(getRepository()); - else + } else { revPool = new RevWalk(getRepository()); + } SEEN = revPool.newFlag("SEEN"); //$NON-NLS-1$ reader = revPool.getObjectReader(); @@ -244,6 +278,27 @@ public class BlameGenerator implements AutoCloseable { return renameDetector; } + /** + * Stats about this generator + * + * @return the stats of this generator + */ + public Stats getStats() { + return stats; + } + + /** + * Enable/disable the use of cache (if present). Enabled by default. + *

+ * If caller need source line numbers, the generator cannot use the cache + * (source lines are not there). Use this method to disable the cache in that case. + * + * @param useCache should this generator use the cache. + */ + public void setUseCache(boolean useCache) { + this.useCache = useCache; + } + /** * Push a candidate blob onto the generator's traversal stack. *

@@ -591,6 +646,20 @@ public class BlameGenerator implements AutoCloseable { Candidate n = pop(); if (n == null) return done(); + stats.candidatesVisited += 1; + if (blameCache != null && useCache) { + List cachedBlame = blameCache.get(repository, + n.sourceCommit, n.sourcePath.getPath()); + if (cachedBlame != null) { + BlameRegionMerger rb = new BlameRegionMerger(repository, revPool, + cachedBlame); + Candidate fullyBlamed = rb.mergeCandidate(n); + if (fullyBlamed != null) { + stats.cacheHit = true; + return result(fullyBlamed); + } + } + } int pCnt = n.getParentCount(); if (pCnt == 1) { @@ -605,7 +674,7 @@ public class BlameGenerator implements AutoCloseable { // Do not generate a tip of a reverse. The region // survives and should not appear to be deleted. - } else /* if (pCnt == 0) */{ + } else /* if (pCnt == 0) */ { // Root commit, with at least one surviving region. // Assign the remaining blame here. return result(n); @@ -846,8 +915,8 @@ public class BlameGenerator implements AutoCloseable { editList = new EditList(0); } else { p.loadText(reader); - editList = diffAlgorithm.diff(textComparator, - p.sourceText, n.sourceText); + editList = diffAlgorithm.diff(textComparator, p.sourceText, + n.sourceText); } if (editList.isEmpty()) { @@ -981,6 +1050,10 @@ public class BlameGenerator implements AutoCloseable { /** * Get first line of the source data that has been blamed for the current * region + *

+ * This value is not reliable when the generator is reusing cached values. + * Cache doesn't keep the source lines, the returned value is based on the + * result and can be off if the region moved in previous commits. * * @return first line of the source data that has been blamed for the * current region. This is line number of where the region was added @@ -994,6 +1067,10 @@ public class BlameGenerator implements AutoCloseable { /** * Get one past the range of the source data that has been blamed for the * current region + *

+ * This value is not reliable when the generator is reusing cached values. + * Cache doesn't keep the source lines, the returned value is based on the + * result and can be off if the region moved in previous commits. * * @return one past the range of the source data that has been blamed for * the current region. This is line number of where the region was @@ -1124,4 +1201,38 @@ public class BlameGenerator implements AutoCloseable { return ent.getChangeType() == ChangeType.RENAME || ent.getChangeType() == ChangeType.COPY; } + + /** + * Stats about the work done by the generator + */ + public static class Stats { + + /** Candidates taken from the queue */ + private int candidatesVisited; + + private boolean cacheHit; + + /** + * Number of candidates taken from the queue + *

+ * The generator could signal it's done without exhausting all + * candidates if there is no more remaining lines or the last visited + * candidate is found in the cache. + * + * @return number of candidates taken from the queue + */ + public int getCandidatesVisited() { + return candidatesVisited; + } + + /** + * The generator found a blamed version in the cache + * + * @return true if we used results from the cache + */ + public boolean isCacheHit() { + return cacheHit; + } + + } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/blame/BlameRegionMerger.java b/org.eclipse.jgit/src/org/eclipse/jgit/blame/BlameRegionMerger.java new file mode 100644 index 0000000000..bf7e0ff736 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/blame/BlameRegionMerger.java @@ -0,0 +1,158 @@ +/* + * Copyright (C) 2025, Google LLC. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.blame; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.eclipse.jgit.blame.cache.CacheRegion; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.treewalk.filter.PathFilter; + +/** + * Translates an unblamed region into one or more blamed regions, using the + * fully blamed data from cache. + *

+ * Blamed and unblamed regions are not symmetrical: An unblamed region is just a + * range of lines over the file. A blamed region is a Candidate (with the commit + * info) with a region inside (the range blamed). + */ +class BlameRegionMerger { + private final Repository repo; + + private final List cachedRegions; + + private final RevWalk rw; + + BlameRegionMerger(Repository repo, RevWalk rw, + List cachedRegions) { + this.repo = repo; + List sorted = new ArrayList<>(cachedRegions); + Collections.sort(sorted); + this.cachedRegions = sorted; + this.rw = rw; + } + + /** + * Return one or more candidates blaming all the regions of the "unblamed" + * incoming candidate. + * + * @param candidate + * a candidate with a list of unblamed regions + * @return A linked list of Candidates with their blamed regions, null if + * there was any error. + */ + Candidate mergeCandidate(Candidate candidate) { + List newCandidates = new ArrayList<>(); + Region r = candidate.regionList; + while (r != null) { + try { + newCandidates.addAll(mergeOneRegion(r)); + } catch (IOException e) { + return null; + } + r = r.next; + } + return asLinkedCandidate(newCandidates); + } + + // Visible for testing + List mergeOneRegion(Region region) throws IOException { + List overlaps = findOverlaps(region); + if (overlaps.isEmpty()) { + throw new IllegalStateException( + "Cached blame should cover all lines"); + } + /* + * Cached regions cover the whole file. We find first which ones overlap + * with our unblamed region. Then we take the overlapping portions with + * the corresponding blame. + */ + List candidates = new ArrayList<>(); + for (CacheRegion overlap : overlaps) { + Region blamedRegions = intersectRegions(region, overlap); + Candidate c = new Candidate(repo, parse(overlap.getSourceCommit()), + PathFilter.create(overlap.getSourcePath())); + c.regionList = blamedRegions; + candidates.add(c); + } + return candidates; + } + + // Visible for testing + List findOverlaps(Region unblamed) { + int unblamedStart = unblamed.sourceStart; + int unblamedEnd = unblamedStart + unblamed.length; + List overlapping = new ArrayList<>(); + for (CacheRegion blamed : cachedRegions) { + // End is not included + if (blamed.getEnd() <= unblamedStart) { + // Blamed region is completely before + continue; + } + + if (blamed.getStart() >= unblamedEnd) { + // Blamed region is completely after + // Blamed regions are sorted by start position, nothing will + // match anymore + break; + } + overlapping.add(blamed); + } + return overlapping; + } + + // Visible for testing + /** + * Calculate the intersection between a Region and a CacheRegion, adjusting + * the start if needed. + *

+ * This should be called only if there is an overlap (filtering the cached + * regions with {@link #findOverlaps(Region)}), otherwise the result is + * meaningless. + * + * @param unblamed + * a region from the blame generator + * @param cached + * a cached region + * @return a new region with the intersection. + */ + static Region intersectRegions(Region unblamed, CacheRegion cached) { + int blamedStart = Math.max(cached.getStart(), unblamed.sourceStart); + int blamedEnd = Math.min(cached.getEnd(), + unblamed.sourceStart + unblamed.length); + int length = blamedEnd - blamedStart; + + // result start and source start should move together + int blameStartDelta = blamedStart - unblamed.sourceStart; + return new Region(unblamed.resultStart + blameStartDelta, blamedStart, + length); + } + + // Tests can override this, so they don't need a real repo, commit and walk + protected RevCommit parse(ObjectId oid) throws IOException { + return rw.parseCommit(oid); + } + + private static Candidate asLinkedCandidate(List c) { + Candidate head = c.get(0); + Candidate tail = head; + for (int i = 1; i < c.size(); i++) { + tail.queueNext = c.get(i); + tail = tail.queueNext; + } + return head; + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/blame/BlameResult.java b/org.eclipse.jgit/src/org/eclipse/jgit/blame/BlameResult.java index 5e2746cc7c..48f6b7e250 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/blame/BlameResult.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/blame/BlameResult.java @@ -79,6 +79,7 @@ public class BlameResult { BlameResult(BlameGenerator bg, String path, RawText text) { generator = bg; + generator.setUseCache(false); resultPath = path; resultContents = text; -- cgit v1.2.3 From ae6665197b175134001d7416d5c8a183dd621454 Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Fri, 21 Feb 2025 10:11:48 -0800 Subject: BlameGenerator: add @since tags to new methods and inner class Change-Id: I849d0a3ff07bb53e93112eb716a6537cf2419290 --- .../src/org/eclipse/jgit/blame/BlameGenerator.java | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/blame/BlameGenerator.java b/org.eclipse.jgit/src/org/eclipse/jgit/blame/BlameGenerator.java index b2296e0123..2d499cafce 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/blame/BlameGenerator.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/blame/BlameGenerator.java @@ -170,6 +170,7 @@ public class BlameGenerator implements AutoCloseable { * @param blameCache * previously calculated blames. This generator will *not* * populate it, just consume it. + * @since 7.2 */ public BlameGenerator(Repository repository, String path, @Nullable BlameCache blameCache) { @@ -282,6 +283,7 @@ public class BlameGenerator implements AutoCloseable { * Stats about this generator * * @return the stats of this generator + * @since 7.2 */ public Stats getStats() { return stats; @@ -291,9 +293,12 @@ public class BlameGenerator implements AutoCloseable { * Enable/disable the use of cache (if present). Enabled by default. *

* If caller need source line numbers, the generator cannot use the cache - * (source lines are not there). Use this method to disable the cache in that case. + * (source lines are not there). Use this method to disable the cache in + * that case. * - * @param useCache should this generator use the cache. + * @param useCache + * should this generator use the cache. + * @since 7.2 */ public void setUseCache(boolean useCache) { this.useCache = useCache; @@ -651,8 +656,8 @@ public class BlameGenerator implements AutoCloseable { List cachedBlame = blameCache.get(repository, n.sourceCommit, n.sourcePath.getPath()); if (cachedBlame != null) { - BlameRegionMerger rb = new BlameRegionMerger(repository, revPool, - cachedBlame); + BlameRegionMerger rb = new BlameRegionMerger(repository, + revPool, cachedBlame); Candidate fullyBlamed = rb.mergeCandidate(n); if (fullyBlamed != null) { stats.cacheHit = true; @@ -1204,6 +1209,8 @@ public class BlameGenerator implements AutoCloseable { /** * Stats about the work done by the generator + * + * @since 7.2 */ public static class Stats { @@ -1233,6 +1240,5 @@ public class BlameGenerator implements AutoCloseable { public boolean isCacheHit() { return cacheHit; } - } } -- cgit v1.2.3 From fab9b3d961393cd2f0e9e994b0c2827d3f3ac4ee Mon Sep 17 00:00:00 2001 From: Antonio Barone Date: Thu, 13 Feb 2025 13:21:04 +0100 Subject: Fix calculation of pack files and objects since bitmap Fix a logic issue where pack files and objects created since the most recent bitmap were incorrectly counted, ignoring their modification time. Since pack files are processed in order from most recent to oldest, we can reliably stop counting as soon as we encounter the first bitmap. By definition, all subsequent pack files are older and should not be included in the count. This ensures accurate repository statistics and prevents overcounting. Bug: jgit-140 Change-Id: I99d85fb70bc7eb42a8d24c74a1fdb8e03334099e --- .../storage/file/GcSinceBitmapStatisticsTest.java | 31 +++++++++++++++++++--- .../org/eclipse/jgit/internal/storage/file/GC.java | 10 ++++--- 2 files changed, 33 insertions(+), 8 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcSinceBitmapStatisticsTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcSinceBitmapStatisticsTest.java index 3cd766c4e9..af52e2cb85 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcSinceBitmapStatisticsTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/GcSinceBitmapStatisticsTest.java @@ -20,6 +20,7 @@ import java.util.stream.StreamSupport; import org.eclipse.jgit.internal.storage.file.GC.RepoStatistics; import org.eclipse.jgit.lib.PersonIdent; import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.storage.pack.PackConfig; import org.junit.Test; public class GcSinceBitmapStatisticsTest extends GcTestCase { @@ -51,11 +52,19 @@ public class GcSinceBitmapStatisticsTest extends GcTestCase { } @Test - public void testShouldReportNoPacksDirectlyAfterGc() throws Exception { - // given + public void testShouldReportNoPacksFilesSinceBitmapWhenPackfilesAreOlderThanBitmapFile() + throws Exception { addCommit(null); - gc.gc().get(); + configureGC(/* buildBitmap */ false).gc().get(); + assertEquals(1L, gc.getStatistics().numberOfPackFiles); + assertEquals(0L, repositoryBitmapFiles()); + assertEquals(1L, gc.getStatistics().numberOfPackFilesSinceBitmap); + + addCommit(null); + configureGC(/* buildBitmap */ true).gc().get(); + assertEquals(1L, repositoryBitmapFiles()); + assertEquals(2L, gc.getStatistics().numberOfPackFiles); assertEquals(0L, gc.getStatistics().numberOfPackFilesSinceBitmap); } @@ -79,8 +88,11 @@ public class GcSinceBitmapStatisticsTest extends GcTestCase { // progress & pack addCommit(parent); - tr.packAndPrune(); + assertEquals(1L, gc.getStatistics().numberOfPackFiles); + assertEquals(0L, gc.getStatistics().numberOfPackFilesSinceBitmap); + tr.packAndPrune(); + assertEquals(2L, gc.getStatistics().numberOfPackFiles); assertEquals(1L, gc.getStatistics().numberOfPackFilesSinceBitmap); } @@ -90,13 +102,17 @@ public class GcSinceBitmapStatisticsTest extends GcTestCase { // commit & gc RevCommit parent = addCommit(null); gc.gc().get(); + assertEquals(0L, gc.getStatistics().numberOfLooseObjects); assertEquals(0L, gc.getStatistics().numberOfObjectsSinceBitmap); // progress & pack addCommit(parent); + assertEquals(1L, gc.getStatistics().numberOfLooseObjects); assertEquals(1L, gc.getStatistics().numberOfObjectsSinceBitmap); tr.packAndPrune(); + assertEquals(0L, gc.getStatistics().numberOfLooseObjects); + // Number of objects contained in the newly created PackFile assertEquals(3L, gc.getStatistics().numberOfObjectsSinceBitmap); } @@ -164,4 +180,11 @@ public class GcSinceBitmapStatisticsTest extends GcTestCase { } }).sum(); } + + private GC configureGC(boolean buildBitmap) { + PackConfig pc = new PackConfig(repo.getObjectDatabase().getConfig()); + pc.setBuildBitmaps(buildBitmap); + gc.setPackConfig(pc); + return gc; + } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java index b33afed131..3cc43ebb04 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java @@ -1579,7 +1579,7 @@ public class GC { public RepoStatistics getStatistics() throws IOException { RepoStatistics ret = new RepoStatistics(); Collection packs = repo.getObjectDatabase().getPacks(); - long latestBitmapTime = Long.MIN_VALUE; + long latestBitmapTime = 0L; for (Pack p : packs) { long packedObjects = p.getIndex().getObjectCount(); ret.numberOfPackedObjects += packedObjects; @@ -1587,9 +1587,11 @@ public class GC { ret.sizeOfPackedObjects += p.getPackFile().length(); if (p.getBitmapIndex() != null) { ret.numberOfBitmaps += p.getBitmapIndex().getBitmapCount(); - latestBitmapTime = p.getFileSnapshot().lastModifiedInstant() - .toEpochMilli(); - } else { + if (latestBitmapTime == 0L) { + latestBitmapTime = p.getFileSnapshot().lastModifiedInstant().toEpochMilli(); + } + } + else if (latestBitmapTime == 0L) { ret.numberOfPackFilesSinceBitmap++; ret.numberOfObjectsSinceBitmap += packedObjects; } -- cgit v1.2.3 From 4aae789d285ee94414395f59a408a734a6bc6a36 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Mon, 24 Feb 2025 20:52:13 +0100 Subject: MultiPackIndexPrettyPrinter: add missing @since tag and suppress nls warnings. Change-Id: I33be306f4d5894e81fce7b2b34fdff30313417de --- .../jgit/internal/storage/midx/MultiPackIndexPrettyPrinter.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexPrettyPrinter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexPrettyPrinter.java index e8428b8fca..795d39e375 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexPrettyPrinter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexPrettyPrinter.java @@ -21,8 +21,10 @@ import org.eclipse.jgit.util.NB; /** * Prints a multipack index file in a human-readable format. + * + * @since 7.2 */ -@SuppressWarnings("nls") +@SuppressWarnings({ "boxing", "nls" }) public class MultiPackIndexPrettyPrinter { /** -- cgit v1.2.3 From 70a3131d6bb7332f50827335babb194677382d7f Mon Sep 17 00:00:00 2001 From: Ben Rohlfs Date: Thu, 20 Feb 2025 10:53:55 +0100 Subject: Update Change-Id insertion logic to insert after footers Before this change we were inserting the Change-Id at the beginning of the footer block, but after any Bug or Issue footers. After this change we are inserting the Change-Id at the end of the footer block, but before any Signed-off-by footers. The overall goal is to stay consistent with Gerrit's commit-msg hook. Change-Id: Id3a73e901ac3c289f79941d12979459c66cb7a13 --- .../tst/org/eclipse/jgit/util/ChangeIdUtilTest.java | 10 +++++++++- org.eclipse.jgit/src/org/eclipse/jgit/util/ChangeIdUtil.java | 7 ++++--- 2 files changed, 13 insertions(+), 4 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/ChangeIdUtilTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/ChangeIdUtilTest.java index e7f2049bb9..95c2fe3b51 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/ChangeIdUtilTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/ChangeIdUtilTest.java @@ -528,7 +528,7 @@ public class ChangeIdUtilTest { } @Test - public void testChangeIdAfterBugOrIssue() throws Exception { + public void testChangeIdAfterOtherFooters() throws Exception { assertEquals("a\n" + // "\n" + // "Bug: 42\n" + // @@ -548,6 +548,14 @@ public class ChangeIdUtilTest { "\n" + // "Issue: 42\n" + // SOB1)); + + assertEquals("a\n" + // + "\n" + // + "Other: none\n" + // + "Change-Id: Ide70e625dea61854206378a377dd12e462ae720f\n",// + call("a\n" + // + "\n" + // + "Other: none\n")); } @Test diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/ChangeIdUtil.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/ChangeIdUtil.java index 12af374b2e..cd169f9bb7 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/ChangeIdUtil.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/ChangeIdUtil.java @@ -86,8 +86,8 @@ public class ChangeIdUtil { } } - private static final Pattern issuePattern = Pattern - .compile("^(Bug|Issue)[a-zA-Z0-9-]*:.*$"); //$NON-NLS-1$ + private static final Pattern signedOffByPattern = Pattern + .compile("^Signed-off-by:.*$"); //$NON-NLS-1$ private static final Pattern footerPattern = Pattern .compile("(^[a-zA-Z0-9-]+:(?!//).*$)"); //$NON-NLS-1$ @@ -159,7 +159,8 @@ public class ChangeIdUtil { int footerFirstLine = indexOfFirstFooterLine(lines); int insertAfter = footerFirstLine; for (int i = footerFirstLine; i < lines.length; ++i) { - if (issuePattern.matcher(lines[i]).matches()) { + if (footerPattern.matcher(lines[i]).matches() && + !signedOffByPattern.matcher(lines[i]).matches()) { insertAfter = i + 1; continue; } -- cgit v1.2.3 From 8720b352a924e321df299e1308eca490e6aa5f68 Mon Sep 17 00:00:00 2001 From: Ben Rohlfs Date: Tue, 25 Feb 2025 10:01:37 +0100 Subject: Insert the Change-Id at the end of the footer block This is a follow-up of change 1208616. The goal is to get even closer to consistency with Gerrit's commit-msg hook. The modified test cases were all verified against what the commit-msg hook would do with the same commit message. The substantial change is that within the footer block we are putting the Change-Id also after lines matching `includeInFooterPattern`, not just after lines matching `footerPattern`. That are lines that start either with a space or with an opening bracket. Change-Id: I39305154e653f8f5adb6b75ca0f6b349b720e9d8 --- .../org/eclipse/jgit/util/ChangeIdUtilTest.java | 24 ++++++++++++++++------ .../src/org/eclipse/jgit/util/ChangeIdUtil.java | 3 +-- 2 files changed, 19 insertions(+), 8 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/ChangeIdUtilTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/ChangeIdUtilTest.java index 95c2fe3b51..44e8632228 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/ChangeIdUtilTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/ChangeIdUtilTest.java @@ -220,23 +220,23 @@ public class ChangeIdUtilTest { @Test public void testACommitWithSubjectBodyBugBrackersAndSob() throws Exception { assertEquals( - "a commit with subject body, bug. brackers and sob\n\nText\n\nBug: 33\nChange-Id: I90ecb589bef766302532c3e00915e10114b00f62\n[bracket]\nSigned-off-by: me@you.too\n", - call("a commit with subject body, bug. brackers and sob\n\nText\n\nBug: 33\n[bracket]\nSigned-off-by: me@you.too\n\n")); + "a commit with subject body, bug, brackers and sob\n\nText\n\nBug: 33\n[bracket]\nChange-Id: I94dc6ed919a4baaa7c1bf8712717b888c6b90363\nSigned-off-by: me@you.too\n", + call("a commit with subject body, bug, brackers and sob\n\nText\n\nBug: 33\n[bracket]\nSigned-off-by: me@you.too\n\n")); } @Test public void testACommitWithSubjectBodyBugLineWithASpaceAndSob() throws Exception { assertEquals( - "a commit with subject body, bug. line with a space and sob\n\nText\n\nBug: 33\nChange-Id: I864e2218bdee033c8ce9a7f923af9e0d5dc16863\n \nSigned-off-by: me@you.too\n", - call("a commit with subject body, bug. line with a space and sob\n\nText\n\nBug: 33\n \nSigned-off-by: me@you.too\n\n")); + "a commit with subject body, bug, line with a space and sob\n\nText\n\nBug: 33\n \nChange-Id: I126b472d2e0e64ad8187d61857f0169f9ccdae86\nSigned-off-by: me@you.too\n", + call("a commit with subject body, bug, line with a space and sob\n\nText\n\nBug: 33\n \nSigned-off-by: me@you.too\n\n")); } @Test public void testACommitWithSubjectBodyBugEmptyLineAndSob() throws Exception { assertEquals( - "a commit with subject body, bug. empty line and sob\n\nText\n\nBug: 33\nChange-Id: I33f119f533313883e6ada3df600c4f0d4db23a76\n \nSigned-off-by: me@you.too\n", - call("a commit with subject body, bug. empty line and sob\n\nText\n\nBug: 33\n \nSigned-off-by: me@you.too\n\n")); + "a commit with subject body, bug, empty line and sob\n\nText\n\nBug: 33\n\nChange-Id: Ic3b61b6e39a0815669b65302e9e75e6a5a019a26\nSigned-off-by: me@you.too\n", + call("a commit with subject body, bug, empty line and sob\n\nText\n\nBug: 33\n\nSigned-off-by: me@you.too\n\n")); } @Test @@ -539,6 +539,18 @@ public class ChangeIdUtilTest { "Bug: 42\n" + // SOB1)); + assertEquals("a\n" + // + "\n" + // + "Bug: 42\n" + // + " multi-line Bug footer\n" + // + "Change-Id: Icc953ef35f1a4ee5eb945132aefd603ae3d9dd9f\n" + // + SOB1,// + call("a\n" + // + "\n" + // + "Bug: 42\n" + // + " multi-line Bug footer\n" + // + SOB1)); + assertEquals("a\n" + // "\n" + // "Issue: 42\n" + // diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/ChangeIdUtil.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/ChangeIdUtil.java index cd169f9bb7..c8421d6012 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/ChangeIdUtil.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/ChangeIdUtil.java @@ -159,8 +159,7 @@ public class ChangeIdUtil { int footerFirstLine = indexOfFirstFooterLine(lines); int insertAfter = footerFirstLine; for (int i = footerFirstLine; i < lines.length; ++i) { - if (footerPattern.matcher(lines[i]).matches() && - !signedOffByPattern.matcher(lines[i]).matches()) { + if (!signedOffByPattern.matcher(lines[i]).matches()) { insertAfter = i + 1; continue; } -- cgit v1.2.3 From 68f454af418224b1ba654337c073bfb06cfb16c6 Mon Sep 17 00:00:00 2001 From: Jonathing Date: Sun, 23 Feb 2025 18:11:01 -0500 Subject: DescribeCommand: Add exclusion matches using setExclude() As of right now, the describe command in JGit only supports adding matches for tag inclusion. It does not support adding matches for excluding tags, which is something that can be done with git on the command line using the "--excludes" flag. This change adds a sister method to setMatches(), called setExcludes(), which does exactly that. A few preliminary tests have also been included in DescribeCommandTest. Change-Id: Id1449c7b83c42f1d875eabd5796c748507d69362 Signed-off-by: Jonathing --- .../org/eclipse/jgit/api/DescribeCommandTest.java | 22 +++++++++ .../src/org/eclipse/jgit/api/DescribeCommand.java | 57 +++++++++++++++++++--- 2 files changed, 72 insertions(+), 7 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/DescribeCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/DescribeCommandTest.java index ab87fa9662..060e6d3e84 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/DescribeCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/DescribeCommandTest.java @@ -12,6 +12,7 @@ package org.eclipse.jgit.api; import static java.nio.charset.StandardCharsets.UTF_8; import static org.eclipse.jgit.lib.Constants.OBJECT_ID_ABBREV_STRING_LENGTH; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; @@ -87,6 +88,9 @@ public class DescribeCommandTest extends RepositoryTestCase { assertEquals("alice-t1", describe(c2, "alice*")); assertEquals("alice-t1", describe(c2, "a*", "b*", "c*")); + assertNotEquals("alice-t1", describeExcluding(c2, "alice*")); + assertNotEquals("alice-t1", describeCommand(c2).setMatch("*").setExclude("alice*").call()); + assertEquals("bob-t2", describe(c3)); assertEquals("bob-t2-0-g44579eb", describe(c3, true, false)); assertEquals("alice-t1-1-g44579eb", describe(c3, "alice*")); @@ -95,6 +99,15 @@ public class DescribeCommandTest extends RepositoryTestCase { assertEquals("bob-t2", describe(c3, "?ob*")); assertEquals("bob-t2", describe(c3, "a*", "b*", "c*")); + assertNotEquals("alice-t1-1-g44579eb", describeExcluding(c3, "alice*")); + assertNotEquals("alice-t1-1-g44579eb", describeCommand(c3).setMatch("*").setExclude("alice*").call()); + assertNotEquals("alice-t1-1-g44579eb", describeExcluding(c3, "a??c?-t*")); + assertNotEquals("alice-t1-1-g44579eb", describeCommand(c3).setMatch("bob*").setExclude("a??c?-t*").call()); + assertNotEquals("bob-t2", describeExcluding(c3, "bob*")); + assertNotEquals("bob-t2", describeCommand(c3).setMatch("alice*").setExclude("bob*")); + assertNotEquals("bob-t2", describeExcluding(c3, "?ob*")); + assertNotEquals("bob-t2", describeCommand(c3).setMatch("a??c?-t*").setExclude("?ob*")); + // the value verified with git-describe(1) assertEquals("bob-t2-1-g3e563c5", describe(c4)); assertEquals("bob-t2-1-g3e563c5", describe(c4, true, false)); @@ -518,6 +531,15 @@ public class DescribeCommandTest extends RepositoryTestCase { .setMatch(patterns).call(); } + private String describeExcluding(ObjectId c1, String... patterns) throws Exception { + return git.describe().setTarget(c1).setTags(describeUseAllTags) + .setExclude(patterns).call(); + } + + private DescribeCommand describeCommand(ObjectId c1) throws Exception { + return git.describe().setTarget(c1).setTags(describeUseAllTags); + } + private static void assertNameStartsWith(ObjectId c4, String prefix) { assertTrue(c4.name(), c4.name().startsWith(prefix)); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/DescribeCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/DescribeCommand.java index 934245781f..898f4464fd 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/DescribeCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/DescribeCommand.java @@ -75,6 +75,11 @@ public class DescribeCommand extends GitCommand { */ private List matchers = new ArrayList<>(); + /** + * Pattern matchers to be applied to tags for exclusion. + */ + private List excludeMatchers = new ArrayList<>(); + /** * Whether to use all refs in the refs/ namespace */ @@ -263,6 +268,27 @@ public class DescribeCommand extends GitCommand { return this; } + /** + * Sets one or more {@code glob(7)} patterns that tags must not match to be + * considered. If multiple patterns are provided, they will all be applied. + * + * @param patterns + * the {@code glob(7)} pattern or patterns + * @return {@code this} + * @throws org.eclipse.jgit.errors.InvalidPatternException + * if the pattern passed in was invalid. + * @see Git documentation about describe + * @since 7.2 + */ + public DescribeCommand setExclude(String... patterns) throws InvalidPatternException { + for (String p : patterns) { + excludeMatchers.add(new FileNameMatcher(p, null)); + } + return this; + } + private final Comparator TAG_TIE_BREAKER = new Comparator<>() { @Override @@ -284,22 +310,39 @@ public class DescribeCommand extends GitCommand { private Optional getBestMatch(List tags) { if (tags == null || tags.isEmpty()) { return Optional.empty(); - } else if (matchers.isEmpty()) { + } else if (matchers.isEmpty() && excludeMatchers.isEmpty()) { Collections.sort(tags, TAG_TIE_BREAKER); return Optional.of(tags.get(0)); } else { - // Find the first tag that matches in the stream of all tags - // filtered by matchers ordered by tie break order - Stream matchingTags = Stream.empty(); - for (FileNameMatcher matcher : matchers) { - Stream m = tags.stream().filter( + Stream matchingTags; + if (!matchers.isEmpty()) { + // Find the first tag that matches in the stream of all tags + // filtered by matchers ordered by tie break order + matchingTags = Stream.empty(); + for (FileNameMatcher matcher : matchers) { + Stream m = tags.stream().filter( tag -> { matcher.append(formatRefName(tag.getName())); boolean result = matcher.isMatch(); matcher.reset(); return result; }); - matchingTags = Stream.of(matchingTags, m).flatMap(i -> i); + matchingTags = Stream.of(matchingTags, m).flatMap(i -> i); + } + } else { + // If there are no matchers, there are only excluders + // Assume all tags match for now before applying excluders + matchingTags = tags.stream(); + } + + for (FileNameMatcher matcher : excludeMatchers) { + matchingTags = matchingTags.filter( + tag -> { + matcher.append(formatRefName(tag.getName())); + boolean result = matcher.isMatch(); + matcher.reset(); + return !result; + }); } return matchingTags.sorted(TAG_TIE_BREAKER).findFirst(); } -- cgit v1.2.3 From 8501e5d6ae374f779ab60abe2ad2eaa7ed07b071 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Tue, 25 Feb 2025 10:02:34 +0100 Subject: DescribeCommand: remove unnecessary else clause and reduce indentation. Change-Id: I60a6f721eed051d67aa385a143e2bd3a950485f7 --- .../src/org/eclipse/jgit/api/DescribeCommand.java | 36 +++++++++++----------- 1 file changed, 18 insertions(+), 18 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/DescribeCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/DescribeCommand.java index 898f4464fd..d2526287f9 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/DescribeCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/DescribeCommand.java @@ -313,39 +313,39 @@ public class DescribeCommand extends GitCommand { } else if (matchers.isEmpty() && excludeMatchers.isEmpty()) { Collections.sort(tags, TAG_TIE_BREAKER); return Optional.of(tags.get(0)); - } else { - Stream matchingTags; - if (!matchers.isEmpty()) { - // Find the first tag that matches in the stream of all tags - // filtered by matchers ordered by tie break order - matchingTags = Stream.empty(); - for (FileNameMatcher matcher : matchers) { - Stream m = tags.stream().filter( + } + + Stream matchingTags; + if (!matchers.isEmpty()) { + // Find the first tag that matches in the stream of all tags + // filtered by matchers ordered by tie break order + matchingTags = Stream.empty(); + for (FileNameMatcher matcher : matchers) { + Stream m = tags.stream().filter( // tag -> { matcher.append(formatRefName(tag.getName())); boolean result = matcher.isMatch(); matcher.reset(); return result; }); - matchingTags = Stream.of(matchingTags, m).flatMap(i -> i); - } - } else { - // If there are no matchers, there are only excluders - // Assume all tags match for now before applying excluders - matchingTags = tags.stream(); + matchingTags = Stream.of(matchingTags, m).flatMap(i -> i); } + } else { + // If there are no matchers, there are only excluders + // Assume all tags match for now before applying excluders + matchingTags = tags.stream(); + } - for (FileNameMatcher matcher : excludeMatchers) { - matchingTags = matchingTags.filter( + for (FileNameMatcher matcher : excludeMatchers) { + matchingTags = matchingTags.filter( // tag -> { matcher.append(formatRefName(tag.getName())); boolean result = matcher.isMatch(); matcher.reset(); return !result; }); - } - return matchingTags.sorted(TAG_TIE_BREAKER).findFirst(); } + return matchingTags.sorted(TAG_TIE_BREAKER).findFirst(); } private ObjectId getObjectIdFromRef(Ref r) throws JGitInternalException { -- cgit v1.2.3 From adab727fdaf77d96302ef8637f14d139409221fb Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Tue, 25 Feb 2025 11:34:49 -0800 Subject: StringUtils: new #trim method In [1] we could use a "trim" function to remove leading/trailing '/' from paths. [1] https://gerrithub.io/q/I1f2a07327d1a1d8149ee482bc2529b7e1a5303db Change-Id: I490e6afe5c8e6c164d07442b1b388f8a131b4c50 --- .../tst/org/eclipse/jgit/util/StringUtilsTest.java | 19 +++++++++++ .../src/org/eclipse/jgit/util/StringUtils.java | 38 ++++++++++++++++++++++ 2 files changed, 57 insertions(+) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/StringUtilsTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/StringUtilsTest.java index 015da164c3..9a1c710752 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/StringUtilsTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/util/StringUtilsTest.java @@ -12,6 +12,7 @@ package org.eclipse.jgit.util; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; @@ -172,4 +173,22 @@ public class StringUtilsTest { assertEquals("foo bar ", StringUtils.commonPrefix("foo bar 42", "foo bar 24")); } + + @Test + public void testTrim() { + assertEquals("a", StringUtils.trim("a", '/')); + assertEquals("aaaa", StringUtils.trim("aaaa", '/')); + assertEquals("aaa", StringUtils.trim("/aaa", '/')); + assertEquals("aaa", StringUtils.trim("aaa/", '/')); + assertEquals("aaa", StringUtils.trim("/aaa/", '/')); + assertEquals("aa/aa", StringUtils.trim("/aa/aa/", '/')); + assertEquals("aa/aa", StringUtils.trim("aa/aa", '/')); + + assertEquals("", StringUtils.trim("", '/')); + assertEquals("", StringUtils.trim("/", '/')); + assertEquals("", StringUtils.trim("//", '/')); + assertEquals("", StringUtils.trim("///", '/')); + + assertNull(StringUtils.trim(null, '/')); + } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/StringUtils.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/StringUtils.java index 2fbd12dcc5..e381a3bcc9 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/StringUtils.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/StringUtils.java @@ -277,6 +277,44 @@ public final class StringUtils { return sb.toString(); } + /** + * Remove the specified character from beginning and end of a string + *

+ * If the character repeats, all copies + * + * @param str input string + * @param c character to remove + * @return the input string with c + * @since 7.2 + */ + public static String trim(String str, char c) { + if (str == null || str.length() == 0) { + return str; + } + + int endPos = str.length()-1; + while (endPos >= 0 && str.charAt(endPos) == c) { + endPos--; + } + + // Whole string is c + if (endPos == -1) { + return EMPTY; + } + + int startPos = 0; + while (startPos < endPos && str.charAt(startPos) == c) { + startPos++; + } + + if (startPos == 0 && endPos == str.length()-1) { + // No need to copy + return str; + } + + return str.substring(startPos, endPos+1); + } + /** * Appends {@link Constants#DOT_GIT_EXT} unless the given name already ends * with that suffix. -- cgit v1.2.3 From e328d203f20b8cd0a9b55678bfe3678ffd5d8179 Mon Sep 17 00:00:00 2001 From: Martin Fick Date: Thu, 13 Feb 2025 15:38:14 -0800 Subject: Do not load bitmap indexes during directory scans Previously, if a bitmap index had not been loaded yet, it would get loaded during a directory scan. Loading a bitmap file can be expensive and there is no immediate need to do so during a scan. Fix this by simply setting bitmap index file names on the Packs during directory scans so that bitmaps can be lazily loaded at some later point if they are needed. This change has the side affect of no longer marking a Pack valid if it is currently invalid simply because a bitmap file has been found, as there is no valid reason to do so and this can incorrectly mark a Pack without an index, or with other issues valid. Since the initial lack of a bitmap file, or an invalid one, or the deletion of one, would not result in the Pack being marked invalid, there is no need to overturn the invalid flag when a new bitmap file is found. Change-Id: I056acc09e7ae6a0982acd81b552d524190ebb4be Signed-off-by: Martin Fick --- .../resources/org/eclipse/jgit/internal/JGitText.properties | 2 -- .../src/org/eclipse/jgit/internal/JGitText.java | 2 -- .../src/org/eclipse/jgit/internal/storage/file/Pack.java | 13 ++----------- .../eclipse/jgit/internal/storage/file/PackDirectory.java | 9 +++------ 4 files changed, 5 insertions(+), 21 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties index 19c90086aa..c54c811b57 100644 --- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties +++ b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties @@ -64,8 +64,6 @@ binaryHunkDecodeError=Binary hunk, line {0}: invalid input binaryHunkInvalidLength=Binary hunk, line {0}: input corrupt; expected length byte, got 0x{1} binaryHunkLineTooShort=Binary hunk, line {0}: input ended prematurely binaryHunkMissingNewline=Binary hunk, line {0}: input line not terminated by newline -bitmapAccessErrorForPackfile=Error whilst trying to access bitmap file for {} -bitmapFailedToGet=Failed to get bitmap index file {} bitmapMissingObject=Bitmap at {0} is missing {1}. bitmapsMustBePrepared=Bitmaps must be prepared before they may be written. bitmapUseNoopNoListener=Use NOOP instance for no listener diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java index 700b54a7a6..b7175508ee 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java @@ -94,8 +94,6 @@ public class JGitText extends TranslationBundle { /***/ public String binaryHunkInvalidLength; /***/ public String binaryHunkLineTooShort; /***/ public String binaryHunkMissingNewline; - /***/ public String bitmapAccessErrorForPackfile; - /***/ public String bitmapFailedToGet; /***/ public String bitmapMissingObject; /***/ public String bitmapsMustBePrepared; /***/ public String bitmapUseNoopNoListener; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java index 122d800582..5813d39e9a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java @@ -116,7 +116,7 @@ public class Pack implements Iterable { private volatile Exception invalidatingCause; @Nullable - private PackFile bitmapIdxFile; + private volatile PackFile bitmapIdxFile; private AtomicInteger transientErrorCount = new AtomicInteger(); @@ -1213,17 +1213,8 @@ public class Pack implements Iterable { return null; } - synchronized void refreshBitmapIndex(PackFile bitmapIndexFile) { - this.bitmapIdx = Optionally.empty(); - this.invalid = false; + void setBitmapIndexFile(PackFile bitmapIndexFile) { this.bitmapIdxFile = bitmapIndexFile; - try { - getBitmapIndex(); - } catch (IOException e) { - LOG.warn(JGitText.get().bitmapFailedToGet, bitmapIdxFile, e); - this.bitmapIdx = Optionally.empty(); - this.bitmapIdxFile = null; - } } private synchronized PackReverseIndex getReverseIdx() throws IOException { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java index e31126f027..5cc7be4f98 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java @@ -459,12 +459,9 @@ class PackDirectory { && !oldPack.getFileSnapshot().isModified(packFile)) { forReuse.remove(packFile.getName()); list.add(oldPack); - try { - if(oldPack.getBitmapIndex() == null) { - oldPack.refreshBitmapIndex(packFilesByExt.get(BITMAP_INDEX)); - } - } catch (IOException e) { - LOG.warn(JGitText.get().bitmapAccessErrorForPackfile, oldPack.getPackName(), e); + PackFile bitMaps = packFilesByExt.get(BITMAP_INDEX); + if (bitMaps != null) { + oldPack.setBitmapIndexFile(bitMaps); } continue; } -- cgit v1.2.3 From 46d742588bb037a4a18f0a85e0e4f6834e654cd5 Mon Sep 17 00:00:00 2001 From: Thomas Wolf Date: Sat, 8 Feb 2025 20:10:47 +0100 Subject: AddCommand: implement --all/--no-all Command-line git stages deletions if file paths are given. (i.e., --all is implied.) File paths are also optional if --update or --all (or --no-all) are given. Add a setter and getter for an "all" flag on AddCommand. Check consistency with the "update" flag in call(). Make file paths optional (imply a "." path) if update is set or if setAll() had been called. If file paths are given, set the all flag. Stage deletions if update is set, or if the "all" flag is set. Add the C git command-line options for the "all" flag to jgit.pgm.Add. Bug: jgit-122 Change-Id: Iedddedcaa2d7a2e75162454ea047f34ec1cf3660 --- .../tst/org/eclipse/jgit/pgm/AddTest.java | 55 +++++++++++-- .../eclipse/jgit/pgm/internal/CLIText.properties | 5 +- .../src/org/eclipse/jgit/pgm/Add.java | 23 +++++- .../src/org/eclipse/jgit/pgm/internal/CLIText.java | 3 +- .../tst/org/eclipse/jgit/api/AddCommandTest.java | 18 ++++- .../tst/org/eclipse/jgit/lib/IndexDiffTest.java | 16 +++- .../src/org/eclipse/jgit/api/AddCommand.java | 89 ++++++++++++++++++++-- 7 files changed, 183 insertions(+), 26 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.pgm.test/tst/org/eclipse/jgit/pgm/AddTest.java b/org.eclipse.jgit.pgm.test/tst/org/eclipse/jgit/pgm/AddTest.java index 6d6374f172..a48fcbcd5a 100644 --- a/org.eclipse.jgit.pgm.test/tst/org/eclipse/jgit/pgm/AddTest.java +++ b/org.eclipse.jgit.pgm.test/tst/org/eclipse/jgit/pgm/AddTest.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2012 Google Inc. and others + * Copyright (C) 2012, 2025 Google Inc. and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -12,7 +12,10 @@ package org.eclipse.jgit.pgm; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.fail; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import java.io.File; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.dircache.DirCache; @@ -32,12 +35,7 @@ public class AddTest extends CLIRepositoryTestCase { @Test public void testAddNothing() throws Exception { - try { - execute("git add"); - fail("Must die"); - } catch (Die e) { - // expected, requires argument - } + assertThrows(Die.class, () -> execute("git add")); } @Test @@ -45,6 +43,17 @@ public class AddTest extends CLIRepositoryTestCase { execute("git add --help"); } + @Test + public void testAddInvalidOptionCombinations() throws Exception { + writeTrashFile("greeting", "Hello, world!"); + assertThrows(Die.class, () -> execute("git add -u -A greeting")); + assertThrows(Die.class, + () -> execute("git add -u --ignore-removed greeting")); + // --renormalize implies -u + assertThrows(Die.class, + () -> execute("git add --renormalize --all greeting")); + } + @Test public void testAddAFile() throws Exception { writeTrashFile("greeting", "Hello, world!"); @@ -78,4 +87,34 @@ public class AddTest extends CLIRepositoryTestCase { assertNotNull(cache.getEntry("greeting")); assertEquals(1, cache.getEntryCount()); } + + @Test + public void testAddDeleted() throws Exception { + File greeting = writeTrashFile("greeting", "Hello, world!"); + git.add().addFilepattern("greeting").call(); + DirCache cache = db.readDirCache(); + assertNotNull(cache.getEntry("greeting")); + assertEquals(1, cache.getEntryCount()); + assertTrue(greeting.delete()); + assertArrayEquals(new String[] { "" }, // + execute("git add greeting")); + + cache = db.readDirCache(); + assertEquals(0, cache.getEntryCount()); + } + + @Test + public void testAddDeleted2() throws Exception { + File greeting = writeTrashFile("greeting", "Hello, world!"); + git.add().addFilepattern("greeting").call(); + DirCache cache = db.readDirCache(); + assertNotNull(cache.getEntry("greeting")); + assertEquals(1, cache.getEntryCount()); + assertTrue(greeting.delete()); + assertArrayEquals(new String[] { "" }, // + execute("git add -A")); + + cache = db.readDirCache(); + assertEquals(0, cache.getEntryCount()); + } } diff --git a/org.eclipse.jgit.pgm/resources/org/eclipse/jgit/pgm/internal/CLIText.properties b/org.eclipse.jgit.pgm/resources/org/eclipse/jgit/pgm/internal/CLIText.properties index 80829b6638..e9630e9499 100644 --- a/org.eclipse.jgit.pgm/resources/org/eclipse/jgit/pgm/internal/CLIText.properties +++ b/org.eclipse.jgit.pgm/resources/org/eclipse/jgit/pgm/internal/CLIText.properties @@ -12,6 +12,7 @@ ARGS=ARGS # default meta variable defined in the org.kohsuke.args4j.spi.OneArgumentOptionHandler N=N +addIncompatibleOptions=--update/-u cannot be combined with --all/-A/--no-ignore-removal or --no-all/--ignore-removal. Note that --renormalize implies --update. alreadyOnBranch=Already on ''{0}'' alreadyUpToDate=Already up-to-date. answerNo=n @@ -255,7 +256,9 @@ unsupportedOperation=Unsupported operation: {0} untrackedFiles=Untracked files: updating=Updating {0}..{1} usage_Abbrev=Instead of using the default number of hexadecimal digits (which will vary according to the number of objects in the repository with a default of 7) of the abbreviated object name, use digits, or as many digits as needed to form a unique object name. An of 0 will suppress long format, only showing the closest tag. -usage_addRenormalize=Apply the "clean" process freshly to tracked files to forcibly add them again to the index. This implies -u. +usage_addRenormalize=Apply the "clean" process freshly to tracked files to forcibly add them again to the index. This implies --update/-u. +usage_addStageDeletions=Add, modify, or remove index entries to match the working tree. Cannot be used with --update/-u. +usage_addDontStageDeletions=Only add or modify index entries, but do not remove index entries for which there is no file. (Don''t stage deletions.) Cannot be used with --update/-u. usage_Aggressive=This option will cause gc to more aggressively optimize the repository at the expense of taking much more time usage_All=Pack all refs, except hidden refs, broken refs, and symbolic refs. usage_AlwaysFallback=Show uniquely abbreviated commit object as fallback diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Add.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Add.java index 2ebab5e5d2..dc9d77df35 100644 --- a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Add.java +++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/Add.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010, Sasa Zivkov and others + * Copyright (C) 2010, 2025 Sasa Zivkov and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -16,6 +16,7 @@ import java.util.List; import org.eclipse.jgit.api.AddCommand; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.pgm.internal.CLIText; import org.kohsuke.args4j.Argument; import org.kohsuke.args4j.Option; @@ -28,17 +29,33 @@ class Add extends TextBuiltin { @Option(name = "--update", aliases = { "-u" }, usage = "usage_onlyMatchAgainstAlreadyTrackedFiles") private boolean update = false; - @Argument(required = true, metaVar = "metaVar_filepattern", usage = "usage_filesToAddContentFrom") + @Option(name = "--all", aliases = { "-A", + "--no-ignore-removal" }, usage = "usage_addStageDeletions") + private Boolean all; + + @Option(name = "--no-all", aliases = { + "--ignore-removal" }, usage = "usage_addDontStageDeletions") + private void noAll(@SuppressWarnings("unused") boolean ignored) { + all = Boolean.FALSE; + } + + @Argument(metaVar = "metaVar_filepattern", usage = "usage_filesToAddContentFrom") private List filepatterns = new ArrayList<>(); @Override protected void run() throws Exception { try (Git git = new Git(db)) { - AddCommand addCmd = git.add(); if (renormalize) { update = true; } + if (update && all != null) { + throw die(CLIText.get().addIncompatibleOptions); + } + AddCommand addCmd = git.add(); addCmd.setUpdate(update).setRenormalize(renormalize); + if (all != null) { + addCmd.setAll(all.booleanValue()); + } for (String p : filepatterns) { addCmd.addFilepattern(p); } diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/internal/CLIText.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/internal/CLIText.java index b5bf6d2bc3..bb1e950542 100644 --- a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/internal/CLIText.java +++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/internal/CLIText.java @@ -1,6 +1,6 @@ /* * Copyright (C) 2010, 2013 Sasa Zivkov - * Copyright (C) 2013, 2021 Obeo and others + * Copyright (C) 2013, 2025 Obeo and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -91,6 +91,7 @@ public class CLIText extends TranslationBundle { } // @formatter:off + /***/ public String addIncompatibleOptions; /***/ public String alreadyOnBranch; /***/ public String alreadyUpToDate; /***/ public String answerNo; diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/AddCommandTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/AddCommandTest.java index 1c2e995bbb..226677229c 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/AddCommandTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/api/AddCommandTest.java @@ -1,6 +1,6 @@ /* * Copyright (C) 2010, Stefan Lay - * Copyright (C) 2010, Christian Halstrick and others + * Copyright (C) 2010, 2025 Christian Halstrick and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -665,11 +665,13 @@ public class AddCommandTest extends RepositoryTestCase { FileUtils.delete(file); // is supposed to do nothing - dc = git.add().addFilepattern("a.txt").call(); + dc = git.add().addFilepattern("a.txt").setAll(false).call(); assertEquals(oid, dc.getEntry(0).getObjectId()); assertEquals( "[a.txt, mode:100644, content:content]", indexState(CONTENT)); + git.add().addFilepattern("a.txt").call(); + assertEquals("", indexState(CONTENT)); } } @@ -690,11 +692,13 @@ public class AddCommandTest extends RepositoryTestCase { FileUtils.delete(file); // is supposed to do nothing - dc = git.add().addFilepattern("a.txt").call(); + dc = git.add().addFilepattern("a.txt").setAll(false).call(); assertEquals(oid, dc.getEntry(0).getObjectId()); assertEquals( "[a.txt, mode:100644, content:content]", indexState(CONTENT)); + git.add().addFilepattern("a.txt").call(); + assertEquals("", indexState(CONTENT)); } } @@ -964,7 +968,7 @@ public class AddCommandTest extends RepositoryTestCase { // file sub/b.txt is deleted FileUtils.delete(file2); - git.add().addFilepattern("sub").call(); + git.add().addFilepattern("sub").setAll(false).call(); // change in sub/a.txt is staged // deletion of sub/b.txt is not staged // sub/c.txt is staged @@ -973,6 +977,12 @@ public class AddCommandTest extends RepositoryTestCase { "[sub/b.txt, mode:100644, content:content b]" + "[sub/c.txt, mode:100644, content:content c]", indexState(CONTENT)); + git.add().addFilepattern("sub").call(); + // deletion of sub/b.txt is staged + assertEquals( + "[sub/a.txt, mode:100644, content:modified content]" + + "[sub/c.txt, mode:100644, content:content c]", + indexState(CONTENT)); } } diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/IndexDiffTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/IndexDiffTest.java index 2b7b6ca76c..cd98606e53 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/IndexDiffTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/lib/IndexDiffTest.java @@ -2,7 +2,7 @@ * Copyright (C) 2007, Dave Watson * Copyright (C) 2008, Robin Rosenberg * Copyright (C) 2008, Shawn O. Pearce - * Copyright (C) 2013, Robin Stocker and others + * Copyright (C) 2013, 2025 Robin Stocker and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -539,7 +539,7 @@ public class IndexDiffTest extends RepositoryTestCase { assertTrue(diff.getAssumeUnchanged().contains("file3")); assertTrue(diff.getModified().contains("file")); - git.add().addFilepattern(".").call(); + git.add().addFilepattern(".").setAll(false).call(); iterator = new FileTreeIterator(db); diff = new IndexDiff(db, Constants.HEAD, iterator); @@ -551,6 +551,18 @@ public class IndexDiffTest extends RepositoryTestCase { assertTrue(diff.getAssumeUnchanged().contains("file3")); assertTrue(diff.getChanged().contains("file")); assertEquals(Collections.EMPTY_SET, diff.getUntrackedFolders()); + + git.add().addFilepattern(".").call(); + + iterator = new FileTreeIterator(db); + diff = new IndexDiff(db, Constants.HEAD, iterator); + diff.diff(); + assertEquals(1, diff.getAssumeUnchanged().size()); + assertEquals(0, diff.getModified().size()); + assertEquals(1, diff.getChanged().size()); + assertTrue(diff.getAssumeUnchanged().contains("file2")); + assertTrue(diff.getChanged().contains("file")); + assertEquals(Collections.EMPTY_SET, diff.getUntrackedFolders()); } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/AddCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/AddCommand.java index c895dc9aaa..fd2ed37bb8 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/AddCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/AddCommand.java @@ -1,6 +1,6 @@ /* * Copyright (C) 2010, Christian Halstrick - * Copyright (C) 2010, Stefan Lay and others + * Copyright (C) 2010, 2025 Stefan Lay and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -17,6 +17,7 @@ import static org.eclipse.jgit.lib.FileMode.TYPE_TREE; import java.io.IOException; import java.io.InputStream; +import java.text.MessageFormat; import java.time.Instant; import java.util.ArrayList; import java.util.List; @@ -59,8 +60,15 @@ public class AddCommand extends GitCommand { private WorkingTreeIterator workingTreeIterator; + // Update only known index entries, don't add new ones. If there's no file + // for an index entry, remove it: stage deletions. private boolean update = false; + // If TRUE, also stage deletions, otherwise only update and add index + // entries. + // If not set explicitly + private Boolean all; + // This defaults to true because it's what JGit has been doing // traditionally. The C git default would be false. private boolean renormalize = true; @@ -82,6 +90,17 @@ public class AddCommand extends GitCommand { * A directory name (e.g. dir to add dir/file1 and * dir/file2) can also be given to add all files in the * directory, recursively. Fileglobs (e.g. *.c) are not yet supported. + *

+ *

+ * If a pattern {@code "."} is added, all changes in the git repository's + * working tree will be added. + *

+ *

+ * File patterns are required unless {@code isUpdate() == true} or + * {@link #setAll(boolean)} is called. If so and no file patterns are given, + * all changes will be added (i.e., a file pattern of {@code "."} is + * implied). + *

* * @param filepattern * repository-relative path of file/directory to add (with @@ -113,15 +132,41 @@ public class AddCommand extends GitCommand { * Executes the {@code Add} command. Each instance of this class should only * be used for one invocation of the command. Don't call this method twice * on an instance. + *

+ * + * @throws JGitInternalException + * on errors, but also if {@code isUpdate() == true} _and_ + * {@link #setAll(boolean)} had been called + * @throws NoFilepatternException + * if no file patterns are given if {@code isUpdate() == false} + * and {@link #setAll(boolean)} was not called */ @Override public DirCache call() throws GitAPIException, NoFilepatternException { - - if (filepatterns.isEmpty()) - throw new NoFilepatternException(JGitText.get().atLeastOnePatternIsRequired); checkCallable(); + + if (update && all != null) { + throw new JGitInternalException(MessageFormat.format( + JGitText.get().illegalCombinationOfArguments, + "--update", "--all/--no-all")); //$NON-NLS-1$ //$NON-NLS-2$ + } + boolean addAll; + if (filepatterns.isEmpty()) { + if (update || all != null) { + addAll = true; + } else { + throw new NoFilepatternException( + JGitText.get().atLeastOnePatternIsRequired); + } + } else { + addAll = filepatterns.contains("."); //$NON-NLS-1$ + if (all == null && !update) { + all = Boolean.TRUE; + } + } + boolean stageDeletions = update || all != null && all.booleanValue(); + DirCache dc = null; - boolean addAll = filepatterns.contains("."); //$NON-NLS-1$ try (ObjectInserter inserter = repo.newObjectInserter(); NameConflictTreeWalk tw = new NameConflictTreeWalk(repo)) { @@ -181,7 +226,8 @@ public class AddCommand extends GitCommand { if (f == null) { // working tree file does not exist if (entry != null - && (!update || GITLINK == entry.getFileMode())) { + && (!stageDeletions + || GITLINK == entry.getFileMode())) { builder.add(entry); } continue; @@ -252,7 +298,8 @@ public class AddCommand extends GitCommand { } /** - * Set whether to only match against already tracked files + * Set whether to only match against already tracked files. If + * {@code update == true}, re-sets a previous {@link #setAll(boolean)}. * * @param update * If set to true, the command only matches {@code filepattern} @@ -314,4 +361,32 @@ public class AddCommand extends GitCommand { public boolean isRenormalize() { return renormalize; } + + /** + * Defines whether the command will use '--all' mode: update existing index + * entries, add new entries, and remove index entries for which there is no + * file. (In other words: also stage deletions.) + *

+ * The setting is independent of {@link #setUpdate(boolean)}. + *

+ * + * @param all + * whether to enable '--all' mode + * @return {@code this} + * @since 7.2 + */ + public AddCommand setAll(boolean all) { + this.all = Boolean.valueOf(all); + return this; + } + + /** + * Tells whether '--all' has been set for this command. + * + * @return {@code true} if it was set; {@code false} otherwise + * @since 7.2 + */ + public boolean isAll() { + return all != null && all.booleanValue(); + } } -- cgit v1.2.3 From 04ff530a2988976ad3016c36fb43f94ca7e21af8 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Wed, 26 Feb 2025 02:16:41 +0100 Subject: AddCommand: Use parenthesis to make the operator precedence explicit Fixes errorprone error [OperatorPrecedence], see https://errorprone.info/bugpattern/OperatorPrecedence Change-Id: I3086ac0238bcf4661de6a69b1c133a4f64a3a8d4 --- org.eclipse.jgit/src/org/eclipse/jgit/api/AddCommand.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/AddCommand.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/AddCommand.java index fd2ed37bb8..b4d1cab513 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/AddCommand.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/AddCommand.java @@ -164,7 +164,7 @@ public class AddCommand extends GitCommand { all = Boolean.TRUE; } } - boolean stageDeletions = update || all != null && all.booleanValue(); + boolean stageDeletions = update || (all != null && all.booleanValue()); DirCache dc = null; -- cgit v1.2.3 From 5db57fe2a932fc05d28e057df9ff9d6a0bd6abe3 Mon Sep 17 00:00:00 2001 From: Ivan Frade Date: Wed, 26 Feb 2025 19:05:53 -0800 Subject: BlameRegionMerger: report invalid regions with checked exception. If the cached regions are invalid the merger throws an IllegalStateException. This is too strict. The caller can just continue working as if there was no cache. Report the error as IOException, that the caller can catch and handle. Change-Id: I19a1061225533b46d3a17936912a11000430f2ce --- .../tst/org/eclipse/jgit/blame/BlameRegionMergerTest.java | 2 +- org.eclipse.jgit/src/org/eclipse/jgit/blame/BlameRegionMerger.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/blame/BlameRegionMergerTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/blame/BlameRegionMergerTest.java index cf0dc452e7..1b28676fbf 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/blame/BlameRegionMergerTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/blame/BlameRegionMergerTest.java @@ -174,7 +174,7 @@ public class BlameRegionMergerTest extends RepositoryTestCase { public void blame_corruptedIndex() { Region outOfRange = new Region(0, 43, 4); // This region is out of the blamed area - assertThrows(IllegalStateException.class, + assertThrows(IOException.class, () -> blamer.mergeOneRegion(outOfRange)); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/blame/BlameRegionMerger.java b/org.eclipse.jgit/src/org/eclipse/jgit/blame/BlameRegionMerger.java index bf7e0ff736..67bc6fb789 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/blame/BlameRegionMerger.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/blame/BlameRegionMerger.java @@ -72,7 +72,7 @@ class BlameRegionMerger { List mergeOneRegion(Region region) throws IOException { List overlaps = findOverlaps(region); if (overlaps.isEmpty()) { - throw new IllegalStateException( + throw new IOException( "Cached blame should cover all lines"); } /* -- cgit v1.2.3 From 1ff9c2a1cbd5ac718559eef152d954755f257bc8 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Fri, 15 Nov 2024 08:37:10 +0100 Subject: FileReftableDatabase: consider ref updates by another process FileReftableDatabase didn't consider that refs might be changed by another process e.g. using git (which started supporting reftable with version 2.45). Add a test creating a light-weight tag which is updated using git running in another process and assert that FileReftableDatabase recognizes the tag modification. FileReftableStack#addReftable checks if the stack is up-to-date while it holds the FileLock for tables.list, if it is not up-to-date the RefUpdate fails with a LOCK_FAILURE to protect against lost ref updates if another instance of FileReftableDatabase or another thread or process updated the reftable stack since we last read it. If option `reftable.autoRefresh = true` or `setAutoRefresh(true)` was called check before each ref resolution if the reftable stack is up-to-date and, if necessary, reload the reftable stack automatically. Calling `setAutoRefresh(true)` takes precedence over the configured value for option `reftable.autoRefresh`. Add a testConcurrentRacyReload which tests that updates still abort ref updates if the reftable stack the update is based on was outdated. Bug: jgit-102 Change-Id: I1f9faa2afdbfff27e83ff295aef6d572babed4fe --- Documentation/config-options.md | 7 ++ org.eclipse.jgit.benchmarks/pom.xml | 4 + .../eclipse/jgit/benchmarks/GetRefsBenchmark.java | 15 +++ .../internal/storage/file/FileReftableTest.java | 134 ++++++++++++++++++++- .../storage/file/FileReftableDatabase.java | 54 +++++++++ .../src/org/eclipse/jgit/lib/ConfigConstants.java | 14 +++ 6 files changed, 227 insertions(+), 1 deletion(-) (limited to 'org.eclipse.jgit/src') diff --git a/Documentation/config-options.md b/Documentation/config-options.md index 807d4a8002..1dfe95f850 100644 --- a/Documentation/config-options.md +++ b/Documentation/config-options.md @@ -134,6 +134,13 @@ Proxy configuration uses the standard Java mechanisms via class `java.net.ProxyS | `pack.window` | `10` | ✅ | Number of objects to try when looking for a delta base per thread searching for deltas. | | `pack.windowMemory` | `0` (unlimited) | ✅ | Maximum number of bytes to put into the delta search window. | +## reftable options + +| option | default | git option | description | +|---------|---------|------------|-------------| +| `reftable.autoRefresh` | `false` | ⃞ | Whether to auto-refresh the reftable stack if it is out of date. | + + ## __repack__ options | option | default | git option | description | diff --git a/org.eclipse.jgit.benchmarks/pom.xml b/org.eclipse.jgit.benchmarks/pom.xml index c92fa9f156..e531245971 100644 --- a/org.eclipse.jgit.benchmarks/pom.xml +++ b/org.eclipse.jgit.benchmarks/pom.xml @@ -52,6 +52,10 @@ org.eclipse.jgit.junit ${project.version} + + junit + junit + diff --git a/org.eclipse.jgit.benchmarks/src/org/eclipse/jgit/benchmarks/GetRefsBenchmark.java b/org.eclipse.jgit.benchmarks/src/org/eclipse/jgit/benchmarks/GetRefsBenchmark.java index ea279fbba9..44e862e7c8 100644 --- a/org.eclipse.jgit.benchmarks/src/org/eclipse/jgit/benchmarks/GetRefsBenchmark.java +++ b/org.eclipse.jgit.benchmarks/src/org/eclipse/jgit/benchmarks/GetRefsBenchmark.java @@ -24,6 +24,7 @@ import java.util.stream.IntStream; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.internal.storage.file.FileReftableDatabase; import org.eclipse.jgit.internal.storage.file.FileRepository; import org.eclipse.jgit.lib.BatchRefUpdate; import org.eclipse.jgit.lib.ConfigConstants; @@ -39,8 +40,10 @@ import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.transport.ReceiveCommand; import org.eclipse.jgit.util.FS; import org.eclipse.jgit.util.FileUtils; +import org.junit.Assume; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; import org.openjdk.jmh.annotations.Measurement; import org.openjdk.jmh.annotations.Mode; import org.openjdk.jmh.annotations.OutputTimeUnit; @@ -67,6 +70,9 @@ public class GetRefsBenchmark { @Param({ "true", "false" }) boolean useRefTable; + @Param({ "true", "false" }) + boolean autoRefresh; + @Param({ "100", "1000", "10000", "100000" }) int numBranches; @@ -82,6 +88,9 @@ public class GetRefsBenchmark { @Setup @SuppressWarnings("boxing") public void setupBenchmark() throws IOException, GitAPIException { + // if we use RefDirectory skip autoRefresh = false + Assume.assumeTrue(useRefTable || autoRefresh); + String firstBranch = "firstbranch"; testDir = Files.createDirectory(Paths.get("testrepos")); String repoName = "branches-" + numBranches + "-trustStat-" @@ -98,6 +107,9 @@ public class GetRefsBenchmark { ((FileRepository) git.getRepository()).convertRefStorage( ConfigConstants.CONFIG_REF_STORAGE_REFTABLE, false, false); + FileReftableDatabase refdb = (FileReftableDatabase) git + .getRepository().getRefDatabase(); + refdb.setAutoRefresh(autoRefresh); } else { cfg.setEnum(ConfigConstants.CONFIG_CORE_SECTION, null, ConfigConstants.CONFIG_KEY_TRUST_STAT, @@ -113,6 +125,7 @@ public class GetRefsBenchmark { System.out.println("Preparing test"); System.out.println("- repository: \t\t" + repoPath); System.out.println("- refDatabase: \t\t" + refDatabaseType()); + System.out.println("- autoRefresh: \t\t" + autoRefresh); System.out.println("- trustStat: \t" + trustStat); System.out.println("- branches: \t\t" + numBranches); @@ -154,6 +167,7 @@ public class GetRefsBenchmark { @OutputTimeUnit(TimeUnit.MICROSECONDS) @Warmup(iterations = 2, time = 100, timeUnit = TimeUnit.MILLISECONDS) @Measurement(iterations = 2, time = 5, timeUnit = TimeUnit.SECONDS) + @Fork(2) public void testGetExactRef(Blackhole blackhole, BenchmarkState state) throws IOException { String branchName = state.branches @@ -166,6 +180,7 @@ public class GetRefsBenchmark { @OutputTimeUnit(TimeUnit.MICROSECONDS) @Warmup(iterations = 2, time = 100, timeUnit = TimeUnit.MILLISECONDS) @Measurement(iterations = 2, time = 5, timeUnit = TimeUnit.SECONDS) + @Fork(2) public void testGetRefsByPrefix(Blackhole blackhole, BenchmarkState state) throws IOException { String branchPrefix = "refs/heads/branch/" + branchIndex.nextInt(100) diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileReftableTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileReftableTest.java index 758ee93c30..5756b41442 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileReftableTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/FileReftableTest.java @@ -23,6 +23,7 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import static org.junit.Assume.assumeTrue; import java.io.File; import java.io.FileOutputStream; @@ -33,8 +34,15 @@ import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; import java.util.List; - import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import org.eclipse.jgit.api.Git; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.NullProgressMonitor; @@ -51,6 +59,10 @@ import org.eclipse.jgit.lib.RepositoryCache; import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.test.resources.SampleDataRepositoryTestCase; import org.eclipse.jgit.transport.ReceiveCommand; +import org.eclipse.jgit.util.FS; +import org.eclipse.jgit.util.FS.ExecutionResult; +import org.eclipse.jgit.util.RawParseUtils; +import org.eclipse.jgit.util.TemporaryBuffer; import org.junit.Test; public class FileReftableTest extends SampleDataRepositoryTestCase { @@ -64,6 +76,30 @@ public class FileReftableTest extends SampleDataRepositoryTestCase { db.convertToReftable(false, false); } + @SuppressWarnings("boxing") + @Test + public void testReloadIfNecessary() throws Exception { + ObjectId id = db.resolve("master"); + try (FileRepository repo1 = new FileRepository(db.getDirectory()); + FileRepository repo2 = new FileRepository(db.getDirectory())) { + ((FileReftableDatabase) repo1.getRefDatabase()) + .setAutoRefresh(true); + ((FileReftableDatabase) repo2.getRefDatabase()) + .setAutoRefresh(true); + FileRepository repos[] = { repo1, repo2 }; + for (int i = 0; i < 10; i++) { + for (int j = 0; j < 2; j++) { + FileRepository repo = repos[j]; + RefUpdate u = repo.getRefDatabase().newUpdate( + String.format("branch%d", i * 10 + j), false); + u.setNewObjectId(id); + RefUpdate.Result r = u.update(); + assertEquals(Result.NEW, r); + } + } + } + } + @SuppressWarnings("boxing") @Test public void testRacyReload() throws Exception { @@ -97,6 +133,54 @@ public class FileReftableTest extends SampleDataRepositoryTestCase { } } + @Test + public void testConcurrentRacyReload() throws Exception { + ObjectId id = db.resolve("master"); + final CyclicBarrier barrier = new CyclicBarrier(2); + + class UpdateRef implements Callable { + + private RefUpdate u; + + UpdateRef(FileRepository repo, String branchName) + throws IOException { + u = repo.getRefDatabase().newUpdate(branchName, + false); + u.setNewObjectId(id); + } + + @Override + public RefUpdate.Result call() throws Exception { + barrier.await(); // wait for the other thread to prepare + return u.update(); + } + } + + ExecutorService pool = Executors.newFixedThreadPool(2); + try (FileRepository repo1 = new FileRepository(db.getDirectory()); + FileRepository repo2 = new FileRepository(db.getDirectory())) { + ((FileReftableDatabase) repo1.getRefDatabase()) + .setAutoRefresh(true); + ((FileReftableDatabase) repo2.getRefDatabase()) + .setAutoRefresh(true); + for (int i = 0; i < 10; i++) { + String branchName = String.format("branch%d", + Integer.valueOf(i)); + Future ru1 = pool + .submit(new UpdateRef(repo1, branchName)); + Future ru2 = pool + .submit(new UpdateRef(repo2, branchName)); + assertTrue((ru1.get() == Result.NEW + && ru2.get() == Result.LOCK_FAILURE) + || (ru1.get() == Result.LOCK_FAILURE + && ru2.get() == Result.NEW)); + } + } finally { + pool.shutdown(); + pool.awaitTermination(Long.MAX_VALUE, TimeUnit.SECONDS); + } + } + @Test public void testCompactFully() throws Exception { ObjectId c1 = db.resolve("master^^"); @@ -651,6 +735,54 @@ public class FileReftableTest extends SampleDataRepositoryTestCase { checkContainsRef(refs, db.exactRef("HEAD")); } + @Test + public void testExternalUpdate_bug_102() throws Exception { + ((FileReftableDatabase) db.getRefDatabase()).setAutoRefresh(true); + assumeTrue(atLeastGitVersion(2, 45)); + Git git = Git.wrap(db); + git.tag().setName("foo").call(); + Ref ref = db.exactRef("refs/tags/foo"); + assertNotNull(ref); + runGitCommand("tag", "--force", "foo", "e"); + Ref e = db.exactRef("refs/heads/e"); + Ref foo = db.exactRef("refs/tags/foo"); + assertEquals(e.getObjectId(), foo.getObjectId()); + } + + private String toString(TemporaryBuffer b) throws IOException { + return RawParseUtils.decode(b.toByteArray()); + } + + private ExecutionResult runGitCommand(String... args) + throws IOException, InterruptedException { + FS fs = db.getFS(); + ProcessBuilder pb = fs.runInShell("git", args); + pb.directory(db.getWorkTree()); + System.err.println("PATH=" + pb.environment().get("PATH")); + ExecutionResult result = fs.execute(pb, null); + assertEquals(0, result.getRc()); + String err = toString(result.getStderr()); + if (!err.isEmpty()) { + System.err.println(err); + } + String out = toString(result.getStdout()); + if (!out.isEmpty()) { + System.out.println(out); + } + return result; + } + + private boolean atLeastGitVersion(int minMajor, int minMinor) + throws IOException, InterruptedException { + String version = toString(runGitCommand("version").getStdout()) + .split(" ")[2]; + System.out.println(version); + String[] digits = version.split("\\."); + int major = Integer.parseInt(digits[0]); + int minor = Integer.parseInt(digits[1]); + return (major >= minMajor) && (minor >= minMinor); + } + private RefUpdate updateRef(String name) throws IOException { final RefUpdate ref = db.updateRef(name); ref.setNewObjectId(db.resolve(Constants.HEAD)); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java index 6040ad554d..d70fa839e9 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java @@ -16,6 +16,7 @@ import static org.eclipse.jgit.lib.Ref.Storage.PACKED; import java.io.File; import java.io.IOException; +import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; @@ -37,6 +38,7 @@ import org.eclipse.jgit.internal.storage.reftable.ReftableBatchRefUpdate; import org.eclipse.jgit.internal.storage.reftable.ReftableDatabase; import org.eclipse.jgit.internal.storage.reftable.ReftableWriter; import org.eclipse.jgit.lib.BatchRefUpdate; +import org.eclipse.jgit.lib.ConfigConstants; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectIdRef; @@ -70,6 +72,8 @@ public class FileReftableDatabase extends RefDatabase { private final FileReftableStack reftableStack; + private boolean autoRefresh; + FileReftableDatabase(FileRepository repo) throws IOException { this(repo, new File(new File(repo.getCommonDirectory(), Constants.REFTABLE), Constants.TABLES_LIST)); @@ -77,6 +81,9 @@ public class FileReftableDatabase extends RefDatabase { FileReftableDatabase(FileRepository repo, File refstackName) throws IOException { this.fileRepository = repo; + this.autoRefresh = repo.getConfig().getBoolean( + ConfigConstants.CONFIG_REFTABLE_SECTION, + ConfigConstants.CONFIG_KEY_AUTOREFRESH, false); this.reftableStack = new FileReftableStack(refstackName, new File(fileRepository.getCommonDirectory(), Constants.REFTABLE), () -> fileRepository.fireEvent(new RefsChangedEvent()), @@ -183,6 +190,7 @@ public class FileReftableDatabase extends RefDatabase { @Override public Ref exactRef(String name) throws IOException { + autoRefresh(); return reftableDatabase.exactRef(name); } @@ -193,6 +201,7 @@ public class FileReftableDatabase extends RefDatabase { @Override public Map getRefs(String prefix) throws IOException { + autoRefresh(); List refs = reftableDatabase.getRefsByPrefix(prefix); RefList.Builder builder = new RefList.Builder<>(refs.size()); for (Ref r : refs) { @@ -205,6 +214,7 @@ public class FileReftableDatabase extends RefDatabase { @Override public List getRefsByPrefixWithExclusions(String include, Set excludes) throws IOException { + autoRefresh(); return reftableDatabase.getRefsByPrefixWithExclusions(include, excludes); } @@ -223,6 +233,50 @@ public class FileReftableDatabase extends RefDatabase { } + /** + * Whether to auto-refresh the reftable stack if it is out of date. + * + * @param autoRefresh + * whether to auto-refresh the reftable stack if it is out of + * date. + */ + public void setAutoRefresh(boolean autoRefresh) { + this.autoRefresh = autoRefresh; + } + + /** + * Whether the reftable stack is auto-refreshed if it is out of date. + * + * @return whether the reftable stack is auto-refreshed if it is out of + * date. + */ + public boolean isAutoRefresh() { + return autoRefresh; + } + + private void autoRefresh() { + if (autoRefresh) { + refresh(); + } + } + + /** + * Check if the reftable stack is up to date, and if not, reload it. + *

+ * {@inheritDoc} + */ + @Override + public void refresh() { + try { + if (!reftableStack.isUpToDate()) { + reftableDatabase.clearCache(); + reftableStack.reload(); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + private Ref doPeel(Ref leaf) throws IOException { try (RevWalk rw = new RevWalk(fileRepository)) { RevObject obj = rw.parseAny(leaf.getObjectId()); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java index 30e7de47c3..f62bf161a7 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java @@ -1093,4 +1093,18 @@ public final class ConfigConstants { * @since 7.1 */ public static final String CONFIG_KEY_LOAD_REV_INDEX_IN_PARALLEL = "loadRevIndexInParallel"; + + /** + * The "reftable" section + * + * @since 7.2 + */ + public static final String CONFIG_REFTABLE_SECTION = "reftable"; + + /** + * The "autorefresh" key + * + * @since 7.2 + */ + public static final String CONFIG_KEY_AUTOREFRESH = "autorefresh"; } -- cgit v1.2.3 From ac5146ffbf9d331e852059b2eb14841927f301a1 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Wed, 8 Jan 2025 18:38:31 +0100 Subject: FileReftableStack: use FileSnapshot to detect modification Reading file attributes is faster than reading file content hence use FileSnapshot to speedup detecting if FileReftableStack is up-to-date. Introduce new option "core.trustTablesListStat" allowing to configure if we can trust file attributes of the "tables.list" file to speedup detection of file modifications. This file is used to store the list of filenames of the files storing Reftables in FileReftableDatabase. If this option is set to "ALWAYS" we trust File attributes and use them to speedup detection of file modifications. If set to "NEVER" the content of the "tables.list" file is always read unconditionally. This can help to avoid caching issues on some filesystems. If set to "AFTER_OPEN" we will open a FileInputStream to refresh File attributes of the "tables.list" file before relying on the refreshed File attributes to detect modifications. This works on some NFS filesystems and is faster than using "NEVER". Change-Id: I3e288d90fb07edf4fa2a03c707a333b26f0c458d --- Documentation/config-options.md | 3 +- .../internal/storage/file/FileReftableStack.java | 40 ++++++++++++++++++++-- .../src/org/eclipse/jgit/lib/ConfigConstants.java | 8 +++++ .../src/org/eclipse/jgit/lib/CoreConfig.java | 24 +++++++++++++ 4 files changed, 71 insertions(+), 4 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/Documentation/config-options.md b/Documentation/config-options.md index 1dfe95f850..4dde8f8c15 100644 --- a/Documentation/config-options.md +++ b/Documentation/config-options.md @@ -58,9 +58,10 @@ For details on native git options see also the official [git config documentatio | ~~`core.trustFolderStat`~~ | `true` | ⃞ | __Deprecated__, use `core.trustStat` instead. If set to `true` translated to `core.trustStat=always`, if `false` translated to `core.trustStat=never`, see below. If both `core.trustFolderStat` and `core.trustStat` are configured then `trustStat` takes precedence and `trustFolderStat` is ignored. | | `core.trustLooseRefStat` | `inherit` | ⃞ | Whether to trust the file attributes of loose refs and its fan-out parent directory. See `core.trustStat` for possible values. If `inherit`, JGit will use the behavior configured in `trustStat`. | | `core.trustPackedRefsStat` | `inherit` | ⃞ | Whether to trust the file attributes of the packed-refs file. See `core.trustStat` for possible values. If `inherit`, JGit will use the behavior configured in `core.trustStat`. | +| `core.trustTablesListStat` | `inherit` | ⃞ | Whether to trust the file attributes of the `tables.list` file used by the reftable ref storage backend to store the list of reftable filenames. See `core.trustStat` for possible values. If `inherit`, JGit will use the behavior configured in `core.trustStat`. The reftable backend is used if `extensions.refStorage = reftable`. | | `core.trustLooseObjectStat` | `inherit` | ⃞ | Whether to trust the file attributes of the loose object file and its fan-out parent directory. See `core.trustStat` for possible values. If `inherit`, JGit will use the behavior configured in `core.trustStat`. | | `core.trustPackStat` | `inherit` | ⃞ | Whether to trust the file attributes of the `objects/pack` directory. See `core.trustStat` for possible values. If `inherit`, JGit will use the behavior configured in `core.trustStat`. | -| `core.trustStat` | `always` | ⃞ | Global option to configure whether to trust file attributes (Java equivalent of stat command on Unix) of files storing git objects. Can be overridden for specific files by configuring `core.trustLooseRefStat, core.trustPackedRefsStat, core.trustLooseObjectStat, core.trustPackStat`. If `never` JGit will ignore the file attributes of the file and always read it. If `always` JGit will trust the file attributes and will only read it if a file attribute has changed. `after_open` behaves the same as `always`, but file attributes are only considered *after* the file itself and any transient parent directories have been opened and closed. An open/close of the file/directory is known to refresh its file attributes, at least on some NFS clients. | +| `core.trustStat` | `always` | ⃞ | Global option to configure whether to trust file attributes (Java equivalent of stat command on Unix) of files storing git objects. Can be overridden for specific files by configuring `core.trustLooseRefStat, core.trustPackedRefsStat, core.trustLooseObjectStat, core.trustPackStat,core.trustTablesListStat`. If `never` JGit will ignore the file attributes of the file and always read it. If `always` JGit will trust the file attributes and will only read it if a file attribute has changed. `after_open` behaves the same as `always`, but file attributes are only considered *after* the file itself and any transient parent directories have been opened and closed. An open/close of the file/directory is known to refresh its file attributes, at least on some NFS clients. | | `core.worktree` | Root directory of the working tree if it is not the parent directory of the `.git` directory | ✅ | The path to the root of the working tree. | ## __fetch__ options diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableStack.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableStack.java index 0f5ff0f9f7..b2c88922b8 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableStack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableStack.java @@ -18,8 +18,10 @@ import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStream; import java.io.InputStreamReader; import java.nio.file.Files; +import java.nio.file.NoSuchFileException; import java.nio.file.StandardCopyOption; import java.security.SecureRandom; import java.util.ArrayList; @@ -27,6 +29,7 @@ import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -39,6 +42,8 @@ import org.eclipse.jgit.internal.storage.reftable.ReftableConfig; import org.eclipse.jgit.internal.storage.reftable.ReftableReader; import org.eclipse.jgit.internal.storage.reftable.ReftableWriter; import org.eclipse.jgit.lib.Config; +import org.eclipse.jgit.lib.CoreConfig; +import org.eclipse.jgit.lib.CoreConfig.TrustStat; import org.eclipse.jgit.util.FileUtils; import org.eclipse.jgit.util.SystemReader; @@ -59,6 +64,9 @@ public class FileReftableStack implements AutoCloseable { private List stack; + private AtomicReference snapshot = new AtomicReference<>( + FileSnapshot.DIRTY); + private long lastNextUpdateIndex; private final File stackPath; @@ -98,6 +106,8 @@ public class FileReftableStack implements AutoCloseable { private final CompactionStats stats; + private final TrustStat trustTablesListStat; + /** * Creates a stack corresponding to the list of reftables in the argument * @@ -126,6 +136,8 @@ public class FileReftableStack implements AutoCloseable { reload(); stats = new CompactionStats(); + trustTablesListStat = configSupplier.get().get(CoreConfig.KEY) + .getTrustTablesListStat(); } CompactionStats getStats() { @@ -272,8 +284,9 @@ public class FileReftableStack implements AutoCloseable { } private List readTableNames() throws IOException { + FileSnapshot old; List names = new ArrayList<>(stack.size() + 1); - + old = snapshot.get(); try (BufferedReader br = new BufferedReader( new InputStreamReader(new FileInputStream(stackPath), UTF_8))) { String line; @@ -282,8 +295,10 @@ public class FileReftableStack implements AutoCloseable { names.add(line); } } + snapshot.compareAndSet(old, FileSnapshot.save(stackPath)); } catch (FileNotFoundException e) { // file isn't there: empty repository. + snapshot.compareAndSet(old, FileSnapshot.MISSING_FILE); } return names; } @@ -294,9 +309,28 @@ public class FileReftableStack implements AutoCloseable { * on IO problem */ boolean isUpToDate() throws IOException { - // We could use FileSnapshot to avoid reading the file, but the file is - // small so it's probably a minor optimization. try { + switch (trustTablesListStat) { + case NEVER: + break; + case AFTER_OPEN: + try (InputStream stream = Files + .newInputStream(stackPath.toPath())) { + // open the tables.list file to refresh attributes (on some + // NFS clients) + } catch (FileNotFoundException | NoSuchFileException e) { + // ignore + } + //$FALL-THROUGH$ + case ALWAYS: + if (!snapshot.get().isModified(stackPath)) { + return true; + } + break; + case INHERIT: + // only used in CoreConfig internally + throw new IllegalStateException(); + } List names = readTableNames(); if (names.size() != stack.size()) { return false; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java index f62bf161a7..c4550329d3 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/ConfigConstants.java @@ -1045,6 +1045,14 @@ public final class ConfigConstants { * @since 7.2 */ public static final String CONFIG_KEY_TRUST_LOOSE_OBJECT_STAT = "trustLooseObjectStat"; + + /** + * The "trustTablesListStat" key + * + * @since 7.2 + */ + public static final String CONFIG_KEY_TRUST_TABLESLIST_STAT = "trustTablesListStat"; + /** * The "pack.preserveOldPacks" key * diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/CoreConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/CoreConfig.java index e43c9653dd..0e27b2743c 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/CoreConfig.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/CoreConfig.java @@ -215,6 +215,8 @@ public class CoreConfig { private final TrustStat trustLooseObjectStat; + private final TrustStat trustTablesListStat; + /** * Options for symlink handling * @@ -268,6 +270,7 @@ public class CoreConfig { trustLooseRefStat = parseTrustLooseRefStat(rc); trustPackStat = parseTrustPackFileStat(rc); trustLooseObjectStat = parseTrustLooseObjectFileStat(rc); + trustTablesListStat = parseTablesListStat(rc); } private static TrustStat parseTrustStat(Config rc) { @@ -318,6 +321,13 @@ public class CoreConfig { return t == TrustStat.INHERIT ? trustStat : t; } + private TrustStat parseTablesListStat(Config rc) { + TrustStat t = rc.getEnum(ConfigConstants.CONFIG_CORE_SECTION, null, + ConfigConstants.CONFIG_KEY_TRUST_TABLESLIST_STAT, + TrustStat.INHERIT); + return t == TrustStat.INHERIT ? trustStat : t; + } + /** * Get the compression level to use when storing loose objects * @@ -418,4 +428,18 @@ public class CoreConfig { public TrustStat getTrustLooseObjectStat() { return trustLooseObjectStat; } + + /** + * Get how far we can trust file attributes of the "tables.list" file which + * is used to store the list of filenames of the files storing + * {@link org.eclipse.jgit.internal.storage.reftable.Reftable}s in + * {@link org.eclipse.jgit.internal.storage.file.FileReftableDatabase}. + * + * @return how far we can trust file attributes of the "tables.list" file. + * + * @since 7.2 + */ + public TrustStat getTrustTablesListStat() { + return trustTablesListStat; + } } -- cgit v1.2.3 From 4ef88700db1da4ab07917b084057769a43735682 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Mon, 3 Mar 2025 22:17:30 +0100 Subject: Ensure access to autoRefresh is thread-safe Change-Id: I7651613c33803daf00882a543dbf0c3f836110fa --- .../jgit/internal/storage/file/FileReftableDatabase.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java index d70fa839e9..e9782e2e18 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeSet; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.stream.Collectors; @@ -72,7 +73,7 @@ public class FileReftableDatabase extends RefDatabase { private final FileReftableStack reftableStack; - private boolean autoRefresh; + private final AtomicBoolean autoRefresh; FileReftableDatabase(FileRepository repo) throws IOException { this(repo, new File(new File(repo.getCommonDirectory(), Constants.REFTABLE), @@ -81,9 +82,9 @@ public class FileReftableDatabase extends RefDatabase { FileReftableDatabase(FileRepository repo, File refstackName) throws IOException { this.fileRepository = repo; - this.autoRefresh = repo.getConfig().getBoolean( + this.autoRefresh = new AtomicBoolean(repo.getConfig().getBoolean( ConfigConstants.CONFIG_REFTABLE_SECTION, - ConfigConstants.CONFIG_KEY_AUTOREFRESH, false); + ConfigConstants.CONFIG_KEY_AUTOREFRESH, false)); this.reftableStack = new FileReftableStack(refstackName, new File(fileRepository.getCommonDirectory(), Constants.REFTABLE), () -> fileRepository.fireEvent(new RefsChangedEvent()), @@ -241,7 +242,7 @@ public class FileReftableDatabase extends RefDatabase { * date. */ public void setAutoRefresh(boolean autoRefresh) { - this.autoRefresh = autoRefresh; + this.autoRefresh.set(autoRefresh); } /** @@ -251,11 +252,11 @@ public class FileReftableDatabase extends RefDatabase { * date. */ public boolean isAutoRefresh() { - return autoRefresh; + return autoRefresh.get(); } private void autoRefresh() { - if (autoRefresh) { + if (autoRefresh.get()) { refresh(); } } -- cgit v1.2.3 From 28136bc499832d6031e1b99d98d0328c17d469d3 Mon Sep 17 00:00:00 2001 From: Matthias Sohn Date: Tue, 4 Mar 2025 09:05:34 +0100 Subject: CacheRegion: fix non translatable text warnings Change-Id: I163957653b075f1f05a6219f4d23b340588ffcbd --- .../resources/org/eclipse/jgit/internal/JGitText.properties | 1 + .../src/org/eclipse/jgit/blame/cache/CacheRegion.java | 8 ++++++-- org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java | 1 + 3 files changed, 8 insertions(+), 2 deletions(-) (limited to 'org.eclipse.jgit/src') diff --git a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties index f8c8ecaeab..27270a1f25 100644 --- a/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties +++ b/org.eclipse.jgit/resources/org/eclipse/jgit/internal/JGitText.properties @@ -76,6 +76,7 @@ branchNameInvalid=Branch name {0} is not allowed buildingBitmaps=Building bitmaps cachedPacksPreventsIndexCreation=Using cached packs prevents index creation cachedPacksPreventsListingObjects=Using cached packs prevents listing objects +cacheRegionAllOrNoneNull=expected all null or none: {0}, {1} cannotAccessLastModifiedForSafeDeletion=Unable to access lastModifiedTime of file {0}, skip deletion since we cannot safely avoid race condition cannotBeCombined=Cannot be combined. cannotBeRecursiveWhenTreesAreIncluded=TreeWalk shouldn't be recursive when tree objects are included. diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/blame/cache/CacheRegion.java b/org.eclipse.jgit/src/org/eclipse/jgit/blame/cache/CacheRegion.java index 6aa4eef2ca..cf3f978044 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/blame/cache/CacheRegion.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/blame/cache/CacheRegion.java @@ -9,6 +9,9 @@ */ package org.eclipse.jgit.blame.cache; +import java.text.MessageFormat; + +import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.lib.ObjectId; /** @@ -92,6 +95,7 @@ public class CacheRegion implements Comparable { return start - o.start; } + @SuppressWarnings("nls") @Override public String toString() { StringBuilder sb = new StringBuilder(); @@ -115,7 +119,7 @@ public class CacheRegion implements Comparable { if (path == null && commit == null) { return; } - throw new IllegalArgumentException(String.format( - "expected all null or none: %s, %s", path, commit)); + throw new IllegalArgumentException(MessageFormat + .format(JGitText.get().cacheRegionAllOrNoneNull, path, commit)); } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java index 91ce3479e2..bf252f9968 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java @@ -106,6 +106,7 @@ public class JGitText extends TranslationBundle { /***/ public String buildingBitmaps; /***/ public String cachedPacksPreventsIndexCreation; /***/ public String cachedPacksPreventsListingObjects; + /***/ public String cacheRegionAllOrNoneNull; /***/ public String cannotAccessLastModifiedForSafeDeletion; /***/ public String cannotBeCombined; /***/ public String cannotBeRecursiveWhenTreesAreIncluded; -- cgit v1.2.3