diff options
Diffstat (limited to 'org.eclipse.jgit/src/org')
35 files changed, 915 insertions, 323 deletions
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/api/Git.java b/org.eclipse.jgit/src/org/eclipse/jgit/api/Git.java index 5bc035a46a..a8b866d25b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/api/Git.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/api/Git.java @@ -201,7 +201,24 @@ public class Git implements AutoCloseable { this(repo, false); } - Git(Repository repo, boolean closeRepo) { + /** + * Construct a new {@link org.eclipse.jgit.api.Git} object which can + * interact with the specified git repository. + * <p> + * All command classes returned by methods of this class will always + * interact with this git repository. + * <p> + * If {@code closeRepo = false} the caller is responsible for closing the + * repository. + * + * @param repo + * the git repository this class is interacting with; + * {@code null} is not allowed. + * @param closeRepo + * whether to close the repository when this instance is closed + * @since 7.4 + */ + public Git(Repository repo, boolean closeRepo) { this.repo = requireNonNull(repo); this.closeRepo = closeRepo; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/ManifestParser.java b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/ManifestParser.java index b033177e05..58b4d3dc56 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/ManifestParser.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/gitrepo/ManifestParser.java @@ -142,7 +142,17 @@ public class ManifestParser extends DefaultHandler { xmlInRead++; final XMLReader xr; try { - xr = SAXParserFactory.newInstance().newSAXParser().getXMLReader(); + SAXParserFactory spf = SAXParserFactory.newInstance(); + spf.setFeature( + "http://xml.org/sax/features/external-general-entities", //$NON-NLS-1$ + false); + spf.setFeature( + "http://xml.org/sax/features/external-parameter-entities", //$NON-NLS-1$ + false); + spf.setFeature( + "http://apache.org/xml/features/disallow-doctype-decl", //$NON-NLS-1$ + true); + xr = spf.newSAXParser().getXMLReader(); } catch (SAXException | ParserConfigurationException e) { throw new IOException(JGitText.get().noXMLParserAvailable, e); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsInserter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsInserter.java index 16315bf4f2..dd9e4b96a4 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsInserter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsInserter.java @@ -83,7 +83,6 @@ public class DfsInserter extends ObjectInserter { DfsPackDescription packDsc; PackStream packOut; private boolean rollback; - private boolean checkExisting = true; /** * Initialize a new inserter. @@ -98,18 +97,6 @@ public class DfsInserter extends ObjectInserter { ConfigConstants.CONFIG_KEY_MIN_BYTES_OBJ_SIZE_INDEX, -1); } - /** - * Check existence - * - * @param check - * if {@code false}, will write out possibly-duplicate objects - * without first checking whether they exist in the repo; default - * is true. - */ - public void checkExisting(boolean check) { - checkExisting = check; - } - void setCompressionLevel(int compression) { this.compression = compression; } @@ -130,8 +117,9 @@ public class DfsInserter extends ObjectInserter { if (objectMap != null && objectMap.contains(id)) return id; // Ignore unreachable (garbage) objects here. - if (checkExisting && db.has(id, true)) + if (db.has(id, true)) { return id; + } long offset = beginObject(type, len); packOut.compress.write(data, off, len); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java index efd666ff27..1a873d1204 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsObjDatabase.java @@ -52,16 +52,6 @@ public abstract class DfsObjDatabase extends ObjectDatabase { boolean dirty() { return true; } - - @Override - void clearDirty() { - // Always dirty. - } - - @Override - public void markDirty() { - // Always dirty. - } }; /** @@ -534,7 +524,7 @@ public abstract class DfsObjDatabase extends ObjectDatabase { DfsPackFile[] packs = new DfsPackFile[1 + o.packs.length]; packs[0] = newPack; System.arraycopy(o.packs, 0, packs, 1, o.packs.length); - n = new PackListImpl(packs, o.reftables); + n = new PackList(packs, o.reftables); } while (!packList.compareAndSet(o, n)); } @@ -559,7 +549,7 @@ public abstract class DfsObjDatabase extends ObjectDatabase { } } tables.add(new DfsReftable(add)); - n = new PackListImpl(o.packs, tables.toArray(new DfsReftable[0])); + n = new PackList(o.packs, tables.toArray(new DfsReftable[0])); } while (!packList.compareAndSet(o, n)); } @@ -613,13 +603,12 @@ public abstract class DfsObjDatabase extends ObjectDatabase { } if (newPacks.isEmpty() && newReftables.isEmpty()) - return new PackListImpl(NO_PACKS.packs, NO_PACKS.reftables); + return new PackList(NO_PACKS.packs, NO_PACKS.reftables); if (!foundNew) { - old.clearDirty(); return old; } Collections.sort(newReftables, reftableComparator()); - return new PackListImpl( + return new PackList( newPacks.toArray(new DfsPackFile[0]), newReftables.toArray(new DfsReftable[0])); } @@ -685,7 +674,7 @@ public abstract class DfsObjDatabase extends ObjectDatabase { } /** Snapshot of packs scanned in a single pass. */ - public abstract static class PackList { + public static class PackList { /** All known packs, sorted. */ public final DfsPackFile[] packs; @@ -715,39 +704,8 @@ public abstract class DfsObjDatabase extends ObjectDatabase { return lastModified; } - abstract boolean dirty(); - abstract void clearDirty(); - - /** - * Mark pack list as dirty. - * <p> - * Used when the caller knows that new data might have been written to the - * repository that could invalidate open readers depending on this pack list, - * for example if refs are newly scanned. - */ - public abstract void markDirty(); - } - - private static final class PackListImpl extends PackList { - private volatile boolean dirty; - - PackListImpl(DfsPackFile[] packs, DfsReftable[] reftables) { - super(packs, reftables); - } - - @Override boolean dirty() { - return dirty; - } - - @Override - void clearDirty() { - dirty = false; - } - - @Override - public void markDirty() { - dirty = true; + return false; } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java index f9c01b9d6e..6339b0326a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackCompactor.java @@ -405,7 +405,7 @@ public class DfsPackCompactor { pw.addObject(obj); obj.add(added); - src.representation(rep, id.offset, ctx, rev); + src.fillRepresentation(rep, id.offset, ctx, rev); if (rep.getFormat() != PACK_DELTA) continue; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java index 9a95ddc370..05b63eaca1 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFile.java @@ -27,6 +27,9 @@ import java.io.InputStream; import java.nio.ByteBuffer; import java.nio.channels.Channels; import java.text.MessageFormat; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; @@ -49,6 +52,7 @@ import org.eclipse.jgit.internal.storage.file.PackObjectSizeIndexLoader; import org.eclipse.jgit.internal.storage.file.PackReverseIndex; import org.eclipse.jgit.internal.storage.file.PackReverseIndexFactory; import org.eclipse.jgit.internal.storage.pack.BinaryDelta; +import org.eclipse.jgit.internal.storage.pack.ObjectToPack; import org.eclipse.jgit.internal.storage.pack.PackOutputStream; import org.eclipse.jgit.internal.storage.pack.StoredObjectRepresentation; import org.eclipse.jgit.lib.AbbreviatedObjectId; @@ -59,6 +63,7 @@ import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectLoader; import org.eclipse.jgit.lib.Repository; import org.eclipse.jgit.lib.StoredConfig; +import org.eclipse.jgit.util.BlockList; import org.eclipse.jgit.util.LongList; /** @@ -71,6 +76,10 @@ public final class DfsPackFile extends BlockBasedFile { private static final long REF_POSITION = 0; + private static final Comparator<DfsObjectToPack> OFFSET_SORT = ( + DfsObjectToPack a, + DfsObjectToPack b) -> Long.signum(a.getOffset() - b.getOffset()); + /** * Loader for the default file-based {@link PackBitmapIndex} implementation. */ @@ -433,6 +442,10 @@ public final class DfsPackFile extends BlockBasedFile { return 0 < offset && !isCorrupt(offset); } + int findIdxPosition(DfsReader ctx, AnyObjectId id) throws IOException { + return idx(ctx).findPosition(id); + } + /** * Get an object from this pack. * @@ -455,23 +468,43 @@ public final class DfsPackFile extends BlockBasedFile { return idx(ctx).findOffset(id); } - void resolve(DfsReader ctx, Set<ObjectId> matches, AbbreviatedObjectId id, - int matchLimit) throws IOException { - idx(ctx).resolve(matches, id, matchLimit); - } - /** - * Obtain the total number of objects available in this pack. This method - * relies on pack index, giving number of effectively available objects. + * Return objects in the list available in this pack, sorted in (pack, + * offset) order. * * @param ctx - * current reader for the calling thread. - * @return number of objects in index of this pack, likewise in this pack + * a reader + * @param objects + * objects we are looking for + * @param skipFound + * ignore objects already found. + * @return list of objects with pack and offset set. * @throws IOException - * the index file cannot be loaded into memory. + * an error occurred */ - long getObjectCount(DfsReader ctx) throws IOException { - return idx(ctx).getObjectCount(); + List<DfsObjectToPack> findAllFromPack(DfsReader ctx, + Iterable<ObjectToPack> objects, boolean skipFound) + throws IOException { + List<DfsObjectToPack> tmp = new BlockList<>(); + for (ObjectToPack obj : objects) { + DfsObjectToPack otp = (DfsObjectToPack) obj; + if (skipFound && otp.isFound()) { + continue; + } + long p = idx(ctx).findOffset(otp); + if (p <= 0 || isCorrupt(p)) { + continue; + } + otp.setOffset(p); + tmp.add(otp); + } + Collections.sort(tmp, OFFSET_SORT); + return tmp; + } + + void resolve(DfsReader ctx, Set<ObjectId> matches, AbbreviatedObjectId id, + int matchLimit) throws IOException { + idx(ctx).resolve(matches, id, matchLimit); } private byte[] decompress(long position, int sz, DfsReader ctx) @@ -1135,31 +1168,29 @@ public final class DfsPackFile extends BlockBasedFile { /** * Return the size of the object from the object-size index. The object * should be a blob. Any other type is not indexed and returns -1. - * - * Caller MUST be sure that the object is in the pack (e.g. with - * {@link #hasObject(DfsReader, AnyObjectId)}) and the pack has object size - * index (e.g. with {@link #hasObjectSizeIndex(DfsReader)}) before asking - * the indexed size. + * <p> + * Caller MUST pass a valid index position, as returned by + * {@link #findIdxPosition(DfsReader, AnyObjectId)} and verify the pack has + * object size index (e.g. with {@link #hasObjectSizeIndex(DfsReader)}) + * before asking the indexed size. * * @param ctx * reader context to support reading from the backing store if * the object size index is not already loaded in memory. - * @param id - * object id of an object in the pack + * @param idxPosition + * position in the primary index of the object we are looking + * for, as returned by findIdxPosition * @return size of the object from the index. Negative if object is not in * the index (below threshold or not a blob) * @throws IOException * could not read the object size index. IO problem or the pack * doesn't have it. */ - long getIndexedObjectSize(DfsReader ctx, AnyObjectId id) + long getIndexedObjectSize(DfsReader ctx, int idxPosition) throws IOException { - int idxPosition = idx(ctx).findPosition(id); if (idxPosition < 0) { - throw new IllegalArgumentException( - "Cannot get size from index since object is not in pack"); //$NON-NLS-1$ + throw new IllegalArgumentException("Invalid index position"); //$NON-NLS-1$ } - PackObjectSizeIndex sizeIdx = getObjectSizeIndex(ctx); if (sizeIdx == null) { throw new IllegalStateException( @@ -1169,12 +1200,47 @@ public final class DfsPackFile extends BlockBasedFile { return sizeIdx.getSize(idxPosition); } - void representation(DfsObjectRepresentation r, final long pos, + /** + * Populates the representation object with the details of how the object at + * "pos" is stored in this pack (e.g. whole or deltified, its packed + * length). + * + * @param r + * represention object to carry data + * @param offset + * offset in this pack of the object + * @param ctx + * a reader + * @throws IOException + * an error reading the object from disk + */ + void fillRepresentation(DfsObjectRepresentation r, long offset, + DfsReader ctx) throws IOException { + fillRepresentation(r, offset, ctx, getReverseIdx(ctx)); + } + + /** + * Populates the representation object with the details of how the object at + * "pos" is stored in this pack (e.g. whole or deltified, its packed + * length). + * + * @param r + * represention object to carry data + * @param offset + * offset in this pack of the object + * @param ctx + * a reader + * @param rev + * reverse index of this pack + * @throws IOException + * an error reading the object from disk + */ + void fillRepresentation(DfsObjectRepresentation r, long offset, DfsReader ctx, PackReverseIndex rev) throws IOException { - r.offset = pos; + r.offset = offset; final byte[] ib = ctx.tempId; - readFully(pos, ib, 0, 20, ctx); + readFully(offset, ib, 0, 20, ctx); int c = ib[0] & 0xff; int p = 1; final int typeCode = (c >> 4) & 7; @@ -1182,7 +1248,7 @@ public final class DfsPackFile extends BlockBasedFile { c = ib[p++] & 0xff; } - long len = rev.findNextOffset(pos, length - 20) - pos; + long len = rev.findNextOffset(offset, length - 20) - offset; switch (typeCode) { case Constants.OBJ_COMMIT: case Constants.OBJ_TREE: @@ -1203,13 +1269,13 @@ public final class DfsPackFile extends BlockBasedFile { ofs += (c & 127); } r.format = StoredObjectRepresentation.PACK_DELTA; - r.baseId = rev.findObject(pos - ofs); + r.baseId = rev.findObject(offset - ofs); r.length = len - p; return; } case Constants.OBJ_REF_DELTA: { - readFully(pos + p, ib, 0, 20, ctx); + readFully(offset + p, ib, 0, 20, ctx); r.format = StoredObjectRepresentation.PACK_DELTA; r.baseId = ObjectId.fromRaw(ib); r.length = len - p - 20; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java index 62f6753e5d..f50cd597e5 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsReader.java @@ -38,8 +38,6 @@ import org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource; import org.eclipse.jgit.internal.storage.dfs.DfsReader.PackLoadListener.DfsBlockData; import org.eclipse.jgit.internal.storage.file.BitmapIndexImpl; import org.eclipse.jgit.internal.storage.file.PackBitmapIndex; -import org.eclipse.jgit.internal.storage.file.PackIndex; -import org.eclipse.jgit.internal.storage.file.PackReverseIndex; import org.eclipse.jgit.internal.storage.pack.CachedPack; import org.eclipse.jgit.internal.storage.pack.ObjectReuseAsIs; import org.eclipse.jgit.internal.storage.pack.ObjectToPack; @@ -58,7 +56,6 @@ import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectLoader; import org.eclipse.jgit.lib.ObjectReader; import org.eclipse.jgit.lib.ProgressMonitor; -import org.eclipse.jgit.util.BlockList; /** * Reader to access repository content through. @@ -190,31 +187,44 @@ public class DfsReader extends ObjectReader implements ObjectReuseAsIs { @Override public boolean has(AnyObjectId objectId) throws IOException { + return findPack(objectId) >= 0; + } + + private int findPack(AnyObjectId objectId) throws IOException { if (last != null - && !skipGarbagePack(last) - && last.hasObject(this, objectId)) - return true; + && !skipGarbagePack(last)) { + int idxPos = last.findIdxPosition(this, objectId); + if (idxPos >= 0) { + return idxPos; + } + } + PackList packList = db.getPackList(); - if (hasImpl(packList, objectId)) { - return true; + int idxPos = findInPackList(packList, objectId); + if (idxPos >= 0) { + return idxPos; } else if (packList.dirty()) { stats.scanPacks++; - return hasImpl(db.scanPacks(packList), objectId); + idxPos = findInPackList(db.scanPacks(packList), objectId); + return idxPos; } - return false; + return -1; } - private boolean hasImpl(PackList packList, AnyObjectId objectId) + // Leave "last" pointing to the pack and return the idx position of the + // object (-1 if not found) + private int findInPackList(PackList packList, AnyObjectId objectId) throws IOException { for (DfsPackFile pack : packList.packs) { if (pack == last || skipGarbagePack(pack)) continue; - if (pack.hasObject(this, objectId)) { + int idxPos = pack.findIdxPosition(this, objectId); + if (idxPos >= 0) { last = pack; - return true; + return idxPos; } } - return false; + return -1; } @Override @@ -502,8 +512,8 @@ public class DfsReader extends ObjectReader implements ObjectReuseAsIs { public long getObjectSize(AnyObjectId objectId, int typeHint) throws MissingObjectException, IncorrectObjectTypeException, IOException { - DfsPackFile pack = findPackWithObject(objectId); - if (pack == null) { + int idxPos = findPack(objectId); + if (idxPos < 0) { if (typeHint == OBJ_ANY) { throw new MissingObjectException(objectId.copy(), JGitText.get().unknownObjectType2); @@ -511,16 +521,15 @@ public class DfsReader extends ObjectReader implements ObjectReuseAsIs { throw new MissingObjectException(objectId.copy(), typeHint); } - if (typeHint != Constants.OBJ_BLOB || !safeHasObjectSizeIndex(pack)) { - return pack.getObjectSize(this, objectId); + if (typeHint != Constants.OBJ_BLOB || !safeHasObjectSizeIndex(last)) { + return last.getObjectSize(this, objectId); } - Optional<Long> maybeSz = safeGetIndexedObjectSize(pack, objectId); - long sz = maybeSz.orElse(-1L); + long sz = safeGetIndexedObjectSize(last, idxPos); if (sz >= 0) { return sz; } - return pack.getObjectSize(this, objectId); + return last.getObjectSize(this, objectId); } @@ -528,8 +537,8 @@ public class DfsReader extends ObjectReader implements ObjectReuseAsIs { public boolean isNotLargerThan(AnyObjectId objectId, int typeHint, long limit) throws MissingObjectException, IncorrectObjectTypeException, IOException { - DfsPackFile pack = findPackWithObject(objectId); - if (pack == null) { + int idxPos = findPack(objectId); + if (idxPos < 0) { if (typeHint == OBJ_ANY) { throw new MissingObjectException(objectId.copy(), JGitText.get().unknownObjectType2); @@ -538,28 +547,22 @@ public class DfsReader extends ObjectReader implements ObjectReuseAsIs { } stats.isNotLargerThanCallCount += 1; - if (typeHint != Constants.OBJ_BLOB || !safeHasObjectSizeIndex(pack)) { - return pack.getObjectSize(this, objectId) <= limit; + if (typeHint != Constants.OBJ_BLOB || !safeHasObjectSizeIndex(last)) { + return last.getObjectSize(this, objectId) <= limit; } - Optional<Long> maybeSz = safeGetIndexedObjectSize(pack, objectId); - if (maybeSz.isEmpty()) { - // Exception in object size index - return pack.getObjectSize(this, objectId) <= limit; - } - - long sz = maybeSz.get(); + long sz = safeGetIndexedObjectSize(last, idxPos); if (sz >= 0) { return sz <= limit; } - if (isLimitInsideIndexThreshold(pack, limit)) { + if (isLimitInsideIndexThreshold(last, limit)) { // With threshold T, not-found means object < T // If limit L > T, then object < T < L return true; } - return pack.getObjectSize(this, objectId) <= limit; + return last.getObjectSize(this, objectId) <= limit; } private boolean safeHasObjectSizeIndex(DfsPackFile pack) { @@ -570,21 +573,22 @@ public class DfsReader extends ObjectReader implements ObjectReuseAsIs { } } - private Optional<Long> safeGetIndexedObjectSize(DfsPackFile pack, - AnyObjectId objectId) { + private long safeGetIndexedObjectSize(DfsPackFile pack, + int idxPos) { long sz; try { - sz = pack.getIndexedObjectSize(this, objectId); + sz = pack.getIndexedObjectSize(this, idxPos); } catch (IOException e) { - // Do not count the exception as an index miss - return Optional.empty(); + // If there is any error in the index, we should have seen it + // on hasObjectSizeIndex. + throw new IllegalStateException(e); } if (sz < 0) { stats.objectSizeIndexMiss += 1; } else { stats.objectSizeIndexHit += 1; } - return Optional.of(sz); + return sz; } private boolean isLimitInsideIndexThreshold(DfsPackFile pack, long limit) { @@ -595,34 +599,11 @@ public class DfsReader extends ObjectReader implements ObjectReuseAsIs { } } - private DfsPackFile findPackWithObject(AnyObjectId objectId) - throws IOException { - if (last != null && !skipGarbagePack(last) - && last.hasObject(this, objectId)) { - return last; - } - PackList packList = db.getPackList(); - // hasImpl doesn't check "last", but leaves "last" pointing to the pack - // with the object - if (hasImpl(packList, objectId)) { - return last; - } else if (packList.dirty()) { - if (hasImpl(db.getPackList(), objectId)) { - return last; - } - } - return null; - } - @Override public DfsObjectToPack newObjectToPack(AnyObjectId objectId, int type) { return new DfsObjectToPack(objectId, type); } - private static final Comparator<DfsObjectToPack> OFFSET_SORT = ( - DfsObjectToPack a, - DfsObjectToPack b) -> Long.signum(a.getOffset() - b.getOffset()); - @Override public void selectObjectRepresentation(PackWriter packer, ProgressMonitor monitor, Iterable<ObjectToPack> objects) @@ -642,16 +623,15 @@ public class DfsReader extends ObjectReader implements ObjectReuseAsIs { ProgressMonitor monitor, Iterable<ObjectToPack> objects, List<DfsPackFile> packs, boolean skipFound) throws IOException { for (DfsPackFile pack : packs) { - List<DfsObjectToPack> tmp = findAllFromPack(pack, objects, skipFound); - if (tmp.isEmpty()) + List<DfsObjectToPack> inPack = pack.findAllFromPack(this, objects, skipFound); + if (inPack.isEmpty()) continue; - Collections.sort(tmp, OFFSET_SORT); - PackReverseIndex rev = pack.getReverseIdx(this); DfsObjectRepresentation rep = new DfsObjectRepresentation(pack); - for (DfsObjectToPack otp : tmp) { - pack.representation(rep, otp.getOffset(), this, rev); + for (DfsObjectToPack otp : inPack) { + // Populate rep.{offset,length} from the pack + pack.fillRepresentation(rep, otp.getOffset(), this); otp.setOffset(0); - packer.select(otp, rep); + packer.select(otp, rep); // Set otp.offset from rep if (!otp.isFound()) { otp.setFound(); monitor.update(1); @@ -698,24 +678,7 @@ public class DfsReader extends ObjectReader implements ObjectReuseAsIs { return false; } - private List<DfsObjectToPack> findAllFromPack(DfsPackFile pack, - Iterable<ObjectToPack> objects, boolean skipFound) - throws IOException { - List<DfsObjectToPack> tmp = new BlockList<>(); - PackIndex idx = pack.getPackIndex(this); - for (ObjectToPack obj : objects) { - DfsObjectToPack otp = (DfsObjectToPack) obj; - if (skipFound && otp.isFound()) { - continue; - } - long p = idx.findOffset(otp); - if (0 < p && !pack.isCorrupt(p)) { - otp.setOffset(p); - tmp.add(otp); - } - } - return tmp; - } + @Override public void copyObjectAsIs(PackOutputStream out, ObjectToPack otp, diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java index b9e9e661e9..64f8c9b0e3 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableDatabase.java @@ -75,16 +75,11 @@ public class FileReftableDatabase extends RefDatabase { private volatile boolean autoRefresh; FileReftableDatabase(FileRepository repo) throws IOException { - this(repo, new File(new File(repo.getCommonDirectory(), Constants.REFTABLE), - Constants.TABLES_LIST)); - } - - FileReftableDatabase(FileRepository repo, File refstackName) throws IOException { this.fileRepository = repo; this.autoRefresh = repo.getConfig().getBoolean( ConfigConstants.CONFIG_REFTABLE_SECTION, ConfigConstants.CONFIG_KEY_AUTOREFRESH, false); - this.reftableStack = new FileReftableStack(refstackName, + this.reftableStack = new FileReftableStack( new File(fileRepository.getCommonDirectory(), Constants.REFTABLE), () -> fileRepository.fireEvent(new RefsChangedEvent()), () -> fileRepository.getConfig()); @@ -269,8 +264,14 @@ public class FileReftableDatabase extends RefDatabase { public void refresh() { try { if (!reftableStack.isUpToDate()) { - reftableDatabase.clearCache(); - reftableStack.reload(); + ReentrantLock lock = getLock(); + lock.lock(); + try { + reftableDatabase.clearCache(); + reftableStack.reload(); + } finally { + lock.unlock(); + } } } catch (IOException e) { throw new UncheckedIOException(e); @@ -683,32 +684,20 @@ public class FileReftableDatabase extends RefDatabase { * the repository * @param writeLogs * whether to write reflogs - * @return a reftable based RefDB from an existing repository. * @throws IOException * on IO error */ - public static FileReftableDatabase convertFrom(FileRepository repo, - boolean writeLogs) throws IOException { - FileReftableDatabase newDb = null; - File reftableList = null; - try { - File reftableDir = new File(repo.getCommonDirectory(), - Constants.REFTABLE); - reftableList = new File(reftableDir, Constants.TABLES_LIST); - if (!reftableDir.isDirectory()) { - reftableDir.mkdir(); - } + public static void convertFrom(FileRepository repo, boolean writeLogs) + throws IOException { + File reftableDir = new File(repo.getCommonDirectory(), + Constants.REFTABLE); + if (!reftableDir.isDirectory()) { + reftableDir.mkdir(); + } - try (FileReftableStack stack = new FileReftableStack(reftableList, - reftableDir, null, () -> repo.getConfig())) { - stack.addReftable(rw -> writeConvertTable(repo, rw, writeLogs)); - } - reftableList = null; - } finally { - if (reftableList != null) { - reftableList.delete(); - } + try (FileReftableStack stack = new FileReftableStack(reftableDir, null, + () -> repo.getConfig())) { + stack.addReftable(rw -> writeConvertTable(repo, rw, writeLogs)); } - return newDb; } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableStack.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableStack.java index b2c88922b8..6658575fc5 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableStack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/FileReftableStack.java @@ -42,6 +42,7 @@ import org.eclipse.jgit.internal.storage.reftable.ReftableConfig; import org.eclipse.jgit.internal.storage.reftable.ReftableReader; import org.eclipse.jgit.internal.storage.reftable.ReftableWriter; import org.eclipse.jgit.lib.Config; +import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.CoreConfig; import org.eclipse.jgit.lib.CoreConfig.TrustStat; import org.eclipse.jgit.util.FileUtils; @@ -69,7 +70,7 @@ public class FileReftableStack implements AutoCloseable { private long lastNextUpdateIndex; - private final File stackPath; + private final File tablesListFile; private final File reftableDir; @@ -111,8 +112,6 @@ public class FileReftableStack implements AutoCloseable { /** * Creates a stack corresponding to the list of reftables in the argument * - * @param stackPath - * the filename for the stack. * @param reftableDir * the dir holding the tables. * @param onChange @@ -122,10 +121,10 @@ public class FileReftableStack implements AutoCloseable { * @throws IOException * on I/O problems */ - public FileReftableStack(File stackPath, File reftableDir, + public FileReftableStack(File reftableDir, @Nullable Runnable onChange, Supplier<Config> configSupplier) throws IOException { - this.stackPath = stackPath; + this.tablesListFile = new File(reftableDir, Constants.TABLES_LIST); this.reftableDir = reftableDir; this.stack = new ArrayList<>(); this.configSupplier = configSupplier; @@ -244,7 +243,7 @@ public class FileReftableStack implements AutoCloseable { } if (!success) { - throw new LockFailedException(stackPath); + throw new LockFailedException(tablesListFile); } mergedReftable = new MergedReftable(stack.stream() @@ -288,14 +287,14 @@ public class FileReftableStack implements AutoCloseable { List<String> names = new ArrayList<>(stack.size() + 1); old = snapshot.get(); try (BufferedReader br = new BufferedReader( - new InputStreamReader(new FileInputStream(stackPath), UTF_8))) { + new InputStreamReader(new FileInputStream(tablesListFile), UTF_8))) { String line; while ((line = br.readLine()) != null) { if (!line.isEmpty()) { names.add(line); } } - snapshot.compareAndSet(old, FileSnapshot.save(stackPath)); + snapshot.compareAndSet(old, FileSnapshot.save(tablesListFile)); } catch (FileNotFoundException e) { // file isn't there: empty repository. snapshot.compareAndSet(old, FileSnapshot.MISSING_FILE); @@ -315,15 +314,16 @@ public class FileReftableStack implements AutoCloseable { break; case AFTER_OPEN: try (InputStream stream = Files - .newInputStream(stackPath.toPath())) { - // open the tables.list file to refresh attributes (on some - // NFS clients) + .newInputStream(reftableDir.toPath())) { + // open the refs/reftable/ directory to refresh attributes + // of reftable files and the tables.list file listing their + // names (on some NFS clients) } catch (FileNotFoundException | NoSuchFileException e) { // ignore } //$FALL-THROUGH$ case ALWAYS: - if (!snapshot.get().isModified(stackPath)) { + if (!snapshot.get().isModified(tablesListFile)) { return true; } break; @@ -387,7 +387,7 @@ public class FileReftableStack implements AutoCloseable { */ @SuppressWarnings("nls") public boolean addReftable(Writer w) throws IOException { - LockFile lock = new LockFile(stackPath); + LockFile lock = new LockFile(tablesListFile); try { if (!lock.lockForAppend()) { return false; @@ -398,8 +398,7 @@ public class FileReftableStack implements AutoCloseable { String fn = filename(nextUpdateIndex(), nextUpdateIndex()); - File tmpTable = File.createTempFile(fn + "_", ".ref", - stackPath.getParentFile()); + File tmpTable = File.createTempFile(fn + "_", ".ref", reftableDir); ReftableWriter.Stats s; try (FileOutputStream fos = new FileOutputStream(tmpTable)) { @@ -453,7 +452,7 @@ public class FileReftableStack implements AutoCloseable { String fn = filename(first, last); File tmpTable = File.createTempFile(fn + "_", ".ref", //$NON-NLS-1$//$NON-NLS-2$ - stackPath.getParentFile()); + reftableDir); try (FileOutputStream fos = new FileOutputStream(tmpTable)) { ReftableCompactor c = new ReftableCompactor(fos) .setConfig(reftableConfig()) @@ -497,7 +496,7 @@ public class FileReftableStack implements AutoCloseable { if (first >= last) { return true; } - LockFile lock = new LockFile(stackPath); + LockFile lock = new LockFile(tablesListFile); File tmpTable = null; List<LockFile> subtableLocks = new ArrayList<>(); @@ -526,7 +525,7 @@ public class FileReftableStack implements AutoCloseable { tmpTable = compactLocked(first, last); - lock = new LockFile(stackPath); + lock = new LockFile(tablesListFile); if (!lock.lock()) { return false; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java index c08a92e5a7..97473bba2a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java @@ -14,6 +14,7 @@ import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.COMMIT_GRAPH; import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.KEEP; +import static org.eclipse.jgit.internal.storage.pack.PackExt.OBJECT_SIZE_INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK; import static org.eclipse.jgit.internal.storage.pack.PackExt.REVERSE_INDEX; @@ -131,7 +132,7 @@ public class GC { private static final Set<PackExt> PARENT_EXTS = Set.of(PACK, KEEP); private static final Set<PackExt> CHILD_EXTS = Set.of(BITMAP_INDEX, INDEX, - REVERSE_INDEX); + REVERSE_INDEX, OBJECT_SIZE_INDEX); private static final int DEFAULT_AUTOPACKLIMIT = 50; @@ -412,6 +413,10 @@ public class GC { */ private void removeOldPack(PackFile packFile, int deleteOptions) throws IOException { + if (!packFile.exists()) { + return; + } + if (pconfig.isPreserveOldPacks()) { File oldPackDir = repo.getObjectDatabase().getPreservedDirectory(); FileUtils.mkdir(oldPackDir, true); @@ -1340,6 +1345,7 @@ public class GC { idxChannel.force(true); } + if (pw.isReverseIndexEnabled()) { File tmpReverseIndexFile = new File(packdir, tmpBase + REVERSE_INDEX.getTmpExtension()); @@ -1356,6 +1362,19 @@ public class GC { .newOutputStream(channel)) { pw.writeReverseIndex(stream); channel.force(true); + } + } + + // write the object size + if (pconfig.isWriteObjSizeIndex()) { + File tmpSizeIdx = new File(packdir, tmpBase + ".objsize_tmp"); //$NON-NLS-1$ + tmpExts.put(OBJECT_SIZE_INDEX, tmpSizeIdx); + try (FileOutputStream fos = new FileOutputStream(tmpSizeIdx); + FileChannel idxChannel = fos.getChannel(); + OutputStream idxStream = Channels + .newOutputStream(idxChannel)) { + pw.writeObjectSizeIndex(idxStream); + idxChannel.force(true); } } @@ -1525,6 +1544,11 @@ public class GC { */ public long numberOfBitmaps; + /** + * The number of objects in the size-index of the packs + */ + public long numberOfSizeIndexedObjects; + @Override public String toString() { final StringBuilder b = new StringBuilder(); @@ -1540,6 +1564,8 @@ public class GC { b.append(", sizeOfLooseObjects=").append(sizeOfLooseObjects); //$NON-NLS-1$ b.append(", sizeOfPackedObjects=").append(sizeOfPackedObjects); //$NON-NLS-1$ b.append(", numberOfBitmaps=").append(numberOfBitmaps); //$NON-NLS-1$ + b.append(", numberOfSizeIndexedObjects=") //$NON-NLS-1$ + .append(numberOfSizeIndexedObjects); return b.toString(); } } @@ -1548,7 +1574,7 @@ public class GC { * Returns information about objects and pack files for a FileRepository. * * @return information about objects and pack files for a FileRepository - * @throws java.io.IOException + * @throws java.io.IOException * if an IO error occurred */ public RepoStatistics getStatistics() throws IOException { @@ -1560,16 +1586,16 @@ public class GC { ret.numberOfPackedObjects += packedObjects; ret.numberOfPackFiles++; ret.sizeOfPackedObjects += p.getPackFile().length(); + ret.numberOfSizeIndexedObjects += p.getObjectSizeIndexCount(); if (p.getBitmapIndex() != null) { ret.numberOfBitmaps += p.getBitmapIndex().getBitmapCount(); if (latestBitmapTime == 0L) { latestBitmapTime = p.getFileSnapshot().lastModifiedInstant().toEpochMilli(); } - } - else if (latestBitmapTime == 0L) { - ret.numberOfPackFilesSinceBitmap++; - ret.numberOfObjectsSinceBitmap += packedObjects; - } + } else if (latestBitmapTime == 0L) { + ret.numberOfPackFilesSinceBitmap++; + ret.numberOfObjectsSinceBitmap += packedObjects; + } } File objDir = repo.getObjectsDirectory(); String[] fanout = objDir.list(); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalObjectRepresentation.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalObjectRepresentation.java index 3f3d78c734..af571622b4 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalObjectRepresentation.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalObjectRepresentation.java @@ -22,6 +22,11 @@ class LocalObjectRepresentation extends StoredObjectRepresentation { public int getFormat() { return PACK_WHOLE; } + + @Override + public boolean wasDeltaAttempted() { + return true; + } }; r.pack = pack; r.offset = offset; @@ -81,5 +86,10 @@ class LocalObjectRepresentation extends StoredObjectRepresentation { public int getFormat() { return PACK_DELTA; } + + @Override + public boolean wasDeltaAttempted() { + return true; + } } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java index 9f21481a13..af1671d442 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java @@ -70,6 +70,11 @@ import org.eclipse.jgit.util.FileUtils; * considered. */ public class ObjectDirectory extends FileObjectDatabase { + @FunctionalInterface + private interface TriFunctionThrowsException<A, A2, A3, R, E extends Exception> { + R apply(A a, A2 a2, A3 a3) throws E; + } + /** Maximum number of candidates offered as resolutions of abbreviation. */ private static final int RESOLVE_ABBREV_LIMIT = 256; @@ -200,6 +205,7 @@ public class ObjectDirectory extends FileObjectDatabase { loose.close(); packed.close(); + preserved.close(); // Fully close all loaded alternates and clear the alternate list. AlternateHandle[] alt = alternates.get(); @@ -347,9 +353,13 @@ public class ObjectDirectory extends FileObjectDatabase { @Override ObjectLoader openObject(WindowCursor curs, AnyObjectId objectId) throws IOException { - ObjectLoader ldr = openObjectWithoutRestoring(curs, objectId); - if (ldr == null && restoreFromSelfOrAlternate(objectId, null)) { + ObjectLoader ldr = getFromLocalObjectToPack(curs, objectId, + (p, c, l) -> p.load(c, l.offset)); + if (ldr == null) { ldr = openObjectWithoutRestoring(curs, objectId); + if (ldr == null && restoreFromSelfOrAlternate(objectId, null)) { + ldr = openObjectWithoutRestoring(curs, objectId); + } } return ldr; } @@ -418,11 +428,16 @@ public class ObjectDirectory extends FileObjectDatabase { return loose.open(curs, id); } + @SuppressWarnings("boxing") @Override long getObjectSize(WindowCursor curs, AnyObjectId id) throws IOException { - long sz = getObjectSizeWithoutRestoring(curs, id); - if (0 > sz && restoreFromSelfOrAlternate(id, null)) { + Long sz = getFromLocalObjectToPack(curs, id, + (p, c, l) -> p.getObjectSize(c, l)); + if (sz == null) { sz = getObjectSizeWithoutRestoring(curs, id); + if (sz < 0 && restoreFromSelfOrAlternate(id, null)) { + sz = getObjectSizeWithoutRestoring(curs, id); + } } return sz; } @@ -431,12 +446,12 @@ public class ObjectDirectory extends FileObjectDatabase { AnyObjectId id) throws IOException { if (loose.hasCached(id)) { long len = loose.getSize(curs, id); - if (0 <= len) { + if (len >= 0) { return len; } } long len = getPackedSizeFromSelfOrAlternate(curs, id, null); - if (0 <= len) { + if (len >= 0) { return len; } return getLooseSizeFromSelfOrAlternate(curs, id, null); @@ -446,14 +461,14 @@ public class ObjectDirectory extends FileObjectDatabase { AnyObjectId id, Set<AlternateHandle.Id> skips) throws PackMismatchException { long len = packed.getSize(curs, id); - if (0 <= len) { + if (len >= 0) { return len; } skips = addMe(skips); for (AlternateHandle alt : myAlternates()) { if (!skips.contains(alt.getId())) { len = alt.db.getPackedSizeFromSelfOrAlternate(curs, id, skips); - if (0 <= len) { + if (len >= 0) { return len; } } @@ -464,14 +479,14 @@ public class ObjectDirectory extends FileObjectDatabase { private long getLooseSizeFromSelfOrAlternate(WindowCursor curs, AnyObjectId id, Set<AlternateHandle.Id> skips) throws IOException { long len = loose.getSize(curs, id); - if (0 <= len) { + if (len >= 0) { return len; } skips = addMe(skips); for (AlternateHandle alt : myAlternates()) { if (!skips.contains(alt.getId())) { len = alt.db.getLooseSizeFromSelfOrAlternate(curs, id, skips); - if (0 <= len) { + if (len >= 0) { return len; } } @@ -479,6 +494,24 @@ public class ObjectDirectory extends FileObjectDatabase { return -1; } + private <R> R getFromLocalObjectToPack(WindowCursor curs, + AnyObjectId objectId, + TriFunctionThrowsException<Pack, WindowCursor, LocalObjectToPack, R, IOException> func) { + if (objectId instanceof LocalObjectToPack) { + LocalObjectToPack lotp = (LocalObjectToPack) objectId; + Pack pack = lotp.pack; + if (pack != null) { + try { + return func.apply(pack, curs, lotp); + } catch (IOException e) { + // lotp potentially repacked, continue as if lotp not + // provided + } + } + } + return null; + } + @Override void selectObjectRepresentation(PackWriter packer, ObjectToPack otp, WindowCursor curs) throws IOException { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java index 746e124e1f..d97d5a7ccd 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java @@ -72,6 +72,12 @@ public class ObjectDirectoryPackParser extends PackParser { */ private File tmpIdx; + /** + * Path of the object-size index created for the pack, to filter quickly + * objects by size in partial clones + */ + private File tmpObjectSizeIndex; + /** Read/write handle to {@link #tmpPack} while it is being parsed. */ private RandomAccessFile out; @@ -163,6 +169,7 @@ public class ObjectDirectoryPackParser extends PackParser { throws IOException { tmpPack = File.createTempFile("incoming_", ".pack", db.getDirectory()); //$NON-NLS-1$ //$NON-NLS-2$ tmpIdx = new File(db.getDirectory(), baseName(tmpPack) + ".idx"); //$NON-NLS-1$ + try { out = new RandomAccessFile(tmpPack, "rw"); //$NON-NLS-1$ @@ -178,6 +185,14 @@ public class ObjectDirectoryPackParser extends PackParser { tmpPack.setReadOnly(); tmpIdx.setReadOnly(); + if (pconfig.isWriteObjSizeIndex()) { + tmpObjectSizeIndex = new File(db.getDirectory(), + baseName(tmpPack) + + PackExt.OBJECT_SIZE_INDEX.getExtension()); + writeObjectSizeIndex(pconfig.getMinBytesForObjSizeIndex()); + tmpObjectSizeIndex.setReadOnly(); + } + return renameAndOpenPack(getLockMessage()); } finally { if (def != null) @@ -295,6 +310,9 @@ public class ObjectDirectoryPackParser extends PackParser { tmpIdx.deleteOnExit(); if (tmpPack != null && !tmpPack.delete() && tmpPack.exists()) tmpPack.deleteOnExit(); + if (tmpObjectSizeIndex != null && !tmpObjectSizeIndex.delete() + && tmpObjectSizeIndex.exists()) + tmpPack.deleteOnExit(); } @Override @@ -395,6 +413,15 @@ public class ObjectDirectoryPackParser extends PackParser { } } + private void writeObjectSizeIndex(int minSize) throws IOException { + try (FileOutputStream os = new FileOutputStream(tmpObjectSizeIndex)) { + PackObjectSizeIndexWriter iw = PackObjectSizeIndexWriter + .createWriter(os, minSize); + iw.write(getSortedObjectList(null)); + os.getChannel().force(true); + } + } + private PackLock renameAndOpenPack(String lockMessage) throws IOException { if (!keepEmpty && getObjectCount() == 0) { @@ -469,6 +496,27 @@ public class ObjectDirectoryPackParser extends PackParser { JGitText.get().cannotMoveIndexTo, finalIdx), e); } + if (pconfig.isWriteObjSizeIndex() && tmpObjectSizeIndex != null) { + PackFile finalObjectSizeIndex = finalPack + .create(PackExt.OBJECT_SIZE_INDEX); + try { + FileUtils.rename(tmpObjectSizeIndex, finalObjectSizeIndex, + StandardCopyOption.ATOMIC_MOVE); + } catch (IOException e) { + cleanupTemporaryFiles(); + keep.unlock(); + if (!finalPack.delete()) + finalPack.deleteOnExit(); + if (!finalIdx.delete()) { + finalIdx.deleteOnExit(); + } + throw new IOException(MessageFormat + .format(JGitText.get().cannotMoveIndexTo, + finalObjectSizeIndex), + e); + } + } + boolean interrupted = false; try { FileSnapshot snapshot = FileSnapshot.save(finalPack); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java index 5813d39e9a..8988b41b55 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java @@ -17,9 +17,11 @@ import static org.eclipse.jgit.internal.storage.pack.PackExt.KEEP; import static org.eclipse.jgit.internal.storage.pack.PackExt.REVERSE_INDEX; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_CORE_SECTION; import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_PACKED_INDEX_GIT_USE_STRONGREFS; +import static org.eclipse.jgit.internal.storage.pack.PackExt.OBJECT_SIZE_INDEX; import java.io.EOFException; import java.io.File; +import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InterruptedIOException; @@ -122,11 +124,16 @@ public class Pack implements Iterable<PackIndex.MutableEntry> { private byte[] packChecksum; - private Optionally<PackIndex> loadedIdx = Optionally.empty(); + private volatile Optionally<PackIndex> loadedIdx = Optionally.empty(); - private Optionally<PackReverseIndex> reverseIdx = Optionally.empty(); + private volatile Optionally<PackReverseIndex> reverseIdx = Optionally.empty(); + + private volatile PackObjectSizeIndex loadedObjSizeIdx; + + private volatile boolean attemptLoadObjSizeIdx; + + private volatile Optionally<PackBitmapIndex> bitmapIdx = Optionally.empty(); - private Optionally<PackBitmapIndex> bitmapIdx = Optionally.empty(); /** * Objects we have tried to read, and discovered to be corrupt. @@ -162,7 +169,15 @@ public class Pack implements Iterable<PackIndex.MutableEntry> { length = Long.MAX_VALUE; } - private synchronized PackIndex idx() throws IOException { + private PackIndex idx() throws IOException { + Optional<PackIndex> optional = loadedIdx.getOptional(); + if (optional.isPresent()) { + return optional.get(); + } + return memoizeIdxIfNeeded(); + } + + private synchronized PackIndex memoizeIdxIfNeeded() throws IOException { Optional<PackIndex> optional = loadedIdx.getOptional(); if (optional.isPresent()) { return optional.get(); @@ -210,6 +225,52 @@ public class Pack implements Iterable<PackIndex.MutableEntry> { } } + private PackObjectSizeIndex objectSizeIndex() throws IOException { + if (loadedObjSizeIdx != null) { + return loadedObjSizeIdx; + } + + if (attemptLoadObjSizeIdx) { + return null; + } + + synchronized (this) { + if (loadedObjSizeIdx != null) { + return loadedObjSizeIdx; + } + + PackObjectSizeIndex sizeIdx; + try { + long start = System.currentTimeMillis(); + PackFile sizeIdxFile = packFile.create(OBJECT_SIZE_INDEX); + if (attemptLoadObjSizeIdx || !sizeIdxFile.exists()) { + attemptLoadObjSizeIdx = true; + return null; + } + sizeIdx = PackObjectSizeIndexLoader.load( + new FileInputStream(sizeIdxFile.getAbsoluteFile())); + if (LOG.isDebugEnabled()) { + LOG.debug(String.format( + "Opening obj size index %s, size %.3f MB took %d ms", //$NON-NLS-1$ + sizeIdxFile.getAbsolutePath(), + Float.valueOf( + sizeIdxFile.length() / (1024f * 1024)), + Long.valueOf(System.currentTimeMillis() - start))); + } + + loadedObjSizeIdx = sizeIdx; + } catch (InterruptedIOException e) { + // don't invalidate the pack, we are interrupted from + // another thread + return null; + } finally { + attemptLoadObjSizeIdx = true; + } + } + + return loadedObjSizeIdx; + } + /** * Get the File object which locates this pack on disk. * @@ -231,6 +292,62 @@ public class Pack implements Iterable<PackIndex.MutableEntry> { } /** + * Get the object size index for this pack file + * + * @return the object size index for this pack file if it exists (null + * otherwise) + * @throws IOException + * problem reading the index + */ + public boolean hasObjectSizeIndex() throws IOException { + return objectSizeIndex() != null; + } + + /** + * Number of objects in the object-size index of this pack + * + * @return number of objects in the index (0 if either the index is empty or + * it doesn't exist) + * @throws IOException + * if an IO error occurred while reading the index + */ + public long getObjectSizeIndexCount() throws IOException { + if (!hasObjectSizeIndex()) { + return 0; + } + + return objectSizeIndex().getObjectCount(); + } + + /** + * Return the size of the object from the object-size index. + * + * Caller MUST check that the pack has object-size index + * ({@link #hasObjectSizeIndex()}) and that the pack contains the object. + * + * @param id + * object id of an object in the pack + * @return size of the object from the index. Negative if the object is not + * in the index. + * @throws IOException + * if an IO error occurred while reading the index + */ + public long getIndexedObjectSize(AnyObjectId id) throws IOException { + int idxPos = idx().findPosition(id); + if (idxPos < 0) { + return -1; + } + + PackObjectSizeIndex sizeIdx = objectSizeIndex(); + if (sizeIdx == null) { + throw new IllegalStateException( + "Asking indexed size from a pack without object size index"); //$NON-NLS-1$ + } + + return sizeIdx.getSize(idxPos); + } + + /** * Get name extracted from {@code pack-*.pack} pattern. * * @return name extracted from {@code pack-*.pack} pattern. @@ -312,9 +429,9 @@ public class Pack implements Iterable<PackIndex.MutableEntry> { } private synchronized void closeIndices() { - loadedIdx.clear(); - reverseIdx.clear(); - bitmapIdx.clear(); + loadedIdx = Optionally.empty(); + reverseIdx = Optionally.empty(); + bitmapIdx = Optionally.empty(); } /** @@ -1182,7 +1299,15 @@ public class Pack implements Iterable<PackIndex.MutableEntry> { return getReverseIdx().findNextOffset(startOffset, maxOffset); } - synchronized PackBitmapIndex getBitmapIndex() throws IOException { + PackBitmapIndex getBitmapIndex() throws IOException { + Optional<PackBitmapIndex> optional = bitmapIdx.getOptional(); + if (optional.isPresent()) { + return optional.get(); + } + return memoizeBitmapIndexIfNeeded(); + } + + private synchronized PackBitmapIndex memoizeBitmapIndexIfNeeded() throws IOException { if (invalid || bitmapIdxFile == null) { return null; } @@ -1217,7 +1342,15 @@ public class Pack implements Iterable<PackIndex.MutableEntry> { this.bitmapIdxFile = bitmapIndexFile; } - private synchronized PackReverseIndex getReverseIdx() throws IOException { + private PackReverseIndex getReverseIdx() throws IOException { + Optional<PackReverseIndex> optional = reverseIdx.getOptional(); + if (optional.isPresent()) { + return optional.get(); + } + return memoizeReverseIdxIfNeeded(); + } + + private synchronized PackReverseIndex memoizeReverseIdxIfNeeded() throws IOException { if (invalid) { throw new PackInvalidException(packFile, invalidatingCause); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java index f50c17eafa..544961b936 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java @@ -27,6 +27,7 @@ import java.util.Collections; import java.util.EnumMap; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -49,6 +50,7 @@ import org.eclipse.jgit.lib.CoreConfig.TrustStat; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectLoader; import org.eclipse.jgit.util.FileUtils; +import org.eclipse.jgit.util.Iterators; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -275,12 +277,14 @@ class PackDirectory { PackList pList = packList.get(); int retries = 0; SEARCH: for (;;) { - for (Pack p : pList.packs) { + for (Pack p : pList.inReverse()) { try { LocalObjectRepresentation rep = p.representation(curs, otp); p.resetTransientErrorCount(); if (rep != null) { - packer.select(otp, rep); + if (!packer.select(otp, rep)) { + return; + } packer.checkSearchForReuseTimeout(); } } catch (SearchForReuseTimeout e) { @@ -584,5 +588,13 @@ class PackDirectory { this.snapshot = monitor; this.packs = packs; } + + Iterable<Pack> inReverse() { + return Iterators.iterable(reverseIterator()); + } + + Iterator<Pack> reverseIterator() { + return Iterators.reverseIterator(packs); + } } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java index c9b05ad025..5f2015b834 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java @@ -185,7 +185,11 @@ public class PackFile extends File { private static PackExt getPackExt(String endsWithExtension) { for (PackExt ext : PackExt.values()) { - if (endsWithExtension.endsWith(ext.getExtension())) { + if (endsWithExtension.equals(ext.getExtension())) { + return ext; + } + + if (endsWithExtension.equals("old-" + ext.getExtension())) { return ext; } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java index 55e047bd43..97a854b8cd 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java @@ -166,7 +166,7 @@ public class PackInserter extends ObjectInserter { long offset = beginObject(type, len); packOut.compress.write(data, off, len); packOut.compress.finish(); - return endObject(id, offset); + return endObject(id, offset, len, type); } @Override @@ -195,7 +195,7 @@ public class PackInserter extends ObjectInserter { len -= n; } packOut.compress.finish(); - return endObject(md.toObjectId(), offset); + return endObject(md.toObjectId(), offset, len, type); } private long beginObject(int type, long len) throws IOException { @@ -207,10 +207,12 @@ public class PackInserter extends ObjectInserter { return offset; } - private ObjectId endObject(ObjectId id, long offset) { + private ObjectId endObject(ObjectId id, long offset, long len, int type) { PackedObjectInfo obj = new PackedObjectInfo(id); + obj.setType(type); obj.setOffset(offset); obj.setCRC((int) packOut.crc32.getValue()); + obj.setFullSize(len); objectList.add(obj); objectMap.addIfAbsent(obj); return id; @@ -223,6 +225,12 @@ public class PackInserter extends ObjectInserter { p.substring(0, p.lastIndexOf('.')) + ".idx"); //$NON-NLS-1$ } + private static File getFileFor(File packFile, PackExt ext) { + String p = packFile.getName(); + return new File(packFile.getParentFile(), + p.substring(0, p.lastIndexOf('.')) + ext.getExtension()); + } + private void beginPack() throws IOException { objectList = new BlockList<>(); objectMap = new ObjectIdOwnerMap<>(); @@ -272,7 +280,11 @@ public class PackInserter extends ObjectInserter { Collections.sort(objectList); File tmpIdx = idxFor(tmpPack); // TODO(nasserg) Use PackFile? writePackIndex(tmpIdx, packHash, objectList); - + File tmpObjSizeIdx = null; + if (pconfig.isWriteObjSizeIndex()) { + tmpObjSizeIdx = getFileFor(tmpPack, PackExt.OBJECT_SIZE_INDEX); + writeObjectSizeIndex(tmpObjSizeIdx, objectList, pconfig); + } PackFile realPack = new PackFile(db.getPackDirectory(), computeName(objectList), PackExt.PACK); db.closeAllPackHandles(realPack); @@ -297,6 +309,13 @@ public class PackInserter extends ObjectInserter { realIdx), e); } + if (tmpObjSizeIdx != null) { + PackFile realObjSizeIdx = realPack + .create(PackExt.OBJECT_SIZE_INDEX); + tmpObjSizeIdx.setReadOnly(); + FileUtils.rename(tmpObjSizeIdx, realObjSizeIdx, ATOMIC_MOVE); + } + boolean interrupted = false; try { FileSnapshot snapshot = FileSnapshot.save(realPack); @@ -327,6 +346,15 @@ public class PackInserter extends ObjectInserter { } } + private static void writeObjectSizeIndex(File objIdx, + List<PackedObjectInfo> list, PackConfig cfg) throws IOException { + try (OutputStream os = new FileOutputStream(objIdx)) { + PackObjectSizeIndexWriter w = PackObjectSizeIndexWriter + .createWriter(os, cfg.getMinBytesForObjSizeIndex()); + w.write(list); + } + } + private ObjectId computeName(List<PackedObjectInfo> list) { SHA1 md = digest().reset(); byte[] buf = buffer(); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java index 05f1ef53a1..319a9ed710 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java @@ -701,41 +701,47 @@ public class RefDirectory extends RefDatabase { } String name = dst.getName(); - // Write the packed-refs file using an atomic update. We might - // wind up reading it twice, before and after the lock, to ensure - // we don't miss an edit made externally. - PackedRefList packed = getPackedRefs(); - if (packed.contains(name)) { - inProcessPackedRefsLock.lock(); + // Get and keep the packed-refs lock while updating packed-refs and + // removing any loose ref + inProcessPackedRefsLock.lock(); + try { + LockFile lck = lockPackedRefsOrThrow(); try { - LockFile lck = lockPackedRefsOrThrow(); - try { + // Write the packed-refs file using an atomic update. We might + // wind up reading it twice, before and after checking if the + // ref to delete is included or not, to ensure + // we don't rely on a PackedRefList that is a result of in-memory + // or NFS caching. + PackedRefList packed = getPackedRefs(); + if (packed.contains(name)) { + // Force update our packed-refs snapshot before writing packed = refreshPackedRefs(); int idx = packed.find(name); if (0 <= idx) { commitPackedRefs(lck, packed.remove(idx), packed, true); } - } finally { - lck.unlock(); } - } finally { - inProcessPackedRefsLock.unlock(); - } - } - RefList<LooseRef> curLoose, newLoose; - do { - curLoose = looseRefs.get(); - int idx = curLoose.find(name); - if (idx < 0) - break; - newLoose = curLoose.remove(idx); - } while (!looseRefs.compareAndSet(curLoose, newLoose)); + RefList<LooseRef> curLoose, newLoose; + do { + curLoose = looseRefs.get(); + int idx = curLoose.find(name); + if (idx < 0) { + break; + } + newLoose = curLoose.remove(idx); + } while (!looseRefs.compareAndSet(curLoose, newLoose)); - int levels = levelsIn(name) - 2; - delete(logFor(name), levels); - if (dst.getStorage().isLoose()) { - deleteAndUnlock(fileFor(name), levels, update); + int levels = levelsIn(name) - 2; + delete(logFor(name), levels); + if (dst.getStorage().isLoose()) { + deleteAndUnlock(fileFor(name), levels, update); + } + } finally { + lck.unlock(); + } + } finally { + inProcessPackedRefsLock.unlock(); } modCnt.incrementAndGet(); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCursor.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCursor.java index 11c45471e4..6612cfc838 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCursor.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/WindowCursor.java @@ -55,6 +55,8 @@ final class WindowCursor extends ObjectReader implements ObjectReuseAsIs { private DeltaBaseCache baseCache; + private Pack lastPack; + @Nullable private final ObjectInserter createdFromInserter; @@ -147,8 +149,7 @@ final class WindowCursor extends ObjectReader implements ObjectReuseAsIs { return db.getShallowCommits(); } - @Override - public long getObjectSize(AnyObjectId objectId, int typeHint) + private long getObjectSizeStorage(AnyObjectId objectId, int typeHint) throws MissingObjectException, IncorrectObjectTypeException, IOException { long sz = db.getObjectSize(this, objectId); @@ -162,6 +163,61 @@ final class WindowCursor extends ObjectReader implements ObjectReuseAsIs { } @Override + public long getObjectSize(AnyObjectId objectId, int typeHint) + throws MissingObjectException, IncorrectObjectTypeException, + IOException { + // Async queue uses hint OBJ_ANY + if (typeHint != Constants.OBJ_BLOB) { + return getObjectSizeStorage(objectId, typeHint); + } + + Pack pack = findPack(objectId); + if (pack == null) { + // Non-packed object (e.g. loose or in alternates) + return getObjectSizeStorage(objectId, typeHint); + } + + long sz = pack.getIndexedObjectSize(objectId); + if (sz >= 0) { + return sz; + } + return getObjectSizeStorage(objectId, typeHint); + } + + @Override + public boolean isNotLargerThan(AnyObjectId objectId, int typeHint, + long size) + throws MissingObjectException, IncorrectObjectTypeException, + IOException { + if (typeHint != Constants.OBJ_BLOB) { + return getObjectSizeStorage(objectId, typeHint) <= size; + } + + Pack pack = findPack(objectId); + if (pack == null || !pack.hasObjectSizeIndex()) { + // Non-packed object (e.g. loose or in alternates) + return getObjectSizeStorage(objectId, typeHint) <= size; + } + + return pack.getIndexedObjectSize(objectId) <= size; + } + + private Pack findPack(AnyObjectId objectId) throws IOException { + if (lastPack != null && lastPack.hasObject(objectId)) { + return lastPack; + } + + for (Pack p : db.getPacks()) { + if (p.hasObject(objectId)) { + lastPack = p; + return p; + } + } + + return null; + } + + @Override public LocalObjectToPack newObjectToPack(AnyObjectId objectId, int type) { return new LocalObjectToPack(objectId, type); } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndex.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndex.java index 845576d8c2..15b52391b8 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndex.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndex.java @@ -10,7 +10,11 @@ package org.eclipse.jgit.internal.storage.midx; +import java.util.Set; + +import org.eclipse.jgit.lib.AbbreviatedObjectId; import org.eclipse.jgit.lib.AnyObjectId; +import org.eclipse.jgit.lib.ObjectId; /** * An index over multiple packs @@ -48,6 +52,20 @@ public interface MultiPackIndex { PackOffset find(AnyObjectId objectId); /** + * Find objects matching the prefix abbreviation. + * + * @param matches + * set to add any located ObjectIds to. This is an output + * parameter. + * @param id + * prefix to search for. + * @param matchLimit + * maximum number of results to return. At most this many + * ObjectIds should be added to matches before returning. + */ + void resolve(Set<ObjectId> matches, AbbreviatedObjectId id, int matchLimit); + + /** * Memory size of this multipack index * * @return size of this multipack index in memory, in bytes diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexConstants.java index 0e45c9dfe5..5d86f44baf 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexConstants.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexConstants.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2025, Google Inc. + * Copyright (C) 2025, Google LLC * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexLoader.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexLoader.java index 0e51e90815..61caddc221 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexLoader.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexLoader.java @@ -257,7 +257,7 @@ public class MultiPackIndexLoader { MultiPackIndexBuilder addPackNames(byte[] buffer) throws MultiPackIndexFormatException { assertChunkNotSeenYet(packNames, MIDX_CHUNKID_PACKNAMES); - packNames = new String(buffer, UTF_8).split("\u0000"); + packNames = new String(buffer, UTF_8).split("\u0000"); //$NON-NLS-1$ return this; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexPrettyPrinter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexPrettyPrinter.java index 795d39e375..948b7bc174 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexPrettyPrinter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexPrettyPrinter.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2025, Google Inc. + * Copyright (C) 2025, Google LLC * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexV1.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexV1.java index 63d77cd967..be752cc4b5 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexV1.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexV1.java @@ -12,12 +12,15 @@ package org.eclipse.jgit.internal.storage.midx; import java.nio.charset.StandardCharsets; import java.util.Arrays; +import java.util.Set; import org.eclipse.jgit.annotations.NonNull; import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.midx.MultiPackIndexLoader.MultiPackIndexFormatException; +import org.eclipse.jgit.lib.AbbreviatedObjectId; import org.eclipse.jgit.lib.AnyObjectId; +import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.util.NB; /** @@ -69,6 +72,12 @@ class MultiPackIndexV1 implements MultiPackIndex { } @Override + public void resolve(Set<ObjectId> matches, AbbreviatedObjectId id, + int matchLimit) { + idx.resolve(matches, id, matchLimit); + } + + @Override public long getMemorySize() { int packNamesSize = Arrays.stream(packNames) .mapToInt(s -> s.getBytes(StandardCharsets.UTF_8).length).sum(); @@ -146,7 +155,8 @@ class MultiPackIndexV1 implements MultiPackIndex { } long getMemorySize() { - return byteArrayLengh(offsets) + byteArrayLengh(largeOffsets); + return (long) byteArrayLengh(offsets) + + byteArrayLengh(largeOffsets); } } @@ -226,8 +236,54 @@ class MultiPackIndexV1 implements MultiPackIndex { return -1; } + void resolve(Set<ObjectId> matches, AbbreviatedObjectId id, + int matchLimit) { + if (matches.size() >= matchLimit) { + return; + } + + if (oidLookup.length == 0) { + return; + } + + int high = fanoutTable[id.getFirstByte()]; + int low = id.getFirstByte() == 0 ? 0 + : fanoutTable[id.getFirstByte() - 1]; + do { + int p = (low + high) >>> 1; + int cmp = id.prefixCompare(oidLookup, idOffset(p)); + if (cmp < 0) { + high = p; + continue; + } + + if (cmp > 0) { + low = p + 1; + continue; + } + + // Got a match. + // We may have landed in the middle of the matches. Move + // backwards to the start of matches, then walk forwards. + while (0 < p + && id.prefixCompare(oidLookup, idOffset(p - 1)) == 0) { + p--; + } + while (p < high && id.prefixCompare(oidLookup, idOffset(p)) == 0 + && matches.size() < matchLimit) { + matches.add(ObjectId.fromRaw(oidLookup, idOffset(p))); + p++; + } + return; + } while (low < high); + } + + private int idOffset(int position) { + return position * hashLength; + } + long getMemorySize() { - return 4 + byteArrayLengh(oidLookup) + (FANOUT * 4); + return 4L + byteArrayLengh(oidLookup) + (FANOUT * 4); } } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexWriter.java index bddf3ac4ad..b42c821a44 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/MultiPackIndexWriter.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2025, Google Inc. + * Copyright (C) 2025, Google LLC * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -65,10 +65,11 @@ public class MultiPackIndexWriter { * @param inputs * pairs of name and index for each pack to include in the * multipack index. + * @return bytes written into the stream * @throws IOException * Error writing to the stream */ - public void write(ProgressMonitor monitor, OutputStream outputStream, + public long write(ProgressMonitor monitor, OutputStream outputStream, Map<String, PackIndex> inputs) throws IOException { PackIndexMerger data = new PackIndexMerger(inputs); @@ -91,6 +92,7 @@ public class MultiPackIndexWriter { Long.valueOf(expectedSize), Long.valueOf(out.length()))); } + return expectedSize; } catch (InterruptedIOException e) { throw new IOException(JGitText.get().multiPackIndexWritingCancelled, e); @@ -297,7 +299,7 @@ public class MultiPackIndexWriter { for (int i = 0; i < ctx.data.getPackCount(); i++) { List<OffsetPosition> offsetsForPack = packOffsets .get(Integer.valueOf(i)); - if (offsetsForPack.isEmpty()) { + if (offsetsForPack == null) { continue; } offsetsForPack.sort(Comparator.comparing(OffsetPosition::offset)); @@ -358,7 +360,6 @@ public class MultiPackIndexWriter { out.flush(); } - private record OffsetPosition(long offset, int position) { } @@ -367,7 +368,8 @@ public class MultiPackIndexWriter { * offset chunk must exist, and offsets larger than 2^31-1 must be stored in * it instead * - * @param offset object offset + * @param offset + * object offset * * @return true if the offset fits in 31 bits */ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/PackIndexMerger.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/PackIndexMerger.java index 89814af107..f23665849e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/PackIndexMerger.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/midx/PackIndexMerger.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2025, Google Inc. + * Copyright (C) 2025, Google LLC * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackExt.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackExt.java index e6daaeaca9..d5bb5f2e2f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackExt.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackExt.java @@ -36,7 +36,10 @@ public enum PackExt { COMMIT_GRAPH("graph"), //$NON-NLS-1$ /** An object size index. */ - OBJECT_SIZE_INDEX("objsize"); //$NON-NLS-1$ + OBJECT_SIZE_INDEX("objsize"), //$NON-NLS-1$ + + /** Multi pack index */ + MULTI_PACK_INDEX("midx"); //$NON-NLS-1$ private final String ext; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java index f025d4e3d5..27fb814f64 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java @@ -2394,17 +2394,46 @@ public class PackWriter implements AutoCloseable { * this method once for each representation available for an object, to * allow the writer to find the most suitable one for the output. * + * This method tries to take a very simple approach to avoiding delta chains + * during object reuse by selecting from the oldest pack that contains them. + * This helps select many objects from the same pack which helps make good + * use of any WindowCache caching, and it helps prevent cycles because a + * pack must not have a cycle in the delta chain. If both objects A and B + * are chosen out of the same source pack then there cannot be an A->B->A + * cycle. + * + * The oldest pack is also the most likely to have the smallest deltas. It + * generally is the biggest pack in the system and probably came from the + * clone (or last GC) of this repository, where all objects were previously + * considered and packed tightly together. If an object appears again (for + * example due to a revert and a push into this repository) the newer copy + * won't be nearly as small as the older delta version of it, even if the + * newer one is also itself a delta. + * + * Thus this method is optimized for being called in an order that presumes + * that earlier representations are better than later ones, and it expects + * representations from older pack files to be tested first, and it will + * shortcut any searching once it is satisfied with the selected + * representation. Perhaps ideally representation testing ordering should be + * based on packfile object count instead of age since file age can be + * altered, and be deceiving for other reasons. Perhaps the presence of a + * bitmap file for a pack file should prioritize it to be tested even + * earlier than object count? + * * @param otp * the object being packed. * @param next * the next available representation from the repository. + * @return whether the search should continue in the hopes of finding a + * better representation */ - public void select(ObjectToPack otp, StoredObjectRepresentation next) { + public boolean select(ObjectToPack otp, StoredObjectRepresentation next) { int nFmt = next.getFormat(); if (!cachedPacks.isEmpty()) { - if (otp.isEdge()) - return; + if (otp.isEdge()) { + return false; + } if (nFmt == PACK_WHOLE || nFmt == PACK_DELTA) { for (CachedPack pack : cachedPacks) { if (pack.hasObject(otp, next)) { @@ -2412,7 +2441,7 @@ public class PackWriter implements AutoCloseable { otp.clearDeltaBase(); otp.clearReuseAsIs(); pruneCurrentObjectList = true; - return; + return false; } } } @@ -2422,23 +2451,22 @@ public class PackWriter implements AutoCloseable { ObjectId baseId = next.getDeltaBase(); ObjectToPack ptr = objectsMap.get(baseId); if (ptr != null && !ptr.isEdge()) { - otp.setDeltaBase(ptr); - otp.setReuseAsIs(); - } else if (thin && have(ptr, baseId)) { - otp.setDeltaBase(baseId); - otp.setReuseAsIs(); - } else { - otp.clearDeltaBase(); - otp.clearReuseAsIs(); + return useDelta(otp, next, ptr); + } + if (thin && have(ptr, baseId)) { + return useDelta(otp, next, baseId); } + otp.clearDeltaBase(); + otp.clearReuseAsIs(); } else if (nFmt == PACK_WHOLE && config.isReuseObjects()) { int nWeight = next.getWeight(); if (otp.isReuseAsIs() && !otp.isDeltaRepresentation()) { // We've chosen another PACK_WHOLE format for this object, // choose the one that has the smaller compressed size. // - if (otp.getWeight() <= nWeight) - return; + if (otp.getWeight() < nWeight) { + return true; + } } otp.clearDeltaBase(); otp.setReuseAsIs(); @@ -2450,6 +2478,15 @@ public class PackWriter implements AutoCloseable { otp.setDeltaAttempted(reuseDeltas && next.wasDeltaAttempted()); otp.select(next); + return true; + } + + private boolean useDelta(ObjectToPack otp, StoredObjectRepresentation rep, ObjectId base) { + otp.setDeltaBase(base); + otp.setReuseAsIs(); + otp.setDeltaAttempted(reuseDeltas && rep.wasDeltaAttempted()); + otp.select(rep); + return false; } private final boolean have(ObjectToPack ptr, AnyObjectId objectId) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/revplot/PlotWalk.java b/org.eclipse.jgit/src/org/eclipse/jgit/revplot/PlotWalk.java index 0f6bd2d6cc..c8c454a228 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/revplot/PlotWalk.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/revplot/PlotWalk.java @@ -169,8 +169,9 @@ public class PlotWalk extends RevWalk { } long timeof(RevObject o) { - if (o instanceof RevCommit) - return ((RevCommit) o).getCommitTime(); + if (o instanceof RevCommit) { + return ((RevCommit) o).getCommitTime() * 1000L; + } if (o instanceof RevTag) { RevTag tag = (RevTag) o; try { @@ -179,7 +180,7 @@ public class PlotWalk extends RevWalk { return 0; } PersonIdent who = tag.getTaggerIdent(); - return who != null ? who.getWhen().getTime() : 0; + return who != null ? who.getWhenAsInstant().toEpochMilli() : 0; } return 0; } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/AmazonS3.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/AmazonS3.java index aaf9f8a08a..9d9f5495fe 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/AmazonS3.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/AmazonS3.java @@ -760,6 +760,15 @@ public class AmazonS3 { SAXParserFactory saxParserFactory = SAXParserFactory .newInstance(); saxParserFactory.setNamespaceAware(true); + saxParserFactory.setFeature( + "http://xml.org/sax/features/external-general-entities", //$NON-NLS-1$ + false); + saxParserFactory.setFeature( + "http://xml.org/sax/features/external-parameter-entities", //$NON-NLS-1$ + false); + saxParserFactory.setFeature( + "http://apache.org/xml/features/disallow-doctype-decl", //$NON-NLS-1$ + true); xr = saxParserFactory.newSAXParser().getXMLReader(); } catch (SAXException | ParserConfigurationException e) { throw new IOException( diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java index a65d0b756c..c9a48bf0e3 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/TransportHttp.java @@ -106,6 +106,7 @@ import org.eclipse.jgit.util.StringUtils; import org.eclipse.jgit.util.SystemReader; import org.eclipse.jgit.util.TemporaryBuffer; import org.eclipse.jgit.util.io.DisabledOutputStream; +import org.eclipse.jgit.util.io.SilentInputStream; import org.eclipse.jgit.util.io.UnionInputStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -1558,7 +1559,8 @@ public class TransportHttp extends HttpTransport implements WalkTransport, throws TransportException { svc = new MultiRequestService(SVC_UPLOAD_PACK, getProtocolVersion()); - try (InputStream svcIn = svc.getInputStream(); + try (InputStream svcIn = new SilentInputStream( + svc.getInputStream()); OutputStream svcOut = svc.getOutputStream()) { init(svcIn, svcOut); super.doFetch(monitor, want, have, outputStream); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/filter/ChangedPathTreeFilter.java b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/filter/ChangedPathTreeFilter.java index 2400e12240..a74b9b617f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/filter/ChangedPathTreeFilter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/treewalk/filter/ChangedPathTreeFilter.java @@ -125,6 +125,7 @@ public class ChangedPathTreeFilter extends TreeFilter { return paths; } + @SuppressWarnings("nls") @Override public String toString() { return "(CHANGED_PATH(" + pathFilter.toString() + ")" // diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java index 59bbacfa76..6a40fad1db 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/FS.java @@ -363,6 +363,7 @@ public abstract class FS { private static FileStoreAttributes getFileStoreAttributes(Path dir) { FileStore s; + CompletableFuture<Optional<FileStoreAttributes>> f = null; try { if (Files.exists(dir)) { s = Files.getFileStore(dir); @@ -385,7 +386,7 @@ public abstract class FS { return FALLBACK_FILESTORE_ATTRIBUTES; } - CompletableFuture<Optional<FileStoreAttributes>> f = CompletableFuture + f = CompletableFuture .supplyAsync(() -> { Lock lock = locks.computeIfAbsent(s, l -> new ReentrantLock()); @@ -455,10 +456,13 @@ public abstract class FS { } // fall through and return fallback } catch (IOException | ExecutionException | CancellationException e) { + cancel(f); LOG.error(e.getMessage(), e); } catch (TimeoutException | SecurityException e) { + cancel(f); // use fallback } catch (InterruptedException e) { + cancel(f); LOG.error(e.getMessage(), e); Thread.currentThread().interrupt(); } @@ -467,6 +471,13 @@ public abstract class FS { return FALLBACK_FILESTORE_ATTRIBUTES; } + private static void cancel( + CompletableFuture<Optional<FileStoreAttributes>> f) { + if (f != null) { + f.cancel(true); + } + } + @SuppressWarnings("boxing") private static Duration measureMinimalRacyInterval(Path dir) { LOG.debug("{}: start measure minimal racy interval in {}", //$NON-NLS-1$ diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/Iterators.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/Iterators.java new file mode 100644 index 0000000000..74b728bdf7 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/Iterators.java @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2025, NVIDIA Corporation. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.util; + +import java.util.Iterator; + +/** + * Utility class for Iterators + * + * @since 6.10.2 + */ +public class Iterators { + /** + * Create an iterator which traverses an array in reverse. + * + * @param array T[] + * @return Iterator<T> + */ + public static <T> Iterator<T> reverseIterator(T[] array) { + return new Iterator<>() { + int index = array.length; + + @Override + public boolean hasNext() { + return index > 0; + } + + @Override + public T next() { + return array[--index]; + } + }; + } + + /** + * Make an iterable for easy use in modern for loops. + * + * @param iterator Iterator<T> + * @return Iterable<T> + */ + public static <T> Iterable<T> iterable(Iterator<T> iterator) { + return new Iterable<>() { + @Override + public Iterator<T> iterator() { + return iterator; + } + }; + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/util/io/SilentInputStream.java b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/SilentInputStream.java new file mode 100644 index 0000000000..8c2c61a434 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/util/io/SilentInputStream.java @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2025 Thomas Wolf <twolf@apache.org> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.util.io; + +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * An {@link InputStream} that swallows exceptions on {@link #close()}. + * + * @since 7.4 + */ +public class SilentInputStream extends FilterInputStream { + + private static final Logger LOG = LoggerFactory + .getLogger(SilentInputStream.class); + + /** + * Wraps an existing {@link InputStream}. + * + * @param in + * {@link InputStream} to wrap + */ + public SilentInputStream(InputStream in) { + super(in); + } + + @Override + public void close() throws IOException { + try { + super.close(); + } catch (IOException e) { + if (LOG.isDebugEnabled()) { + LOG.debug("Exception ignored while closing input stream", e); //$NON-NLS-1$ + } + } + } +} |