aboutsummaryrefslogtreecommitdiffstats
path: root/org.eclipse.jgit/src
diff options
context:
space:
mode:
Diffstat (limited to 'org.eclipse.jgit/src')
-rw-r--r--org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java40
-rw-r--r--org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java48
-rw-r--r--org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java108
-rw-r--r--org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java36
4 files changed, 221 insertions, 11 deletions
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java
index c08a92e5a7..97473bba2a 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java
@@ -14,6 +14,7 @@ import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX;
import static org.eclipse.jgit.internal.storage.pack.PackExt.COMMIT_GRAPH;
import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX;
import static org.eclipse.jgit.internal.storage.pack.PackExt.KEEP;
+import static org.eclipse.jgit.internal.storage.pack.PackExt.OBJECT_SIZE_INDEX;
import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;
import static org.eclipse.jgit.internal.storage.pack.PackExt.REVERSE_INDEX;
@@ -131,7 +132,7 @@ public class GC {
private static final Set<PackExt> PARENT_EXTS = Set.of(PACK, KEEP);
private static final Set<PackExt> CHILD_EXTS = Set.of(BITMAP_INDEX, INDEX,
- REVERSE_INDEX);
+ REVERSE_INDEX, OBJECT_SIZE_INDEX);
private static final int DEFAULT_AUTOPACKLIMIT = 50;
@@ -412,6 +413,10 @@ public class GC {
*/
private void removeOldPack(PackFile packFile, int deleteOptions)
throws IOException {
+ if (!packFile.exists()) {
+ return;
+ }
+
if (pconfig.isPreserveOldPacks()) {
File oldPackDir = repo.getObjectDatabase().getPreservedDirectory();
FileUtils.mkdir(oldPackDir, true);
@@ -1340,6 +1345,7 @@ public class GC {
idxChannel.force(true);
}
+
if (pw.isReverseIndexEnabled()) {
File tmpReverseIndexFile = new File(packdir,
tmpBase + REVERSE_INDEX.getTmpExtension());
@@ -1356,6 +1362,19 @@ public class GC {
.newOutputStream(channel)) {
pw.writeReverseIndex(stream);
channel.force(true);
+ }
+ }
+
+ // write the object size
+ if (pconfig.isWriteObjSizeIndex()) {
+ File tmpSizeIdx = new File(packdir, tmpBase + ".objsize_tmp"); //$NON-NLS-1$
+ tmpExts.put(OBJECT_SIZE_INDEX, tmpSizeIdx);
+ try (FileOutputStream fos = new FileOutputStream(tmpSizeIdx);
+ FileChannel idxChannel = fos.getChannel();
+ OutputStream idxStream = Channels
+ .newOutputStream(idxChannel)) {
+ pw.writeObjectSizeIndex(idxStream);
+ idxChannel.force(true);
}
}
@@ -1525,6 +1544,11 @@ public class GC {
*/
public long numberOfBitmaps;
+ /**
+ * The number of objects in the size-index of the packs
+ */
+ public long numberOfSizeIndexedObjects;
+
@Override
public String toString() {
final StringBuilder b = new StringBuilder();
@@ -1540,6 +1564,8 @@ public class GC {
b.append(", sizeOfLooseObjects=").append(sizeOfLooseObjects); //$NON-NLS-1$
b.append(", sizeOfPackedObjects=").append(sizeOfPackedObjects); //$NON-NLS-1$
b.append(", numberOfBitmaps=").append(numberOfBitmaps); //$NON-NLS-1$
+ b.append(", numberOfSizeIndexedObjects=") //$NON-NLS-1$
+ .append(numberOfSizeIndexedObjects);
return b.toString();
}
}
@@ -1548,7 +1574,7 @@ public class GC {
* Returns information about objects and pack files for a FileRepository.
*
* @return information about objects and pack files for a FileRepository
- * @throws java.io.IOException
+ * @throws java.io.IOException
* if an IO error occurred
*/
public RepoStatistics getStatistics() throws IOException {
@@ -1560,16 +1586,16 @@ public class GC {
ret.numberOfPackedObjects += packedObjects;
ret.numberOfPackFiles++;
ret.sizeOfPackedObjects += p.getPackFile().length();
+ ret.numberOfSizeIndexedObjects += p.getObjectSizeIndexCount();
if (p.getBitmapIndex() != null) {
ret.numberOfBitmaps += p.getBitmapIndex().getBitmapCount();
if (latestBitmapTime == 0L) {
latestBitmapTime = p.getFileSnapshot().lastModifiedInstant().toEpochMilli();
}
- }
- else if (latestBitmapTime == 0L) {
- ret.numberOfPackFilesSinceBitmap++;
- ret.numberOfObjectsSinceBitmap += packedObjects;
- }
+ } else if (latestBitmapTime == 0L) {
+ ret.numberOfPackFilesSinceBitmap++;
+ ret.numberOfObjectsSinceBitmap += packedObjects;
+ }
}
File objDir = repo.getObjectsDirectory();
String[] fanout = objDir.list();
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java
index 746e124e1f..d97d5a7ccd 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java
@@ -72,6 +72,12 @@ public class ObjectDirectoryPackParser extends PackParser {
*/
private File tmpIdx;
+ /**
+ * Path of the object-size index created for the pack, to filter quickly
+ * objects by size in partial clones
+ */
+ private File tmpObjectSizeIndex;
+
/** Read/write handle to {@link #tmpPack} while it is being parsed. */
private RandomAccessFile out;
@@ -163,6 +169,7 @@ public class ObjectDirectoryPackParser extends PackParser {
throws IOException {
tmpPack = File.createTempFile("incoming_", ".pack", db.getDirectory()); //$NON-NLS-1$ //$NON-NLS-2$
tmpIdx = new File(db.getDirectory(), baseName(tmpPack) + ".idx"); //$NON-NLS-1$
+
try {
out = new RandomAccessFile(tmpPack, "rw"); //$NON-NLS-1$
@@ -178,6 +185,14 @@ public class ObjectDirectoryPackParser extends PackParser {
tmpPack.setReadOnly();
tmpIdx.setReadOnly();
+ if (pconfig.isWriteObjSizeIndex()) {
+ tmpObjectSizeIndex = new File(db.getDirectory(),
+ baseName(tmpPack)
+ + PackExt.OBJECT_SIZE_INDEX.getExtension());
+ writeObjectSizeIndex(pconfig.getMinBytesForObjSizeIndex());
+ tmpObjectSizeIndex.setReadOnly();
+ }
+
return renameAndOpenPack(getLockMessage());
} finally {
if (def != null)
@@ -295,6 +310,9 @@ public class ObjectDirectoryPackParser extends PackParser {
tmpIdx.deleteOnExit();
if (tmpPack != null && !tmpPack.delete() && tmpPack.exists())
tmpPack.deleteOnExit();
+ if (tmpObjectSizeIndex != null && !tmpObjectSizeIndex.delete()
+ && tmpObjectSizeIndex.exists())
+ tmpPack.deleteOnExit();
}
@Override
@@ -395,6 +413,15 @@ public class ObjectDirectoryPackParser extends PackParser {
}
}
+ private void writeObjectSizeIndex(int minSize) throws IOException {
+ try (FileOutputStream os = new FileOutputStream(tmpObjectSizeIndex)) {
+ PackObjectSizeIndexWriter iw = PackObjectSizeIndexWriter
+ .createWriter(os, minSize);
+ iw.write(getSortedObjectList(null));
+ os.getChannel().force(true);
+ }
+ }
+
private PackLock renameAndOpenPack(String lockMessage)
throws IOException {
if (!keepEmpty && getObjectCount() == 0) {
@@ -469,6 +496,27 @@ public class ObjectDirectoryPackParser extends PackParser {
JGitText.get().cannotMoveIndexTo, finalIdx), e);
}
+ if (pconfig.isWriteObjSizeIndex() && tmpObjectSizeIndex != null) {
+ PackFile finalObjectSizeIndex = finalPack
+ .create(PackExt.OBJECT_SIZE_INDEX);
+ try {
+ FileUtils.rename(tmpObjectSizeIndex, finalObjectSizeIndex,
+ StandardCopyOption.ATOMIC_MOVE);
+ } catch (IOException e) {
+ cleanupTemporaryFiles();
+ keep.unlock();
+ if (!finalPack.delete())
+ finalPack.deleteOnExit();
+ if (!finalIdx.delete()) {
+ finalIdx.deleteOnExit();
+ }
+ throw new IOException(MessageFormat
+ .format(JGitText.get().cannotMoveIndexTo,
+ finalObjectSizeIndex),
+ e);
+ }
+ }
+
boolean interrupted = false;
try {
FileSnapshot snapshot = FileSnapshot.save(finalPack);
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java
index 5813d39e9a..f2f54947af 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java
@@ -17,9 +17,11 @@ import static org.eclipse.jgit.internal.storage.pack.PackExt.KEEP;
import static org.eclipse.jgit.internal.storage.pack.PackExt.REVERSE_INDEX;
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_CORE_SECTION;
import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_PACKED_INDEX_GIT_USE_STRONGREFS;
+import static org.eclipse.jgit.internal.storage.pack.PackExt.OBJECT_SIZE_INDEX;
import java.io.EOFException;
import java.io.File;
+import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InterruptedIOException;
@@ -126,6 +128,10 @@ public class Pack implements Iterable<PackIndex.MutableEntry> {
private Optionally<PackReverseIndex> reverseIdx = Optionally.empty();
+ private volatile PackObjectSizeIndex loadedObjSizeIdx;
+
+ private volatile boolean attemptLoadObjSizeIdx;
+
private Optionally<PackBitmapIndex> bitmapIdx = Optionally.empty();
/**
@@ -210,6 +216,52 @@ public class Pack implements Iterable<PackIndex.MutableEntry> {
}
}
+ private PackObjectSizeIndex objectSizeIndex() throws IOException {
+ if (loadedObjSizeIdx != null) {
+ return loadedObjSizeIdx;
+ }
+
+ if (attemptLoadObjSizeIdx) {
+ return null;
+ }
+
+ synchronized (this) {
+ if (loadedObjSizeIdx != null) {
+ return loadedObjSizeIdx;
+ }
+
+ PackObjectSizeIndex sizeIdx;
+ try {
+ long start = System.currentTimeMillis();
+ PackFile sizeIdxFile = packFile.create(OBJECT_SIZE_INDEX);
+ if (attemptLoadObjSizeIdx || !sizeIdxFile.exists()) {
+ attemptLoadObjSizeIdx = true;
+ return null;
+ }
+ sizeIdx = PackObjectSizeIndexLoader.load(
+ new FileInputStream(sizeIdxFile.getAbsoluteFile()));
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(String.format(
+ "Opening obj size index %s, size %.3f MB took %d ms", //$NON-NLS-1$
+ sizeIdxFile.getAbsolutePath(),
+ Float.valueOf(
+ sizeIdxFile.length() / (1024f * 1024)),
+ Long.valueOf(System.currentTimeMillis() - start)));
+ }
+
+ loadedObjSizeIdx = sizeIdx;
+ } catch (InterruptedIOException e) {
+ // don't invalidate the pack, we are interrupted from
+ // another thread
+ return null;
+ } finally {
+ attemptLoadObjSizeIdx = true;
+ }
+ }
+
+ return loadedObjSizeIdx;
+ }
+
/**
* Get the File object which locates this pack on disk.
*
@@ -231,6 +283,62 @@ public class Pack implements Iterable<PackIndex.MutableEntry> {
}
/**
+ * Get the object size index for this pack file
+ *
+ * @return the object size index for this pack file if it exists (null
+ * otherwise)
+ * @throws IOException
+ * problem reading the index
+ */
+ public boolean hasObjectSizeIndex() throws IOException {
+ return objectSizeIndex() != null;
+ }
+
+ /**
+ * Number of objects in the object-size index of this pack
+ *
+ * @return number of objects in the index (0 if either the index is empty or
+ * it doesn't exist)
+ * @throws IOException
+ * if an IO error occurred while reading the index
+ */
+ public long getObjectSizeIndexCount() throws IOException {
+ if (!hasObjectSizeIndex()) {
+ return 0;
+ }
+
+ return objectSizeIndex().getObjectCount();
+ }
+
+ /**
+ * Return the size of the object from the object-size index.
+ *
+ * Caller MUST check that the pack has object-size index
+ * ({@link #hasObjectSizeIndex()}) and that the pack contains the object.
+ *
+ * @param id
+ * object id of an object in the pack
+ * @return size of the object from the index. Negative if the object is not
+ * in the index.
+ * @throws IOException
+ * if an IO error occurred while reading the index
+ */
+ public long getIndexedObjectSize(AnyObjectId id) throws IOException {
+ int idxPos = idx().findPosition(id);
+ if (idxPos < 0) {
+ return -1;
+ }
+
+ PackObjectSizeIndex sizeIdx = objectSizeIndex();
+ if (sizeIdx == null) {
+ throw new IllegalStateException(
+ "Asking indexed size from a pack without object size index"); //$NON-NLS-1$
+ }
+
+ return sizeIdx.getSize(idxPos);
+ }
+
+ /**
* Get name extracted from {@code pack-*.pack} pattern.
*
* @return name extracted from {@code pack-*.pack} pattern.
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java
index 55e047bd43..97a854b8cd 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java
@@ -166,7 +166,7 @@ public class PackInserter extends ObjectInserter {
long offset = beginObject(type, len);
packOut.compress.write(data, off, len);
packOut.compress.finish();
- return endObject(id, offset);
+ return endObject(id, offset, len, type);
}
@Override
@@ -195,7 +195,7 @@ public class PackInserter extends ObjectInserter {
len -= n;
}
packOut.compress.finish();
- return endObject(md.toObjectId(), offset);
+ return endObject(md.toObjectId(), offset, len, type);
}
private long beginObject(int type, long len) throws IOException {
@@ -207,10 +207,12 @@ public class PackInserter extends ObjectInserter {
return offset;
}
- private ObjectId endObject(ObjectId id, long offset) {
+ private ObjectId endObject(ObjectId id, long offset, long len, int type) {
PackedObjectInfo obj = new PackedObjectInfo(id);
+ obj.setType(type);
obj.setOffset(offset);
obj.setCRC((int) packOut.crc32.getValue());
+ obj.setFullSize(len);
objectList.add(obj);
objectMap.addIfAbsent(obj);
return id;
@@ -223,6 +225,12 @@ public class PackInserter extends ObjectInserter {
p.substring(0, p.lastIndexOf('.')) + ".idx"); //$NON-NLS-1$
}
+ private static File getFileFor(File packFile, PackExt ext) {
+ String p = packFile.getName();
+ return new File(packFile.getParentFile(),
+ p.substring(0, p.lastIndexOf('.')) + ext.getExtension());
+ }
+
private void beginPack() throws IOException {
objectList = new BlockList<>();
objectMap = new ObjectIdOwnerMap<>();
@@ -272,7 +280,11 @@ public class PackInserter extends ObjectInserter {
Collections.sort(objectList);
File tmpIdx = idxFor(tmpPack); // TODO(nasserg) Use PackFile?
writePackIndex(tmpIdx, packHash, objectList);
-
+ File tmpObjSizeIdx = null;
+ if (pconfig.isWriteObjSizeIndex()) {
+ tmpObjSizeIdx = getFileFor(tmpPack, PackExt.OBJECT_SIZE_INDEX);
+ writeObjectSizeIndex(tmpObjSizeIdx, objectList, pconfig);
+ }
PackFile realPack = new PackFile(db.getPackDirectory(),
computeName(objectList), PackExt.PACK);
db.closeAllPackHandles(realPack);
@@ -297,6 +309,13 @@ public class PackInserter extends ObjectInserter {
realIdx), e);
}
+ if (tmpObjSizeIdx != null) {
+ PackFile realObjSizeIdx = realPack
+ .create(PackExt.OBJECT_SIZE_INDEX);
+ tmpObjSizeIdx.setReadOnly();
+ FileUtils.rename(tmpObjSizeIdx, realObjSizeIdx, ATOMIC_MOVE);
+ }
+
boolean interrupted = false;
try {
FileSnapshot snapshot = FileSnapshot.save(realPack);
@@ -327,6 +346,15 @@ public class PackInserter extends ObjectInserter {
}
}
+ private static void writeObjectSizeIndex(File objIdx,
+ List<PackedObjectInfo> list, PackConfig cfg) throws IOException {
+ try (OutputStream os = new FileOutputStream(objIdx)) {
+ PackObjectSizeIndexWriter w = PackObjectSizeIndexWriter
+ .createWriter(os, cfg.getMinBytesForObjSizeIndex());
+ w.write(list);
+ }
+ }
+
private ObjectId computeName(List<PackedObjectInfo> list) {
SHA1 md = digest().reset();
byte[] buf = buffer();