diff options
Diffstat (limited to 'org.eclipse.jgit/src/org')
11 files changed, 455 insertions, 231 deletions
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java index 9d215ca455..95265feb4e 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/JGitText.java @@ -771,6 +771,7 @@ public class JGitText extends TranslationBundle { /***/ public String unmergedPaths; /***/ public String unpackException; /***/ public String unreadablePackIndex; + /***/ public String unrecognizedPackExtension; /***/ public String unrecognizedRef; /***/ public String unsetMark; /***/ public String unsupportedAlternates; diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java index 75de3be89e..9ffff9f662 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/GC.java @@ -12,6 +12,8 @@ package org.eclipse.jgit.internal.storage.file; import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX; +import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK; +import static org.eclipse.jgit.internal.storage.pack.PackExt.KEEP; import java.io.File; import java.io.FileOutputStream; @@ -346,7 +348,7 @@ public class GC { if (shouldLoosen) { loosen(inserter, reader, oldPack, ids); } - prunePack(oldName); + prunePack(oldPack.getPackFile()); } } @@ -360,19 +362,17 @@ public class GC { * moves the pack file to the preserved directory * * @param packFile - * @param packName - * @param ext * @param deleteOptions * @throws IOException */ - private void removeOldPack(File packFile, String packName, PackExt ext, - int deleteOptions) throws IOException { + private void removeOldPack(PackFile packFile, int deleteOptions) + throws IOException { if (pconfig.isPreserveOldPacks()) { File oldPackDir = repo.getObjectDatabase().getPreservedDirectory(); FileUtils.mkdir(oldPackDir, true); - String oldPackName = "pack-" + packName + ".old-" + ext.getExtension(); //$NON-NLS-1$ //$NON-NLS-2$ - File oldPackFile = new File(oldPackDir, oldPackName); + PackFile oldPackFile = packFile + .createPreservedForDirectory(oldPackDir); FileUtils.rename(packFile, oldPackFile); } else { FileUtils.delete(packFile, deleteOptions); @@ -401,27 +401,21 @@ public class GC { * ".index" file and when failing to delete the ".pack" file we are left * with a ".pack" file without a ".index" file. * - * @param packName + * @param packFile */ - private void prunePack(String packName) { - PackExt[] extensions = PackExt.values(); + private void prunePack(PackFile packFile) { try { // Delete the .pack file first and if this fails give up on deleting // the other files int deleteOptions = FileUtils.RETRY | FileUtils.SKIP_MISSING; - for (PackExt ext : extensions) - if (PackExt.PACK.equals(ext)) { - File f = nameFor(packName, "." + ext.getExtension()); //$NON-NLS-1$ - removeOldPack(f, packName, ext, deleteOptions); - break; - } + removeOldPack(packFile.create(PackExt.PACK), deleteOptions); + // The .pack file has been deleted. Delete as many as the other // files as you can. deleteOptions |= FileUtils.IGNORE_ERRORS; - for (PackExt ext : extensions) { + for (PackExt ext : PackExt.values()) { if (!PackExt.PACK.equals(ext)) { - File f = nameFor(packName, "." + ext.getExtension()); //$NON-NLS-1$ - removeOldPack(f, packName, ext, deleteOptions); + removeOldPack(packFile.create(ext), deleteOptions); } } } catch (IOException e) { @@ -973,20 +967,21 @@ public class GC { return; } - String base = null; + String latestId = null; for (String n : fileNames) { - if (n.endsWith(PACK_EXT) || n.endsWith(KEEP_EXT)) { - base = n.substring(0, n.lastIndexOf('.')); - } else { - if (base == null || !n.startsWith(base)) { - try { - Path delete = packDir.resolve(n); - FileUtils.delete(delete.toFile(), - FileUtils.RETRY | FileUtils.SKIP_MISSING); - LOG.warn(JGitText.get().deletedOrphanInPackDir, delete); - } catch (IOException e) { - LOG.error(e.getMessage(), e); - } + PackFile pf = new PackFile(packDir.toFile(), n); + PackExt ext = pf.getPackExt(); + if (ext.equals(PACK) || ext.equals(KEEP)) { + latestId = pf.getId(); + } + if (latestId == null || !pf.getId().equals(latestId)) { + // no pack or keep for this id + try { + FileUtils.delete(pf, + FileUtils.RETRY | FileUtils.SKIP_MISSING); + LOG.warn(JGitText.get().deletedOrphanInPackDir, pf); + } catch (IOException e) { + LOG.error(e.getMessage(), e); } } } @@ -1168,7 +1163,7 @@ public class GC { checkCancelled(); // create temporary files - String id = pw.computeName().getName(); + ObjectId id = pw.computeName(); File packdir = repo.getObjectDatabase().getPackDirectory(); packdir.mkdirs(); tmpPack = File.createTempFile("gc_", ".pack_tmp", packdir); //$NON-NLS-1$ //$NON-NLS-2$ @@ -1218,7 +1213,8 @@ public class GC { } // rename the temporary files to real files - File realPack = nameFor(id, ".pack"); //$NON-NLS-1$ + File packDir = repo.getObjectDatabase().getPackDirectory(); + PackFile realPack = new PackFile(packDir, id, PackExt.PACK); repo.getObjectDatabase().closeAllPackHandles(realPack); tmpPack.setReadOnly(); @@ -1228,8 +1224,7 @@ public class GC { File tmpExt = tmpEntry.getValue(); tmpExt.setReadOnly(); - File realExt = nameFor(id, - "." + tmpEntry.getKey().getExtension()); //$NON-NLS-1$ + PackFile realExt = new PackFile(packDir, id, tmpEntry.getKey()); try { FileUtils.rename(tmpExt, realExt, StandardCopyOption.ATOMIC_MOVE); @@ -1275,11 +1270,6 @@ public class GC { } } - private File nameFor(String name, String ext) { - File packdir = repo.getObjectDatabase().getPackDirectory(); - return new File(packdir, "pack-" + name + ext); //$NON-NLS-1$ - } - private void checkCancelled() throws CancelledException { if (pm.isCancelled() || Thread.currentThread().isInterrupted()) { throw new CancelledException(JGitText.get().operationCanceled); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalCachedPack.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalCachedPack.java index ae5bce6985..f112947bae 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalCachedPack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/LocalCachedPack.java @@ -17,6 +17,7 @@ import java.util.List; import org.eclipse.jgit.internal.storage.pack.CachedPack; import org.eclipse.jgit.internal.storage.pack.ObjectToPack; +import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.internal.storage.pack.PackOutputStream; import org.eclipse.jgit.internal.storage.pack.StoredObjectRepresentation; @@ -88,6 +89,6 @@ class LocalCachedPack extends CachedPack { private String getPackFilePath(String packName) { final File packDir = odb.getPackDirectory(); - return new File(packDir, "pack-" + packName + ".pack").getPath(); //$NON-NLS-1$ //$NON-NLS-2$ + return new PackFile(packDir, packName, PackExt.PACK).getPath(); } } diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java index e71a960603..627facca02 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectory.java @@ -11,8 +11,9 @@ package org.eclipse.jgit.internal.storage.file; import static java.nio.charset.StandardCharsets.UTF_8; -import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK; +import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX; +import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX; import java.io.BufferedReader; import java.io.File; @@ -79,7 +80,7 @@ public class ObjectDirectory extends FileObjectDatabase { private final PackDirectory packed; - private final File preservedDirectory; + private final PackDirectory preserved; private final File alternatesFile; @@ -117,10 +118,11 @@ public class ObjectDirectory extends FileObjectDatabase { objects = dir; infoDirectory = new File(objects, "info"); //$NON-NLS-1$ File packDirectory = new File(objects, "pack"); //$NON-NLS-1$ - preservedDirectory = new File(packDirectory, "preserved"); //$NON-NLS-1$ + File preservedDirectory = new File(packDirectory, "preserved"); //$NON-NLS-1$ alternatesFile = new File(objects, Constants.INFO_ALTERNATES); loose = new LooseObjects(objects); packed = new PackDirectory(config, packDirectory); + preserved = new PackDirectory(config, preservedDirectory); this.fs = fs; this.shallowFile = shallowFile; @@ -156,7 +158,7 @@ public class ObjectDirectory extends FileObjectDatabase { * @return the location of the <code>preserved</code> directory. */ public final File getPreservedDirectory() { - return preservedDirectory; + return preserved.getDirectory(); } /** {@inheritDoc} */ @@ -216,26 +218,26 @@ public class ObjectDirectory extends FileObjectDatabase { * Add a single existing pack to the list of available pack files. */ @Override - public Pack openPack(File pack) - throws IOException { - final String p = pack.getName(); - if (p.length() != 50 || !p.startsWith("pack-") || !p.endsWith(".pack")) //$NON-NLS-1$ //$NON-NLS-2$ - throw new IOException(MessageFormat.format(JGitText.get().notAValidPack, pack)); - - // The pack and index are assumed to exist. The existence of other - // extensions needs to be explicitly checked. - // - int extensions = PACK.getBit() | INDEX.getBit(); - final String base = p.substring(0, p.length() - 4); - for (PackExt ext : PackExt.values()) { - if ((extensions & ext.getBit()) == 0) { - final String name = base + ext.getExtension(); - if (new File(pack.getParentFile(), name).exists()) - extensions |= ext.getBit(); - } - } - - Pack res = new Pack(pack, extensions); + public Pack openPack(File pack) throws IOException { + PackFile pf; + try { + pf = new PackFile(pack); + } catch (IllegalArgumentException e) { + throw new IOException( + MessageFormat.format(JGitText.get().notAValidPack, pack), + e); + } + + String p = pf.getName(); + // TODO(nasserg): See if PackFile can do these checks instead + if (p.length() != 50 || !p.startsWith("pack-") //$NON-NLS-1$ + || !pf.getPackExt().equals(PACK)) { + throw new IOException( + MessageFormat.format(JGitText.get().notAValidPack, pack)); + } + + PackFile bitmapIdx = pf.create(BITMAP_INDEX); + Pack res = new Pack(pack, bitmapIdx.exists() ? bitmapIdx : null); packed.insert(res); return res; } @@ -250,7 +252,13 @@ public class ObjectDirectory extends FileObjectDatabase { @Override public boolean has(AnyObjectId objectId) { return loose.hasCached(objectId) - || hasPackedInSelfOrAlternate(objectId, null) + || hasPackedOrLooseInSelfOrAlternate(objectId) + || (restoreFromSelfOrAlternate(objectId, null) + && hasPackedOrLooseInSelfOrAlternate(objectId)); + } + + private boolean hasPackedOrLooseInSelfOrAlternate(AnyObjectId objectId) { + return hasPackedInSelfOrAlternate(objectId, null) || hasLooseInSelfOrAlternate(objectId, null); } @@ -319,6 +327,15 @@ public class ObjectDirectory extends FileObjectDatabase { @Override ObjectLoader openObject(WindowCursor curs, AnyObjectId objectId) throws IOException { + ObjectLoader ldr = openObjectWithoutRestoring(curs, objectId); + if (ldr == null && restoreFromSelfOrAlternate(objectId, null)) { + ldr = openObjectWithoutRestoring(curs, objectId); + } + return ldr; + } + + private ObjectLoader openObjectWithoutRestoring(WindowCursor curs, AnyObjectId objectId) + throws IOException { if (loose.hasCached(objectId)) { ObjectLoader ldr = openLooseObject(curs, objectId); if (ldr != null) { @@ -380,8 +397,16 @@ public class ObjectDirectory extends FileObjectDatabase { } @Override - long getObjectSize(WindowCursor curs, AnyObjectId id) - throws IOException { + long getObjectSize(WindowCursor curs, AnyObjectId id) throws IOException { + long sz = getObjectSizeWithoutRestoring(curs, id); + if (0 > sz && restoreFromSelfOrAlternate(id, null)) { + sz = getObjectSizeWithoutRestoring(curs, id); + } + return sz; + } + + private long getObjectSizeWithoutRestoring(WindowCursor curs, + AnyObjectId id) throws IOException { if (loose.hasCached(id)) { long len = loose.getSize(curs, id); if (0 <= len) { @@ -449,6 +474,51 @@ public class ObjectDirectory extends FileObjectDatabase { } } + private boolean restoreFromSelfOrAlternate(AnyObjectId objectId, + Set<AlternateHandle.Id> skips) { + if (restoreFromSelf(objectId)) { + return true; + } + + skips = addMe(skips); + for (AlternateHandle alt : myAlternates()) { + if (!skips.contains(alt.getId())) { + if (alt.db.restoreFromSelfOrAlternate(objectId, skips)) { + return true; + } + } + } + return false; + } + + private boolean restoreFromSelf(AnyObjectId objectId) { + Pack preservedPack = preserved.getPack(objectId); + if (preservedPack == null) { + return false; + } + PackFile preservedFile = new PackFile(preservedPack.getPackFile()); + // Restore the index last since the set will be considered for use once + // the index appears. + for (PackExt ext : PackExt.values()) { + if (!INDEX.equals(ext)) { + restore(preservedFile.create(ext)); + } + } + restore(preservedFile.create(INDEX)); + return true; + } + + private boolean restore(PackFile preservedPack) { + PackFile restored = preservedPack + .createForDirectory(packed.getDirectory()); + try { + Files.createLink(restored.toPath(), preservedPack.toPath()); + } catch (IOException e) { + return false; + } + return true; + } + @Override InsertLooseObjectResult insertUnpackedObject(File tmp, ObjectId id, boolean createDuplicate) throws IOException { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java index 04d2ff8ab2..dba8ccd99b 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/ObjectDirectoryPackParser.java @@ -27,6 +27,7 @@ import java.util.zip.Deflater; import org.eclipse.jgit.errors.LockFailedException; import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.CoreConfig; @@ -426,10 +427,10 @@ public class ObjectDirectoryPackParser extends PackParser { d.update(oeBytes); } - final String name = ObjectId.fromRaw(d.digest()).name(); - final File packDir = new File(db.getDirectory(), "pack"); //$NON-NLS-1$ - final File finalPack = new File(packDir, "pack-" + name + ".pack"); //$NON-NLS-1$ //$NON-NLS-2$ - final File finalIdx = new File(packDir, "pack-" + name + ".idx"); //$NON-NLS-1$ //$NON-NLS-2$ + ObjectId id = ObjectId.fromRaw(d.digest()); + File packDir = new File(db.getDirectory(), "pack"); //$NON-NLS-1$ + PackFile finalPack = new PackFile(packDir, id, PackExt.PACK); + PackFile finalIdx = finalPack.create(PackExt.INDEX); final PackLock keep = new PackLock(finalPack, db.getFS()); if (!packDir.exists() && !packDir.mkdir() && !packDir.exists()) { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java index d928633a73..5efd4c5bfc 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/Pack.java @@ -12,7 +12,6 @@ package org.eclipse.jgit.internal.storage.file; -import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.KEEP; @@ -38,6 +37,7 @@ import java.util.zip.CRC32; import java.util.zip.DataFormatException; import java.util.zip.Inflater; +import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.errors.CorruptObjectException; import org.eclipse.jgit.errors.LargeObjectException; import org.eclipse.jgit.errors.MissingObjectException; @@ -51,7 +51,6 @@ import org.eclipse.jgit.errors.UnsupportedPackVersionException; import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.storage.pack.BinaryDelta; import org.eclipse.jgit.internal.storage.pack.ObjectToPack; -import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.internal.storage.pack.PackOutputStream; import org.eclipse.jgit.lib.AbbreviatedObjectId; import org.eclipse.jgit.lib.AnyObjectId; @@ -78,13 +77,9 @@ public class Pack implements Iterable<PackIndex.MutableEntry> { public static final Comparator<Pack> SORT = (a, b) -> b.packLastModified .compareTo(a.packLastModified); - private final File packFile; + private final PackFile packFile; - private final int extensions; - - private File keepFile; - - private volatile String packName; + private PackFile keepFile; final int hash; @@ -107,7 +102,8 @@ public class Pack implements Iterable<PackIndex.MutableEntry> { private volatile Exception invalidatingCause; - private boolean invalidBitmap; + @Nullable + private PackFile bitmapIdxFile; private AtomicInteger transientErrorCount = new AtomicInteger(); @@ -133,14 +129,14 @@ public class Pack implements Iterable<PackIndex.MutableEntry> { * * @param packFile * path of the <code>.pack</code> file holding the data. - * @param extensions - * additional pack file extensions with the same base as the pack + * @param bitmapIdxFile + * existing bitmap index file with the same base as the pack */ - public Pack(File packFile, int extensions) { - this.packFile = packFile; + public Pack(File packFile, @Nullable PackFile bitmapIdxFile) { + this.packFile = new PackFile(packFile); this.fileSnapshot = PackFileSnapshot.save(packFile); this.packLastModified = fileSnapshot.lastModifiedInstant(); - this.extensions = extensions; + this.bitmapIdxFile = bitmapIdxFile; // Multiply by 31 here so we can more directly combine with another // value in WindowCache.hash(), without doing the multiply there. @@ -156,16 +152,18 @@ public class Pack implements Iterable<PackIndex.MutableEntry> { idx = loadedIdx; if (idx == null) { if (invalid) { - throw new PackInvalidException(packFile, invalidatingCause); + throw new PackInvalidException(packFile, + invalidatingCause); } try { long start = System.currentTimeMillis(); - idx = PackIndex.open(extFile(INDEX)); + PackFile idxFile = packFile.create(INDEX); + idx = PackIndex.open(idxFile); if (LOG.isDebugEnabled()) { LOG.debug(String.format( "Opening pack index %s, size %.3f MB took %d ms", //$NON-NLS-1$ - extFile(INDEX).getAbsolutePath(), - Float.valueOf(extFile(INDEX).length() + idxFile.getAbsolutePath(), + Float.valueOf(idxFile.length() / (1024f * 1024)), Long.valueOf(System.currentTimeMillis() - start))); @@ -205,7 +203,7 @@ public class Pack implements Iterable<PackIndex.MutableEntry> { * * @return the File object which locates this pack on disk. */ - public File getPackFile() { + public PackFile getPackFile() { return packFile; } @@ -225,16 +223,7 @@ public class Pack implements Iterable<PackIndex.MutableEntry> { * @return name extracted from {@code pack-*.pack} pattern. */ public String getPackName() { - String name = packName; - if (name == null) { - name = getPackFile().getName(); - if (name.startsWith("pack-")) //$NON-NLS-1$ - name = name.substring("pack-".length()); //$NON-NLS-1$ - if (name.endsWith(".pack")) //$NON-NLS-1$ - name = name.substring(0, name.length() - ".pack".length()); //$NON-NLS-1$ - packName = name; - } - return name; + return packFile.getId(); } /** @@ -261,8 +250,9 @@ public class Pack implements Iterable<PackIndex.MutableEntry> { * @return true if a .keep file exist. */ public boolean shouldBeKept() { - if (keepFile == null) - keepFile = extFile(KEEP); + if (keepFile == null) { + keepFile = packFile.create(KEEP); + } return keepFile.exists(); } @@ -1132,26 +1122,28 @@ public class Pack implements Iterable<PackIndex.MutableEntry> { } synchronized PackBitmapIndex getBitmapIndex() throws IOException { - if (invalid || invalidBitmap) + if (invalid || bitmapIdxFile == null) { return null; - if (bitmapIdx == null && hasExt(BITMAP_INDEX)) { + } + if (bitmapIdx == null) { final PackBitmapIndex idx; try { - idx = PackBitmapIndex.open(extFile(BITMAP_INDEX), idx(), + idx = PackBitmapIndex.open(bitmapIdxFile, idx(), getReverseIdx()); } catch (FileNotFoundException e) { // Once upon a time this bitmap file existed. Now it // has been removed. Most likely an external gc has // removed this packfile and the bitmap - invalidBitmap = true; - return null; + bitmapIdxFile = null; + return null; } // At this point, idx() will have set packChecksum. - if (Arrays.equals(packChecksum, idx.packChecksum)) + if (Arrays.equals(packChecksum, idx.packChecksum)) { bitmapIdx = idx; - else - invalidBitmap = true; + } else { + bitmapIdxFile = null; + } } return bitmapIdx; } @@ -1187,17 +1179,6 @@ public class Pack implements Iterable<PackIndex.MutableEntry> { } } - private File extFile(PackExt ext) { - String p = packFile.getName(); - int dot = p.lastIndexOf('.'); - String b = (dot < 0) ? p : p.substring(0, dot); - return new File(packFile.getParentFile(), b + '.' + ext.getExtension()); - } - - private boolean hasExt(PackExt ext) { - return (extensions & ext.getBit()) != 0; - } - @SuppressWarnings("nls") @Override public String toString() { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java index b2ba36bf91..73745d8c64 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackDirectory.java @@ -10,6 +10,8 @@ package org.eclipse.jgit.internal.storage.file; +import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX; +import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX; import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK; import java.io.File; @@ -20,13 +22,14 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.EnumMap; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicReference; +import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.errors.CorruptObjectException; import org.eclipse.jgit.errors.PackInvalidException; import org.eclipse.jgit.errors.PackMismatchException; @@ -121,21 +124,36 @@ class PackDirectory { * * @param objectId * identity of the object to test for existence of. - * @return true if the specified object is stored in this PackDirectory. + * @return {@code true} if the specified object is stored in this PackDirectory. */ boolean has(AnyObjectId objectId) { + return getPack(objectId) != null; + } + + /** + * Get the {@link org.eclipse.jgit.internal.storage.file.Pack} for the + * specified object if it is stored in this PackDirectory. + * + * @param objectId + * identity of the object to find the Pack for. + * @return {@link org.eclipse.jgit.internal.storage.file.Pack} which + * contains the specified object or {@code null} if it is not stored + * in this PackDirectory. + */ + @Nullable + Pack getPack(AnyObjectId objectId) { PackList pList; do { pList = packList.get(); for (Pack p : pList.packs) { try { if (p.hasObject(objectId)) { - return true; + return p; } } catch (IOException e) { - // The hasObject call should have only touched the index, - // so any failure here indicates the index is unreadable - // by this process, and the pack is likewise not readable. + // The hasObject call should have only touched the index, so + // any failure here indicates the index is unreadable by + // this process, and the pack is likewise not readable. LOG.warn(MessageFormat.format( JGitText.get().unableToReadPackfile, p.getPackFile().getAbsolutePath()), e); @@ -143,7 +161,7 @@ class PackDirectory { } } } while (searchPacksAgain(pList)); - return false; + return null; } /** @@ -398,43 +416,29 @@ class PackDirectory { private PackList scanPacksImpl(PackList old) { final Map<String, Pack> forReuse = reuseMap(old); final FileSnapshot snapshot = FileSnapshot.save(directory); - final Set<String> names = listPackDirectory(); - final List<Pack> list = new ArrayList<>(names.size() >> 2); + Map<String, Map<PackExt, PackFile>> packFilesByExtById = getPackFilesByExtById(); + List<Pack> list = new ArrayList<>(packFilesByExtById.size()); boolean foundNew = false; - for (String indexName : names) { - // Must match "pack-[0-9a-f]{40}.idx" to be an index. - // - if (indexName.length() != 49 || !indexName.endsWith(".idx")) { //$NON-NLS-1$ - continue; - } - - final String base = indexName.substring(0, indexName.length() - 3); - int extensions = 0; - for (PackExt ext : PackExt.values()) { - if (names.contains(base + ext.getExtension())) { - extensions |= ext.getBit(); - } - } - - if ((extensions & PACK.getBit()) == 0) { + for (Map<PackExt, PackFile> packFilesByExt : packFilesByExtById + .values()) { + PackFile packFile = packFilesByExt.get(PACK); + if (packFile == null || !packFilesByExt.containsKey(INDEX)) { // Sometimes C Git's HTTP fetch transport leaves a // .idx file behind and does not download the .pack. // We have to skip over such useless indexes. - // + // Also skip if we don't have any index for this id continue; } - final String packName = base + PACK.getExtension(); - final File packFile = new File(directory, packName); - final Pack oldPack = forReuse.get(packName); + Pack oldPack = forReuse.get(packFile.getName()); if (oldPack != null && !oldPack.getFileSnapshot().isModified(packFile)) { - forReuse.remove(packName); + forReuse.remove(packFile.getName()); list.add(oldPack); continue; } - list.add(new Pack(packFile, extensions)); + list.add(new Pack(packFile, packFilesByExt.get(BITMAP_INDEX))); foundNew = true; } @@ -487,18 +491,42 @@ class PackDirectory { return forReuse; } - private Set<String> listPackDirectory() { + /** + * Scans the pack directory for + * {@link org.eclipse.jgit.internal.storage.file.PackFile}s and returns them + * organized by their extensions and their pack ids + * + * Skips files in the directory that we cannot create a + * {@link org.eclipse.jgit.internal.storage.file.PackFile} for. + * + * @return a map of {@link org.eclipse.jgit.internal.storage.file.PackFile}s + * and {@link org.eclipse.jgit.internal.storage.pack.PackExt}s keyed + * by pack ids + */ + private Map<String, Map<PackExt, PackFile>> getPackFilesByExtById() { final String[] nameList = directory.list(); if (nameList == null) { - return Collections.emptySet(); + return Collections.emptyMap(); } - final Set<String> nameSet = new HashSet<>(nameList.length << 1); + Map<String, Map<PackExt, PackFile>> packFilesByExtById = new HashMap<>( + nameList.length / 2); // assume roughly 2 files per id for (String name : nameList) { - if (name.startsWith("pack-")) { //$NON-NLS-1$ - nameSet.add(name); + try { + PackFile pack = new PackFile(directory, name); + if (pack.getPackExt() != null) { + Map<PackExt, PackFile> packByExt = packFilesByExtById + .get(pack.getId()); + if (packByExt == null) { + packByExt = new EnumMap<>(PackExt.class); + packFilesByExtById.put(pack.getId(), packByExt); + } + packByExt.put(pack.getPackExt(), pack); + } + } catch (IllegalArgumentException e) { + continue; } } - return nameSet; + return packFilesByExtById; } static final class PackList { diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java new file mode 100644 index 0000000000..19979d0ed5 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackFile.java @@ -0,0 +1,187 @@ +/* + * Copyright (c) 2021 Qualcomm Innovation Center, Inc. + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.internal.storage.file; + +import java.io.File; +import java.text.MessageFormat; + +import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.internal.storage.pack.PackExt; +import org.eclipse.jgit.lib.ObjectId; + +/** + * A pack file (or pack related) File. + * + * Example: "pack-0123456789012345678901234567890123456789.idx" + */ +public class PackFile extends File { + private static final long serialVersionUID = 1L; + + private static final String PREFIX = "pack-"; //$NON-NLS-1$ + + private final String base; // PREFIX + id i.e. + // pack-0123456789012345678901234567890123456789 + + private final String id; // i.e. 0123456789012345678901234567890123456789 + + private final boolean hasOldPrefix; + + private final PackExt packExt; + + private static String createName(String id, PackExt extension) { + return PREFIX + id + '.' + extension.getExtension(); + } + + /** + * Create a PackFile for a pack or related file. + * + * @param file + * File pointing to the location of the file. + */ + public PackFile(File file) { + this(file.getParentFile(), file.getName()); + } + + /** + * Create a PackFile for a pack or related file. + * + * @param directory + * Directory to create the PackFile in. + * @param id + * the {@link ObjectId} for this pack + * @param ext + * the <code>packExt</code> of the name. + */ + public PackFile(File directory, ObjectId id, PackExt ext) { + this(directory, id.name(), ext); + } + + /** + * Create a PackFile for a pack or related file. + * + * @param directory + * Directory to create the PackFile in. + * @param id + * the <code>id</code> (40 Hex char) section of the pack name. + * @param ext + * the <code>packExt</code> of the name. + */ + public PackFile(File directory, String id, PackExt ext) { + this(directory, createName(id, ext)); + } + + /** + * Create a PackFile for a pack or related file. + * + * @param directory + * Directory to create the PackFile in. + * @param name + * Filename (last path section) of the PackFile + */ + public PackFile(File directory, String name) { + super(directory, name); + int dot = name.lastIndexOf('.'); + + if (dot < 0) { + base = name; + hasOldPrefix = false; + packExt = null; + } else { + base = name.substring(0, dot); + String tail = name.substring(dot + 1); // ["old-"] + extension + packExt = getPackExt(tail); + String old = tail.substring(0, + tail.length() - getExtension().length()); + hasOldPrefix = old.equals(getExtPrefix(true)); + } + + id = base.startsWith(PREFIX) ? base.substring(PREFIX.length()) : base; + } + + /** + * Getter for the field <code>id</code>. + * + * @return the <code>id</code> (40 Hex char) section of the name. + */ + public String getId() { + return id; + } + + /** + * Getter for the field <code>packExt</code>. + * + * @return the <code>packExt</code> of the name. + */ + public PackExt getPackExt() { + return packExt; + } + + /** + * Create a new similar PackFile with the given extension instead. + * + * @param ext + * PackExt the extension to use. + * @return a PackFile instance with specified extension + */ + public PackFile create(PackExt ext) { + return new PackFile(getParentFile(), getName(ext)); + } + + /** + * Create a new similar PackFile in the given directory. + * + * @param directory + * Directory to create the new PackFile in. + * @return a PackFile in the given directory + */ + public PackFile createForDirectory(File directory) { + return new PackFile(directory, getName(false)); + } + + /** + * Create a new similar preserved PackFile in the given directory. + * + * @param directory + * Directory to create the new PackFile in. + * @return a PackFile in the given directory with "old-" prefixing the + * extension + */ + public PackFile createPreservedForDirectory(File directory) { + return new PackFile(directory, getName(true)); + } + + private String getName(PackExt ext) { + return base + '.' + getExtPrefix(hasOldPrefix) + ext.getExtension(); + } + + private String getName(boolean isPreserved) { + return base + '.' + getExtPrefix(isPreserved) + getExtension(); + } + + private String getExtension() { + return packExt == null ? "" : packExt.getExtension(); //$NON-NLS-1$ + } + + private static String getExtPrefix(boolean isPreserved) { + return isPreserved ? "old-" : ""; //$NON-NLS-1$ //$NON-NLS-2$ + } + + private static PackExt getPackExt(String endsWithExtension) { + for (PackExt ext : PackExt.values()) { + if (endsWithExtension.endsWith(ext.getExtension())) { + return ext; + } + } + throw new IllegalArgumentException(MessageFormat.format( + JGitText.get().unrecognizedPackExtension, endsWithExtension)); + } +} diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java index a27a2b00c3..d6209c4a79 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/PackInserter.java @@ -76,6 +76,7 @@ import org.eclipse.jgit.errors.IncorrectObjectTypeException; import org.eclipse.jgit.errors.LargeObjectException; import org.eclipse.jgit.errors.MissingObjectException; import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.lib.AbbreviatedObjectId; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.Constants; @@ -273,16 +274,16 @@ public class PackInserter extends ObjectInserter { } Collections.sort(objectList); - File tmpIdx = idxFor(tmpPack); + File tmpIdx = idxFor(tmpPack); // TODO(nasserg) Use PackFile? writePackIndex(tmpIdx, packHash, objectList); - File realPack = new File(db.getPackDirectory(), - "pack-" + computeName(objectList).name() + ".pack"); //$NON-NLS-1$ //$NON-NLS-2$ + PackFile realPack = new PackFile(db.getPackDirectory(), + computeName(objectList), PackExt.PACK); db.closeAllPackHandles(realPack); tmpPack.setReadOnly(); FileUtils.rename(tmpPack, realPack, ATOMIC_MOVE); - File realIdx = idxFor(realPack); + PackFile realIdx = realPack.create(PackExt.INDEX); tmpIdx.setReadOnly(); try { FileUtils.rename(tmpIdx, realIdx, ATOMIC_MOVE); diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackExt.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackExt.java index bedc6939c8..6fb775da8d 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackExt.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackExt.java @@ -13,66 +13,26 @@ package org.eclipse.jgit.internal.storage.pack; /** * A pack file extension. */ -public class PackExt { - private static volatile PackExt[] VALUES = new PackExt[] {}; - +public enum PackExt { /** A pack file extension. */ - public static final PackExt PACK = newPackExt("pack"); //$NON-NLS-1$ + PACK("pack"), //$NON-NLS-1$ /** A pack index file extension. */ - public static final PackExt INDEX = newPackExt("idx"); //$NON-NLS-1$ + INDEX("idx"), //$NON-NLS-1$ /** A keep pack file extension. */ - public static final PackExt KEEP = newPackExt("keep"); //$NON-NLS-1$ + KEEP("keep"), //$NON-NLS-1$ /** A pack bitmap index file extension. */ - public static final PackExt BITMAP_INDEX = newPackExt("bitmap"); //$NON-NLS-1$ + BITMAP_INDEX("bitmap"), //$NON-NLS-1$ /** A reftable file. */ - public static final PackExt REFTABLE = newPackExt("ref"); //$NON-NLS-1$ - - /** - * Get all of the PackExt values. - * - * @return all of the PackExt values. - */ - public static PackExt[] values() { - return VALUES; - } - - /** - * Returns a PackExt for the file extension and registers it in the values - * array. - * - * @param ext - * the file extension. - * @return the PackExt for the ext - */ - public static synchronized PackExt newPackExt(String ext) { - PackExt[] dst = new PackExt[VALUES.length + 1]; - for (int i = 0; i < VALUES.length; i++) { - PackExt packExt = VALUES[i]; - if (packExt.getExtension().equals(ext)) - return packExt; - dst[i] = packExt; - } - if (VALUES.length >= 32) - throw new IllegalStateException( - "maximum number of pack extensions exceeded"); //$NON-NLS-1$ - - PackExt value = new PackExt(ext, VALUES.length); - dst[VALUES.length] = value; - VALUES = dst; - return value; - } + REFTABLE("ref"); //$NON-NLS-1$ private final String ext; - private final int pos; - - private PackExt(String ext, int pos) { + private PackExt(String ext) { this.ext = ext; - this.pos = pos; } /** @@ -85,12 +45,12 @@ public class PackExt { } /** - * Get the position of the extension in the values array. + * Get the position of the extension in the enum declaration. * - * @return the position of the extension in the values array. + * @return the position of the extension in the enum declaration. */ public int getPosition() { - return pos; + return ordinal(); } /** diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkPushConnection.java b/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkPushConnection.java index f2eac8d24a..03ef852c7f 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkPushConnection.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/transport/WalkPushConnection.java @@ -13,6 +13,7 @@ package org.eclipse.jgit.transport; import static org.eclipse.jgit.transport.WalkRemoteObjectDatabase.ROOT_DIR; import java.io.BufferedOutputStream; +import java.io.File; import java.io.IOException; import java.io.OutputStream; import java.util.ArrayList; @@ -26,6 +27,8 @@ import java.util.TreeMap; import org.eclipse.jgit.errors.TransportException; import org.eclipse.jgit.internal.JGitText; +import org.eclipse.jgit.internal.storage.file.PackFile; +import org.eclipse.jgit.internal.storage.pack.PackExt; import org.eclipse.jgit.internal.storage.pack.PackWriter; import org.eclipse.jgit.lib.AnyObjectId; import org.eclipse.jgit.lib.Constants; @@ -189,9 +192,8 @@ class WalkPushConnection extends BaseConnection implements PushConnection { private void sendpack(final List<RemoteRefUpdate> updates, final ProgressMonitor monitor) throws TransportException { - String pathPack = null; - String pathIdx = null; - + PackFile pack = null; + PackFile idx = null; try (PackWriter writer = new PackWriter(transport.getPackConfig(), local.newObjectReader())) { @@ -217,31 +219,33 @@ class WalkPushConnection extends BaseConnection implements PushConnection { for (String n : dest.getPackNames()) packNames.put(n, n); - final String base = "pack-" + writer.computeName().name(); //$NON-NLS-1$ - final String packName = base + ".pack"; //$NON-NLS-1$ - pathPack = "pack/" + packName; //$NON-NLS-1$ - pathIdx = "pack/" + base + ".idx"; //$NON-NLS-1$ //$NON-NLS-2$ + File packDir = new File("pack"); //$NON-NLS-1$ + pack = new PackFile(packDir, writer.computeName(), + PackExt.PACK); + idx = pack.create(PackExt.INDEX); - if (packNames.remove(packName) != null) { + if (packNames.remove(pack.getName()) != null) { // The remote already contains this pack. We should // remove the index before overwriting to prevent bad // offsets from appearing to clients. // dest.writeInfoPacks(packNames.keySet()); - dest.deleteFile(pathIdx); + dest.deleteFile(idx.getPath()); } // Write the pack file, then the index, as readers look the // other direction (index, then pack file). // - String wt = "Put " + base.substring(0, 12); //$NON-NLS-1$ + String wt = "Put " + pack.getName().substring(0, 12); //$NON-NLS-1$ try (OutputStream os = new BufferedOutputStream( - dest.writeFile(pathPack, monitor, wt + "..pack"))) { //$NON-NLS-1$ + dest.writeFile(pack.getPath(), monitor, + wt + "." + pack.getPackExt().getExtension()))) { //$NON-NLS-1$ writer.writePack(monitor, monitor, os); } try (OutputStream os = new BufferedOutputStream( - dest.writeFile(pathIdx, monitor, wt + "..idx"))) { //$NON-NLS-1$ + dest.writeFile(idx.getPath(), monitor, + wt + "." + idx.getPackExt().getExtension()))) { //$NON-NLS-1$ writer.writeIndex(os); } @@ -250,22 +254,22 @@ class WalkPushConnection extends BaseConnection implements PushConnection { // and discover the most recent objects there. // final ArrayList<String> infoPacks = new ArrayList<>(); - infoPacks.add(packName); + infoPacks.add(pack.getName()); infoPacks.addAll(packNames.keySet()); dest.writeInfoPacks(infoPacks); } catch (IOException err) { - safeDelete(pathIdx); - safeDelete(pathPack); + safeDelete(idx); + safeDelete(pack); throw new TransportException(uri, JGitText.get().cannotStoreObjects, err); } } - private void safeDelete(String path) { + private void safeDelete(File path) { if (path != null) { try { - dest.deleteFile(path); + dest.deleteFile(path.getPath()); } catch (IOException cleanupFailure) { // Ignore the deletion failure. We probably are // already failing and were just trying to pick |