Explorar el Código

Disable CRC32 computation when no PackIndex will be created

If a server is streaming 3GiB worth of pack data to a client there
is no reason to compute the CRC32 checksum on the objects. The
CRC32 code computed by PackWriter is used only in the new index
created by writeIndex(), which is never invoked for the native Git
network protocols.

Object reuse may still compute its own CRC32 to verify the data
being copied from an existing pack has not been corrupted. This
check is done by the ObjectReader that implements ObjectReuseAsIs
and has no relationship to the CRC32 being skipped during output.

Change-Id: I05626f2e0d6ce19119b57d8a27193922636d60a7
tags/v3.0.0.201305080800-m7
Shawn Pearce hace 11 años
padre
commit
eb17495ca4

+ 0
- 15
org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackOutputStream.java Ver fichero

import java.io.IOException; import java.io.IOException;
import java.io.OutputStream; import java.io.OutputStream;
import java.security.MessageDigest; import java.security.MessageDigest;
import java.util.zip.CRC32;


import org.eclipse.jgit.internal.JGitText; import org.eclipse.jgit.internal.JGitText;
import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.Constants;


private final PackWriter packWriter; private final PackWriter packWriter;


private final CRC32 crc = new CRC32();

private final MessageDigest md = Constants.newMessageDigest(); private final MessageDigest md = Constants.newMessageDigest();


private long count; private long count;
public void write(final int b) throws IOException { public void write(final int b) throws IOException {
count++; count++;
out.write(b); out.write(b);
crc.update(b);
md.update((byte) b); md.update((byte) b);
} }


} }


out.write(b, off, n); out.write(b, off, n);
crc.update(b, off, n);
md.update(b, off, n); md.update(b, off, n);


off += n; off += n;
return count; return count;
} }


/** @return obtain the current CRC32 register. */
int getCRC32() {
return (int) crc.getValue();
}

/** Reinitialize the CRC32 register for a new region. */
void resetCRC32() {
crc.reset();
}

/** @return obtain the current SHA-1 digest. */ /** @return obtain the current SHA-1 digest. */
byte[] getDigest() { byte[] getDigest() {
return md.digest(); return md.digest();

+ 32
- 8
org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/pack/PackWriter.java Ver fichero

import java.util.concurrent.Executors; import java.util.concurrent.Executors;
import java.util.concurrent.Future; import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
import java.util.zip.CRC32;
import java.util.zip.CheckedOutputStream;
import java.util.zip.Deflater; import java.util.zip.Deflater;
import java.util.zip.DeflaterOutputStream; import java.util.zip.DeflaterOutputStream;




private boolean canBuildBitmaps; private boolean canBuildBitmaps;


private boolean indexDisabled;

private int depth; private int depth;


private Collection<? extends ObjectId> unshallowObjects; private Collection<? extends ObjectId> unshallowObjects;


private PackBitmapIndexBuilder writeBitmaps; private PackBitmapIndexBuilder writeBitmaps;


private CRC32 crc32;

/** /**
* Create writer for specified repository. * Create writer for specified repository.
* <p> * <p>
this.useBitmaps = useBitmaps; this.useBitmaps = useBitmaps;
} }


/** @return true if the index file cannot be created by this PackWriter. */
public boolean isIndexDisabled() {
return indexDisabled || !cachedPacks.isEmpty();
}

/**
* @param noIndex
* true to disable creation of the index file.
*/
public void setIndexDisabled(boolean noIndex) {
this.indexDisabled = noIndex;
}

/** /**
* @return true to ignore objects that are uninteresting and also not found * @return true to ignore objects that are uninteresting and also not found
* on local disk; false to throw a {@link MissingObjectException} * on local disk; false to throw a {@link MissingObjectException}
* the index data could not be written to the supplied stream. * the index data could not be written to the supplied stream.
*/ */
public void writeIndex(final OutputStream indexStream) throws IOException { public void writeIndex(final OutputStream indexStream) throws IOException {
if (!cachedPacks.isEmpty())
if (isIndexDisabled())
throw new IOException(JGitText.get().cachedPacksPreventsIndexCreation); throw new IOException(JGitText.get().cachedPacksPreventsIndexCreation);


long writeStart = System.currentTimeMillis(); long writeStart = System.currentTimeMillis();
if (config.isDeltaCompress()) if (config.isDeltaCompress())
searchForDeltas(compressMonitor); searchForDeltas(compressMonitor);


final PackOutputStream out = new PackOutputStream(writeMonitor,
packStream, this);
crc32 = new CRC32();
final PackOutputStream out = new PackOutputStream(
writeMonitor,
isIndexDisabled()
? packStream
: new CheckedOutputStream(packStream, crc32),
this);


long objCnt = getObjectCount(); long objCnt = getObjectCount();
stats.totalObjects = objCnt; stats.totalObjects = objCnt;
if (otp.isWritten()) if (otp.isWritten())
return; // Delta chain cycle caused this to write already. return; // Delta chain cycle caused this to write already.


out.resetCRC32();
crc32.reset();
otp.setOffset(out.length()); otp.setOffset(out.length());
try { try {
reuseSupport.copyObjectAsIs(out, otp, reuseValidate); reuseSupport.copyObjectAsIs(out, otp, reuseValidate);
out.endObject(); out.endObject();
otp.setCRC(out.getCRC32());
otp.setCRC((int) crc32.getValue());
typeStats.reusedObjects++; typeStats.reusedObjects++;
if (otp.isDeltaRepresentation()) { if (otp.isDeltaRepresentation()) {
typeStats.reusedDeltas++; typeStats.reusedDeltas++;
else else
writeWholeObjectDeflate(out, otp); writeWholeObjectDeflate(out, otp);
out.endObject(); out.endObject();
otp.setCRC(out.getCRC32());
otp.setCRC((int) crc32.getValue());
} }


private void writeBase(PackOutputStream out, ObjectToPack base) private void writeBase(PackOutputStream out, ObjectToPack base)
final Deflater deflater = deflater(); final Deflater deflater = deflater();
final ObjectLoader ldr = reader.open(otp, otp.getType()); final ObjectLoader ldr = reader.open(otp, otp.getType());


out.resetCRC32();
crc32.reset();
otp.setOffset(out.length()); otp.setOffset(out.length());
out.writeHeader(otp, ldr.getSize()); out.writeHeader(otp, ldr.getSize());


final ObjectToPack otp) throws IOException { final ObjectToPack otp) throws IOException {
writeBase(out, otp.getDeltaBase()); writeBase(out, otp.getDeltaBase());


out.resetCRC32();
crc32.reset();
otp.setOffset(out.length()); otp.setOffset(out.length());


DeltaCache.Ref ref = otp.popCachedDelta(); DeltaCache.Ref ref = otp.popCachedDelta();

+ 1
- 0
org.eclipse.jgit/src/org/eclipse/jgit/transport/BasePackPushConnection.java Ver fichero

newObjects.add(r.getNewObjectId()); newObjects.add(r.getNewObjectId());
} }


writer.setIndexDisabled(true);
writer.setUseCachedPacks(true); writer.setUseCachedPacks(true);
writer.setUseBitmaps(true); writer.setUseBitmaps(true);
writer.setThin(thinPack); writer.setThin(thinPack);

+ 1
- 0
org.eclipse.jgit/src/org/eclipse/jgit/transport/BundleWriter.java Ver fichero

inc.addAll(include.values()); inc.addAll(include.values());
for (final RevCommit r : assume) for (final RevCommit r : assume)
exc.add(r.getId()); exc.add(r.getId());
packWriter.setIndexDisabled(true);
packWriter.setDeltaBaseAsOffset(true); packWriter.setDeltaBaseAsOffset(true);
packWriter.setThin(exc.size() > 0); packWriter.setThin(exc.size() > 0);
packWriter.setReuseValidatingObjects(false); packWriter.setReuseValidatingObjects(false);

+ 1
- 0
org.eclipse.jgit/src/org/eclipse/jgit/transport/UploadPack.java Ver fichero

cfg = new PackConfig(db); cfg = new PackConfig(db);
final PackWriter pw = new PackWriter(cfg, walk.getObjectReader()); final PackWriter pw = new PackWriter(cfg, walk.getObjectReader());
try { try {
pw.setIndexDisabled(true);
pw.setUseCachedPacks(true); pw.setUseCachedPacks(true);
pw.setUseBitmaps(true); pw.setUseBitmaps(true);
pw.setReuseDeltaCommits(true); pw.setReuseDeltaCommits(true);

Cargando…
Cancelar
Guardar