* stable-5.1: Prepare 5.1.8-SNAPSHOT builds JGit v5.1.7.201904200442-r ObjectUploadListener: Add callback interface Prepare 4.11.9-SNAPSHOT builds JGit v4.11.8.201904181247-r Prepare 4.9.11-SNAPSHOT builds JGit v4.9.10.201904181027-r Prepare 4.7.10-SNAPSHOT builds JGit v4.7.9.201904161809-r Prepare 4.5.8-SNAPSHOT builds JGit v4.5.7.201904151645-r Remember the cause for invalidating a packfile Fix API problem filters Fix pack files scan when filesnapshot isn't modified Change-Id: I76761002eedf360e93d0559942ebc927a40428d6 Signed-off-by: David Pursehouse <david.pursehouse@gmail.com> Signed-off-by: Matthias Sohn <matthias.sohn@sap.com>tags/v5.3.1.201904271842-r
<?xml version="1.0" encoding="UTF-8" standalone="no"?> | |||||
<component id="org.eclipse.jgit.lfs.server" version="2"> | |||||
<resource path="META-INF/MANIFEST.MF"> | |||||
<filter id="924844039"> | |||||
<message_arguments> | |||||
<message_argument value="5.2.2"/> | |||||
<message_argument value="5.2.0"/> | |||||
</message_arguments> | |||||
</filter> | |||||
</resource> | |||||
<resource path="src/org/eclipse/jgit/lfs/server/fs/ObjectUploadListener.java" type="org.eclipse.jgit.lfs.server.fs.ObjectUploadListener"> | |||||
<filter id="1142947843"> | |||||
<message_arguments> | |||||
<message_argument value="5.1.7"/> | |||||
<message_argument value="setCallback(ObjectUploadListener.Callback)"/> | |||||
</message_arguments> | |||||
</filter> | |||||
</resource> | |||||
<resource path="src/org/eclipse/jgit/lfs/server/fs/ObjectUploadListener.java" type="org.eclipse.jgit.lfs.server.fs.ObjectUploadListener$Callback"> | |||||
<filter id="1142947843"> | |||||
<message_arguments> | |||||
<message_argument value="5.1.7"/> | |||||
<message_argument value="Callback"/> | |||||
</message_arguments> | |||||
</filter> | |||||
</resource> | |||||
</component> |
import java.nio.channels.Channels; | import java.nio.channels.Channels; | ||||
import java.nio.channels.ReadableByteChannel; | import java.nio.channels.ReadableByteChannel; | ||||
import java.nio.channels.WritableByteChannel; | import java.nio.channels.WritableByteChannel; | ||||
import java.nio.file.Path; | |||||
import java.util.logging.Level; | import java.util.logging.Level; | ||||
import java.util.logging.Logger; | import java.util.logging.Logger; | ||||
private final ByteBuffer buffer = ByteBuffer.allocateDirect(8192); | private final ByteBuffer buffer = ByteBuffer.allocateDirect(8192); | ||||
private final Path path; | |||||
private long uploaded; | |||||
private Callback callback; | |||||
/** | |||||
* Callback invoked after object upload completed. | |||||
* | |||||
* @since 5.1.7 | |||||
*/ | |||||
public interface Callback { | |||||
/** | |||||
* Notified after object upload completed. | |||||
* | |||||
* @param path | |||||
* path to the object on the backend | |||||
* @param size | |||||
* uploaded size in bytes | |||||
*/ | |||||
void uploadCompleted(String path, long size); | |||||
} | |||||
/** | /** | ||||
* Constructor for ObjectUploadListener. | * Constructor for ObjectUploadListener. | ||||
* | * | ||||
this.inChannel = Channels.newChannel(in); | this.inChannel = Channels.newChannel(in); | ||||
this.out = repository.getOutputStream(id); | this.out = repository.getOutputStream(id); | ||||
this.channel = Channels.newChannel(out); | this.channel = Channels.newChannel(out); | ||||
this.path = repository.getPath(id); | |||||
this.uploaded = 0L; | |||||
response.setContentType(Constants.CONTENT_TYPE_GIT_LFS_JSON); | response.setContentType(Constants.CONTENT_TYPE_GIT_LFS_JSON); | ||||
} | } | ||||
/** | |||||
* Set the callback to invoke after upload completed. | |||||
* | |||||
* @param callback | |||||
* the callback | |||||
* @return {@code this}. | |||||
* @since 5.1.7 | |||||
*/ | |||||
public ObjectUploadListener setCallback(Callback callback) { | |||||
this.callback = callback; | |||||
return this; | |||||
} | |||||
/** | /** | ||||
* {@inheritDoc} | * {@inheritDoc} | ||||
* | * | ||||
while (in.isReady()) { | while (in.isReady()) { | ||||
if (inChannel.read(buffer) > 0) { | if (inChannel.read(buffer) > 0) { | ||||
buffer.flip(); | buffer.flip(); | ||||
channel.write(buffer); | |||||
uploaded += Integer.valueOf(channel.write(buffer)).longValue(); | |||||
buffer.compact(); | buffer.compact(); | ||||
} else { | } else { | ||||
buffer.flip(); | buffer.flip(); | ||||
while (buffer.hasRemaining()) { | while (buffer.hasRemaining()) { | ||||
channel.write(buffer); | |||||
uploaded += Integer.valueOf(channel.write(buffer)) | |||||
.longValue(); | |||||
} | } | ||||
close(); | close(); | ||||
return; | return; | ||||
if (!response.isCommitted()) { | if (!response.isCommitted()) { | ||||
response.setStatus(HttpServletResponse.SC_OK); | response.setStatus(HttpServletResponse.SC_OK); | ||||
} | } | ||||
if (callback != null) { | |||||
callback.uploadCompleted(path.toString(), uploaded); | |||||
} | |||||
} finally { | } finally { | ||||
context.complete(); | context.complete(); | ||||
} | } |
</message_arguments> | </message_arguments> | ||||
</filter> | </filter> | ||||
</resource> | </resource> | ||||
<resource path="src/org/eclipse/jgit/errors/PackInvalidException.java" type="org.eclipse.jgit.errors.PackInvalidException"> | |||||
<filter id="1142947843"> | |||||
<message_arguments> | |||||
<message_argument value="4.5.7"/> | |||||
<message_argument value="PackInvalidException(File, Throwable)"/> | |||||
</message_arguments> | |||||
</filter> | |||||
<filter id="1142947843"> | |||||
<message_arguments> | |||||
<message_argument value="4.5.7"/> | |||||
<message_argument value="PackInvalidException(String, Throwable)"/> | |||||
</message_arguments> | |||||
</filter> | |||||
</resource> | |||||
<resource path="src/org/eclipse/jgit/transport/TransferConfig.java" type="org.eclipse.jgit.transport.TransferConfig"> | <resource path="src/org/eclipse/jgit/transport/TransferConfig.java" type="org.eclipse.jgit.transport.TransferConfig"> | ||||
<filter id="1159725059"> | <filter id="1159725059"> | ||||
<message_arguments> | <message_arguments> |
* | * | ||||
* @param path | * @param path | ||||
* path of the invalid pack file. | * path of the invalid pack file. | ||||
* @deprecated Use {@link #PackInvalidException(File, Throwable)}. | |||||
*/ | */ | ||||
@Deprecated | |||||
public PackInvalidException(File path) { | public PackInvalidException(File path) { | ||||
this(path.getAbsolutePath()); | |||||
this(path, null); | |||||
} | |||||
/** | |||||
* Construct a pack invalid error with cause. | |||||
* | |||||
* @param path | |||||
* path of the invalid pack file. | |||||
* @param cause | |||||
* cause of the pack file becoming invalid. | |||||
* @since 4.5.7 | |||||
*/ | |||||
public PackInvalidException(File path, Throwable cause) { | |||||
this(path.getAbsolutePath(), cause); | |||||
} | } | ||||
/** | /** | ||||
* | * | ||||
* @param path | * @param path | ||||
* path of the invalid pack file. | * path of the invalid pack file. | ||||
* @deprecated Use {@link #PackInvalidException(String, Throwable)}. | |||||
*/ | */ | ||||
@Deprecated | |||||
public PackInvalidException(String path) { | public PackInvalidException(String path) { | ||||
super(MessageFormat.format(JGitText.get().packFileInvalid, path)); | |||||
this(path, null); | |||||
} | |||||
/** | |||||
* Construct a pack invalid error with cause. | |||||
* | |||||
* @param path | |||||
* path of the invalid pack file. | |||||
* @param cause | |||||
* cause of the pack file becoming invalid. | |||||
* @since 4.5.7 | |||||
*/ | |||||
public PackInvalidException(String path, Throwable cause) { | |||||
super(MessageFormat.format(JGitText.get().packFileInvalid, path), cause); | |||||
} | } | ||||
} | } |
/** True once corruption has been detected that cannot be worked around. */ | /** True once corruption has been detected that cannot be worked around. */ | ||||
volatile boolean invalid; | volatile boolean invalid; | ||||
/** Exception that caused the packfile to be flagged as invalid */ | |||||
protected volatile Exception invalidatingCause; | |||||
BlockBasedFile(DfsBlockCache cache, DfsPackDescription desc, PackExt ext) { | BlockBasedFile(DfsBlockCache cache, DfsPackDescription desc, PackExt ext) { | ||||
this.cache = cache; | this.cache = cache; | ||||
this.key = desc.getStreamKey(ext); | this.key = desc.getStreamKey(ext); | ||||
DfsBlock readOneBlock(long pos, DfsReader ctx, | DfsBlock readOneBlock(long pos, DfsReader ctx, | ||||
@Nullable ReadableChannel fileChannel) throws IOException { | @Nullable ReadableChannel fileChannel) throws IOException { | ||||
if (invalid) | |||||
throw new PackInvalidException(getFileName()); | |||||
if (invalid) { | |||||
throw new PackInvalidException(getFileName(), invalidatingCause); | |||||
} | |||||
ctx.stats.readBlock++; | ctx.stats.readBlock++; | ||||
long start = System.nanoTime(); | long start = System.nanoTime(); |
return idx; | return idx; | ||||
} | } | ||||
if (invalid) | |||||
throw new PackInvalidException(getFileName()); | |||||
if (invalid) { | |||||
throw new PackInvalidException(getFileName(), invalidatingCause); | |||||
} | |||||
Repository.getGlobalListenerList() | Repository.getGlobalListenerList() | ||||
.dispatch(new BeforeDfsPackIndexLoadedEvent(this)); | .dispatch(new BeforeDfsPackIndexLoadedEvent(this)); | ||||
} | } | ||||
} catch (EOFException e) { | } catch (EOFException e) { | ||||
invalid = true; | invalid = true; | ||||
invalidatingCause = e; | |||||
throw new IOException(MessageFormat.format( | throw new IOException(MessageFormat.format( | ||||
DfsText.get().shortReadOfIndex, | DfsText.get().shortReadOfIndex, | ||||
desc.getFileName(INDEX)), e); | desc.getFileName(INDEX)), e); | ||||
} catch (IOException e) { | } catch (IOException e) { | ||||
invalid = true; | invalid = true; | ||||
invalidatingCause = e; | |||||
throw new IOException(MessageFormat.format( | throw new IOException(MessageFormat.format( | ||||
DfsText.get().cannotReadIndex, | DfsText.get().cannotReadIndex, | ||||
desc.getFileName(INDEX)), e); | desc.getFileName(INDEX)), e); | ||||
private IOException packfileIsTruncated() { | private IOException packfileIsTruncated() { | ||||
invalid = true; | invalid = true; | ||||
return new IOException(MessageFormat.format( | |||||
IOException exc = new IOException(MessageFormat.format( | |||||
JGitText.get().packfileIsTruncated, getFileName())); | JGitText.get().packfileIsTruncated, getFileName())); | ||||
invalidatingCause = exc; | |||||
return exc; | |||||
} | } | ||||
private void readFully(long position, byte[] dstbuf, int dstoff, int cnt, | private void readFully(long position, byte[] dstbuf, int dstoff, int cnt, |
private final File alternatesFile; | private final File alternatesFile; | ||||
private final AtomicReference<PackList> packList; | |||||
private final FS fs; | private final FS fs; | ||||
private final AtomicReference<AlternateHandle[]> alternates; | private final AtomicReference<AlternateHandle[]> alternates; | ||||
private Set<ObjectId> shallowCommitsIds; | private Set<ObjectId> shallowCommitsIds; | ||||
final AtomicReference<PackList> packList; | |||||
/** | /** | ||||
* Initialize a reference to an on-disk object directory. | * Initialize a reference to an on-disk object directory. | ||||
* | * | ||||
transientErrorCount = p.incrementTransientErrorCount(); | transientErrorCount = p.incrementTransientErrorCount(); | ||||
} | } | ||||
if (warnTmpl != null) { | if (warnTmpl != null) { | ||||
if (LOG.isDebugEnabled()) { | |||||
LOG.debug(MessageFormat.format(warnTmpl, | |||||
p.getPackFile().getAbsolutePath()), e); | |||||
} else { | |||||
LOG.warn(MessageFormat.format(warnTmpl, | |||||
p.getPackFile().getAbsolutePath())); | |||||
} | |||||
LOG.warn(MessageFormat.format(warnTmpl, | |||||
p.getPackFile().getAbsolutePath()), e); | |||||
} else { | } else { | ||||
if (doLogExponentialBackoff(transientErrorCount)) { | if (doLogExponentialBackoff(transientErrorCount)) { | ||||
// Don't remove the pack from the list, as the error may be | // Don't remove the pack from the list, as the error may be | ||||
return InsertLooseObjectResult.FAILURE; | return InsertLooseObjectResult.FAILURE; | ||||
} | } | ||||
private boolean searchPacksAgain(PackList old) { | |||||
boolean searchPacksAgain(PackList old) { | |||||
// Whether to trust the pack folder's modification time. If set | // Whether to trust the pack folder's modification time. If set | ||||
// to false we will always scan the .git/objects/pack folder to | // to false we will always scan the .git/objects/pack folder to | ||||
// check for new pack files. If set to true (default) we use the | // check for new pack files. If set to true (default) we use the | ||||
final String packName = base + PACK.getExtension(); | final String packName = base + PACK.getExtension(); | ||||
final File packFile = new File(packDirectory, packName); | final File packFile = new File(packDirectory, packName); | ||||
final PackFile oldPack = forReuse.remove(packName); | final PackFile oldPack = forReuse.remove(packName); | ||||
if (oldPack != null && oldPack.getFileSnapshot().isModified(packFile)) { | |||||
if (oldPack != null | |||||
&& !oldPack.getFileSnapshot().isModified(packFile)) { | |||||
list.add(oldPack); | list.add(oldPack); | ||||
continue; | continue; | ||||
} | } | ||||
return new File(new File(getDirectory(), d), f); | return new File(new File(getDirectory(), d), f); | ||||
} | } | ||||
private static final class PackList { | |||||
static final class PackList { | |||||
/** State just before reading the pack directory. */ | /** State just before reading the pack directory. */ | ||||
final FileSnapshot snapshot; | final FileSnapshot snapshot; | ||||
private volatile boolean invalid; | private volatile boolean invalid; | ||||
private volatile Exception invalidatingCause; | |||||
private boolean invalidBitmap; | private boolean invalidBitmap; | ||||
private AtomicInteger transientErrorCount = new AtomicInteger(); | private AtomicInteger transientErrorCount = new AtomicInteger(); | ||||
idx = loadedIdx; | idx = loadedIdx; | ||||
if (idx == null) { | if (idx == null) { | ||||
if (invalid) { | if (invalid) { | ||||
throw new PackInvalidException(packFile); | |||||
throw new PackInvalidException(packFile, invalidatingCause); | |||||
} | } | ||||
try { | try { | ||||
idx = PackIndex.open(extFile(INDEX)); | idx = PackIndex.open(extFile(INDEX)); | ||||
throw e; | throw e; | ||||
} catch (IOException e) { | } catch (IOException e) { | ||||
invalid = true; | invalid = true; | ||||
invalidatingCause = e; | |||||
throw e; | throw e; | ||||
} | } | ||||
} | } | ||||
private void doOpen() throws IOException { | private void doOpen() throws IOException { | ||||
if (invalid) { | if (invalid) { | ||||
throw new PackInvalidException(packFile); | |||||
throw new PackInvalidException(packFile, invalidatingCause); | |||||
} | } | ||||
try { | try { | ||||
synchronized (readLock) { | synchronized (readLock) { | ||||
} | } | ||||
} catch (InterruptedIOException e) { | } catch (InterruptedIOException e) { | ||||
// don't invalidate the pack, we are interrupted from another thread | // don't invalidate the pack, we are interrupted from another thread | ||||
openFail(false); | |||||
openFail(false, e); | |||||
throw e; | throw e; | ||||
} catch (FileNotFoundException fn) { | } catch (FileNotFoundException fn) { | ||||
// don't invalidate the pack if opening an existing file failed | // don't invalidate the pack if opening an existing file failed | ||||
// since it may be related to a temporary lack of resources (e.g. | // since it may be related to a temporary lack of resources (e.g. | ||||
// max open files) | // max open files) | ||||
openFail(!packFile.exists()); | |||||
openFail(!packFile.exists(), fn); | |||||
throw fn; | throw fn; | ||||
} catch (EOFException | AccessDeniedException | NoSuchFileException | } catch (EOFException | AccessDeniedException | NoSuchFileException | ||||
| CorruptObjectException | NoPackSignatureException | | CorruptObjectException | NoPackSignatureException | ||||
| UnsupportedPackIndexVersionException | | UnsupportedPackIndexVersionException | ||||
| UnsupportedPackVersionException pe) { | | UnsupportedPackVersionException pe) { | ||||
// exceptions signaling permanent problems with a pack | // exceptions signaling permanent problems with a pack | ||||
openFail(true); | |||||
openFail(true, pe); | |||||
throw pe; | throw pe; | ||||
} catch (IOException | RuntimeException ge) { | } catch (IOException | RuntimeException ge) { | ||||
// generic exceptions could be transient so we should not mark the | // generic exceptions could be transient so we should not mark the | ||||
// pack invalid to avoid false MissingObjectExceptions | // pack invalid to avoid false MissingObjectExceptions | ||||
openFail(false); | |||||
openFail(false, ge); | |||||
throw ge; | throw ge; | ||||
} | } | ||||
} | } | ||||
private void openFail(boolean invalidate) { | |||||
private void openFail(boolean invalidate, Exception cause) { | |||||
activeWindows = 0; | activeWindows = 0; | ||||
activeCopyRawData = 0; | activeCopyRawData = 0; | ||||
invalid = invalidate; | invalid = invalidate; | ||||
invalidatingCause = cause; | |||||
doClose(); | doClose(); | ||||
} | } | ||||
// Detect the situation and throw a proper exception so that can be properly | // Detect the situation and throw a proper exception so that can be properly | ||||
// managed by the main packfile search loop and the Git client won't receive | // managed by the main packfile search loop and the Git client won't receive | ||||
// any failures. | // any failures. | ||||
throw new PackInvalidException(packFile); | |||||
throw new PackInvalidException(packFile, invalidatingCause); | |||||
} | } | ||||
if (length < pos + size) | if (length < pos + size) | ||||
size = (int) (length - pos); | size = (int) (length - pos); |