diff options
author | Markus Duft <markus.duft@ssi-schaefer.com> | 2018-03-02 10:11:42 +0100 |
---|---|---|
committer | Matthias Sohn <matthias.sohn@sap.com> | 2018-03-03 11:39:43 +0100 |
commit | d3ed64bcd467e3e8976b018095e71ed3e3033eae (patch) | |
tree | 8593dfe6131b1d3f890ebc92b3a78dead4c15e4d /org.eclipse.jgit.lfs | |
parent | 169de08a789b6fc1eb25350e49f4904e11f732cf (diff) | |
download | jgit-d3ed64bcd467e3e8976b018095e71ed3e3033eae.tar.gz jgit-d3ed64bcd467e3e8976b018095e71ed3e3033eae.zip |
LFS: support merge/rebase/cherry-pick/diff/compare with LFS files
Respect merge=lfs and diff=lfs attributes where required to replace (in
memory) the content of LFS pointers with the actual blob content from
the LFS storage (and vice versa when staging/merging).
Does not implement general support for merge/diff attributes for any
other use case apart from LFS.
Change-Id: Ibad8875de1e0bee8fe3a1dffb1add93111534cae
Signed-off-by: Markus Duft <markus.duft@ssi-schaefer.com>
Signed-off-by: Matthias Sohn <matthias.sohn@sap.com>
Diffstat (limited to 'org.eclipse.jgit.lfs')
8 files changed, 417 insertions, 11 deletions
diff --git a/org.eclipse.jgit.lfs/.settings/.api_filters b/org.eclipse.jgit.lfs/.settings/.api_filters index 097fd20658..f4887e272b 100644 --- a/org.eclipse.jgit.lfs/.settings/.api_filters +++ b/org.eclipse.jgit.lfs/.settings/.api_filters @@ -1,5 +1,13 @@ <?xml version="1.0" encoding="UTF-8" standalone="no"?> <component id="org.eclipse.jgit.lfs" version="2"> + <resource path="src/org/eclipse/jgit/lfs/CleanFilter.java" type="org.eclipse.jgit.lfs.CleanFilter"> + <filter id="421572723"> + <message_arguments> + <message_argument value="org.eclipse.jgit.lfs.CleanFilter"/> + <message_argument value="register()"/> + </message_arguments> + </filter> + </resource> <resource path="src/org/eclipse/jgit/lfs/LfsPointer.java" type="org.eclipse.jgit.lfs.LfsPointer"> <filter id="336658481"> <message_arguments> @@ -8,4 +16,12 @@ </message_arguments> </filter> </resource> + <resource path="src/org/eclipse/jgit/lfs/SmudgeFilter.java" type="org.eclipse.jgit.lfs.SmudgeFilter"> + <filter id="421572723"> + <message_arguments> + <message_argument value="org.eclipse.jgit.lfs.SmudgeFilter"/> + <message_argument value="register()"/> + </message_arguments> + </filter> + </resource> </component> diff --git a/org.eclipse.jgit.lfs/META-INF/MANIFEST.MF b/org.eclipse.jgit.lfs/META-INF/MANIFEST.MF index 798fccdfc4..6d94877148 100644 --- a/org.eclipse.jgit.lfs/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit.lfs/META-INF/MANIFEST.MF @@ -18,6 +18,7 @@ Import-Package: com.google.gson;version="[2.8.2,3.0.0)", org.eclipse.jgit.annotations;version="[4.11.0,4.12.0)";resolution:=optional, org.eclipse.jgit.api.errors;version="[4.11.0,4.12.0)", org.eclipse.jgit.attributes;version="[4.11.0,4.12.0)", + org.eclipse.jgit.diff;version="[4.11.0,4.12.0)", org.eclipse.jgit.errors;version="[4.11.0,4.12.0)", org.eclipse.jgit.hooks;version="[4.11.0,4.12.0)", org.eclipse.jgit.internal.storage.file;version="[4.11.0,4.12.0)", @@ -25,6 +26,7 @@ Import-Package: com.google.gson;version="[2.8.2,3.0.0)", org.eclipse.jgit.nls;version="[4.11.0,4.12.0)", org.eclipse.jgit.revwalk;version="[4.11.0,4.12.0)", org.eclipse.jgit.storage.file;version="[4.11.0,4.12.0)", + org.eclipse.jgit.storage.pack;version="[4.11.0,4.12.0)", org.eclipse.jgit.transport;version="[4.11.0,4.12.0)", org.eclipse.jgit.transport.http;version="[4.11.0,4.12.0)", org.eclipse.jgit.treewalk;version="[4.11.0,4.12.0)", diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/BuiltinLFS.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/BuiltinLFS.java new file mode 100644 index 0000000000..e1b9e34ed5 --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/BuiltinLFS.java @@ -0,0 +1,140 @@ +/* + * Copyright (C) 2017, Markus Duft <markus.duft@ssi-schaefer.com> + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.eclipse.jgit.lfs; + +import java.io.IOException; +import java.io.InputStream; +import java.io.PrintStream; + +import org.eclipse.jgit.annotations.Nullable; +import org.eclipse.jgit.attributes.Attribute; +import org.eclipse.jgit.hooks.PrePushHook; +import org.eclipse.jgit.lfs.lib.Constants; +import org.eclipse.jgit.lib.ConfigConstants; +import org.eclipse.jgit.lib.ObjectLoader; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.util.LfsFactory; + +/** + * Implementation of {@link LfsFactory}, using built-in (optional) LFS support. + * + * @since 4.11 + */ +public class BuiltinLFS extends LfsFactory { + + private BuiltinLFS() { + SmudgeFilter.register(); + CleanFilter.register(); + } + + /** + * Activates the built-in LFS support. + */ + public static void register() { + setInstance(new BuiltinLFS()); + } + + @Override + public boolean isAvailable() { + return true; + } + + @Override + public ObjectLoader applySmudgeFilter(Repository db, ObjectLoader loader, + Attribute attribute) throws IOException { + if (isEnabled(db) && (attribute == null || isEnabled(db, attribute))) { + return LfsBlobFilter.smudgeLfsBlob(db, loader); + } else { + return loader; + } + } + + @Override + public LfsInputStream applyCleanFilter(Repository db, InputStream input, + long length, Attribute attribute) throws IOException { + if (isEnabled(db, attribute)) { + return new LfsInputStream(LfsBlobFilter.cleanLfsBlob(db, input)); + } else { + return new LfsInputStream(input, length); + } + } + + @Override + public @Nullable PrePushHook getPrePushHook(Repository repo, + PrintStream outputStream) { + if (isEnabled(repo)) { + return new LfsPrePushHook(repo, outputStream); + } + return null; + } + + /** + * @param db + * the repository + * @return whether LFS is requested for the given repo. + */ + private boolean isEnabled(Repository db) { + if (db == null) { + return false; + } + return db.getConfig().getBoolean(ConfigConstants.CONFIG_FILTER_SECTION, + Constants.LFS, ConfigConstants.CONFIG_KEY_USEJGITBUILTIN, + false); + } + + /** + * @param db + * the repository + * @param attribute + * the attribute to check + * @return whether LFS filter is enabled for the given .gitattribute + * attribute. + */ + private boolean isEnabled(Repository db, Attribute attribute) { + if (attribute == null) { + return false; + } + return isEnabled(db) && Constants.LFS.equals(attribute.getValue()); + } + +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/CleanFilter.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/CleanFilter.java index 3e6f9961a8..fccbae7955 100644 --- a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/CleanFilter.java +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/CleanFilter.java @@ -91,7 +91,7 @@ public class CleanFilter extends FilterCommand { * Registers this filter by calling * {@link FilterCommandRegistry#register(String, FilterCommandFactory)} */ - public final static void register() { + static void register() { FilterCommandRegistry .register(org.eclipse.jgit.lib.Constants.BUILTIN_FILTER_PREFIX + Constants.ATTR_FILTER_DRIVER_PREFIX diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsBlobFilter.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsBlobFilter.java new file mode 100644 index 0000000000..a7d218f803 --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsBlobFilter.java @@ -0,0 +1,130 @@ +/* + * Copyright (C) 2017, Markus Duft <markus.duft@ssi-schaefer.com> + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.eclipse.jgit.lfs; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.eclipse.jgit.lfs.lib.AnyLongObjectId; +import org.eclipse.jgit.lib.ObjectLoader; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.util.TemporaryBuffer; +import org.eclipse.jgit.util.TemporaryBuffer.LocalFile; + +/** + * Provides transparently either a stream to the blob or a LFS media file if + * managed by LFS. + * + * @since 4.11 + */ +public class LfsBlobFilter { + + /** + * In case the given {@link ObjectLoader} points to a LFS pointer file + * replace the loader with one pointing to the LFS media file contents. + * Missing LFS files are downloaded on the fly - same logic as the smudge + * filter. + * + * @param db + * the repo + * @param loader + * the loader for the blob + * @return either the original loader, or a loader for the LFS media file if + * managed by LFS. Files are downloaded on demand if required. + * @throws IOException + * in case of an error + */ + public static ObjectLoader smudgeLfsBlob(Repository db, ObjectLoader loader) + throws IOException { + if (loader.getSize() > LfsPointer.SIZE_THRESHOLD) { + return loader; + } + + try (InputStream is = loader.openStream()) { + LfsPointer ptr = LfsPointer.parseLfsPointer(is); + if (ptr != null) { + Lfs lfs = new Lfs(db); + AnyLongObjectId oid = ptr.getOid(); + Path mediaFile = lfs.getMediaFile(oid); + if (!Files.exists(mediaFile)) { + SmudgeFilter.downloadLfsResource(lfs, db, ptr); + } + + return new LfsBlobLoader(mediaFile); + } + } + + return loader; + } + + /** + * Run the LFS clean filter on the given stream and return a stream to the + * LFS pointer file buffer. Used when inserting objects. + * + * @param db + * the {@link Repository} + * @param originalContent + * the {@link InputStream} to the original content + * @return a {@link TemporaryBuffer} representing the LFS pointer. The + * caller is responsible to destroy the buffer. + * @throws IOException + * in case of any error. + */ + public static TemporaryBuffer cleanLfsBlob(Repository db, + InputStream originalContent) throws IOException { + LocalFile buffer = new TemporaryBuffer.LocalFile(null); + CleanFilter f = new CleanFilter(db, originalContent, buffer); + try { + while (f.run() != -1) { + // loop as long as f.run() tells there is work to do + } + } catch (IOException e) { + buffer.destroy(); + throw e; + } + return buffer; + } + +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsBlobLoader.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsBlobLoader.java new file mode 100644 index 0000000000..697ae47a68 --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsBlobLoader.java @@ -0,0 +1,119 @@ +/* + * Copyright (C) 2017, Markus Duft <markus.duft@ssi-schaefer.com> + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.eclipse.jgit.lfs; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; + +import org.eclipse.jgit.errors.LargeObjectException; +import org.eclipse.jgit.errors.MissingObjectException; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.ObjectLoader; +import org.eclipse.jgit.lib.ObjectStream; +import org.eclipse.jgit.storage.pack.PackConfig; +import org.eclipse.jgit.util.IO; + +/** + * An {@link ObjectLoader} implementation that reads a media file from the LFS + * storage. + * + * @since 4.11 + */ +public class LfsBlobLoader extends ObjectLoader { + + private Path mediaFile; + + private BasicFileAttributes attributes; + + private byte[] cached; + + /** + * Create a loader for the LFS media file at the given path. + * + * @param mediaFile + * path to the file + * @throws IOException + * in case of an error reading attributes + */ + public LfsBlobLoader(Path mediaFile) throws IOException { + this.mediaFile = mediaFile; + this.attributes = Files.readAttributes(mediaFile, + BasicFileAttributes.class); + } + + @Override + public int getType() { + return Constants.OBJ_BLOB; + } + + @Override + public long getSize() { + return attributes.size(); + } + + @Override + public byte[] getCachedBytes() throws LargeObjectException { + if (getSize() > PackConfig.DEFAULT_BIG_FILE_THRESHOLD) { + throw new LargeObjectException(); + } + + if (cached == null) { + try { + cached = IO.readFully(mediaFile.toFile()); + } catch (IOException ioe) { + throw new LargeObjectException(ioe); + } + } + return cached; + } + + @Override + public ObjectStream openStream() + throws MissingObjectException, IOException { + return new ObjectStream.Filter(getType(), getSize(), + Files.newInputStream(mediaFile)); + } + +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/SmudgeFilter.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/SmudgeFilter.java index ae7fab83af..142e74df6a 100644 --- a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/SmudgeFilter.java +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/SmudgeFilter.java @@ -100,9 +100,9 @@ public class SmudgeFilter extends FilterCommand { }; /** - * Registers this filter in JGit by calling + * Register this filter in JGit */ - public final static void register() { + static void register() { FilterCommandRegistry .register(org.eclipse.jgit.lib.Constants.BUILTIN_FILTER_PREFIX + Constants.ATTR_FILTER_DRIVER_PREFIX @@ -110,8 +110,6 @@ public class SmudgeFilter extends FilterCommand { FACTORY); } - private Lfs lfs; - /** * Constructor for SmudgeFilter. * @@ -126,13 +124,13 @@ public class SmudgeFilter extends FilterCommand { public SmudgeFilter(Repository db, InputStream in, OutputStream out) throws IOException { super(in, out); - lfs = new Lfs(db); + Lfs lfs = new Lfs(db); LfsPointer res = LfsPointer.parseLfsPointer(in); if (res != null) { AnyLongObjectId oid = res.getOid(); Path mediaFile = lfs.getMediaFile(oid); if (!Files.exists(mediaFile)) { - downloadLfsResource(db, res); + downloadLfsResource(lfs, db, res); } this.in = Files.newInputStream(mediaFile); } @@ -141,6 +139,8 @@ public class SmudgeFilter extends FilterCommand { /** * Download content which is hosted on a LFS server * + * @param lfs + * local {@link Lfs} storage. * @param db * the repository to work with * @param res @@ -148,9 +148,8 @@ public class SmudgeFilter extends FilterCommand { * @return the paths of all mediafiles which have been downloaded * @throws IOException */ - private Collection<Path> downloadLfsResource(Repository db, - LfsPointer... res) - throws IOException { + public static Collection<Path> downloadLfsResource(Lfs lfs, Repository db, + LfsPointer... res) throws IOException { Collection<Path> downloadedPaths = new ArrayList<>(); Map<String, LfsPointer> oidStr2ptr = new HashMap<>(); for (LfsPointer p : res) { diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/Constants.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/Constants.java index fbfbf377bd..835d7be02b 100644 --- a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/Constants.java +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/Constants.java @@ -56,7 +56,7 @@ import org.eclipse.jgit.lfs.internal.LfsText; @SuppressWarnings("nls") public final class Constants { /** - * lfs folder + * lfs folder/section/filter name * * @since 4.6 */ |