diff options
Diffstat (limited to 'org.eclipse.jgit.lfs')
8 files changed, 534 insertions, 5 deletions
diff --git a/org.eclipse.jgit.lfs/META-INF/MANIFEST.MF b/org.eclipse.jgit.lfs/META-INF/MANIFEST.MF index d7a2f37498..f1eeb512aa 100644 --- a/org.eclipse.jgit.lfs/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit.lfs/META-INF/MANIFEST.MF @@ -5,11 +5,13 @@ Bundle-SymbolicName: org.eclipse.jgit.lfs Bundle-Version: 4.6.0.qualifier Bundle-Localization: plugin Bundle-Vendor: %provider_name -Export-Package: org.eclipse.jgit.lfs.errors;version="4.6.0", +Export-Package: org.eclipse.jgit.lfs;version="4.6.0", + org.eclipse.jgit.lfs.errors;version="4.6.0", org.eclipse.jgit.lfs.internal;version="4.6.0";x-friends:="org.eclipse.jgit.lfs.test", org.eclipse.jgit.lfs.lib;version="4.6.0" Bundle-RequiredExecutionEnvironment: JavaSE-1.7 -Import-Package: org.eclipse.jgit.internal.storage.file;version="[4.6.0,4.7.0)", +Import-Package: org.eclipse.jgit.attributes;version="[4.6.0,4.7.0)", + org.eclipse.jgit.internal.storage.file;version="[4.6.0,4.7.0)", org.eclipse.jgit.lib;version="[4.6.0,4.7.0)", org.eclipse.jgit.nls;version="[4.6.0,4.7.0)", org.eclipse.jgit.util;version="[4.6.0,4.7.0)" diff --git a/org.eclipse.jgit.lfs/resources/org/eclipse/jgit/lfs/internal/LfsText.properties b/org.eclipse.jgit.lfs/resources/org/eclipse/jgit/lfs/internal/LfsText.properties index 7c3aea2261..25c101f0b0 100644 --- a/org.eclipse.jgit.lfs/resources/org/eclipse/jgit/lfs/internal/LfsText.properties +++ b/org.eclipse.jgit.lfs/resources/org/eclipse/jgit/lfs/internal/LfsText.properties @@ -1,7 +1,8 @@ incorrectLONG_OBJECT_ID_LENGTH=Incorrect LONG_OBJECT_ID_LENGTH. +inconsistentMediafileLength=mediafile {0} has unexpected length; expected {1} but found {2}. invalidLongId=Invalid id: {0} invalidLongIdLength=Invalid id length {0}; should be {1} requiredHashFunctionNotAvailable=Required hash function {0} not available. repositoryNotFound=Repository {0} not found repositoryReadOnly=Repository {0} is read-only -lfsUnavailable=LFS is not available for repository {0}
\ No newline at end of file +lfsUnavailable=LFS is not available for repository {0} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/CleanFilter.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/CleanFilter.java new file mode 100644 index 0000000000..f7b55e579b --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/CleanFilter.java @@ -0,0 +1,176 @@ +/* + * Copyright (C) 2016, Christian Halstrick <christian.halstrick@sap.com> + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.eclipse.jgit.lfs; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.security.DigestOutputStream; + +import org.eclipse.jgit.attributes.FilterCommand; +import org.eclipse.jgit.attributes.FilterCommandFactory; +import org.eclipse.jgit.attributes.FilterCommandRegistry; +import org.eclipse.jgit.lfs.errors.CorruptMediaFile; +import org.eclipse.jgit.lfs.lib.Constants; +import org.eclipse.jgit.lfs.lib.LongObjectId; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.util.FileUtils; + +/** + * Built-in LFS clean filter + * + * When new content is about to be added to the git repository and this filter + * is configured for that content, then this filter will replace the original + * content with content of a so-called LFS pointer file. The pointer file + * content will then be added to the git repository. Additionally this filter + * writes the original content in a so-called 'media file' to '.git/lfs/objects/ + * <first-two-characters-of-contentid>/<rest-of-contentid>' + * + * @see <a href="https://github.com/github/git-lfs/blob/master/docs/spec.md">Git + * LFS Specification</a> + * @since 4.6 + */ +public class CleanFilter extends FilterCommand { + /** + * The factory is responsible for creating instances of {@link CleanFilter} + */ + public final static FilterCommandFactory FACTORY = new FilterCommandFactory() { + + @Override + public FilterCommand create(Repository db, InputStream in, + OutputStream out) throws IOException { + return new CleanFilter(db, in, out); + } + }; + + /** + * Registers this filter by calling + * {@link FilterCommandRegistry#register(String, FilterCommandFactory)} + */ + public final static void register() { + FilterCommandRegistry.register( + org.eclipse.jgit.lib.Constants.BUILTIN_FILTER_PREFIX + + "lfs/clean", //$NON-NLS-1$ + FACTORY); + } + + // The OutputStream to a temporary file which will be renamed to mediafile + // when the operation succeeds + private OutputStream tmpOut; + + // Used to compute the hash for the original content + private DigestOutputStream dOut; + + private Lfs lfsUtil; + + // the size of the original content + private long size; + + // a temporary file into which the original content is written. When no + // errors occur this file will be renamed to the mediafile + private Path tmpFile; + + /** + * @param db + * the repository + * @param in + * an {@link InputStream} providing the original content + * @param out + * the {@link OutputStream} into which the content of the pointer + * file should be written. That's the content which will be added + * to the git repository + * @throws IOException + * when the creation of the temporary file fails or when no + * {@link OutputStream} for this file can be created + */ + public CleanFilter(Repository db, InputStream in, OutputStream out) + throws IOException { + super(in, out); + lfsUtil = new Lfs(db.getDirectory().toPath().resolve("lfs")); //$NON-NLS-1$ + Files.createDirectories(lfsUtil.getLfsTmpDir()); + tmpFile = lfsUtil.createTmpFile(); + tmpOut = Files.newOutputStream(tmpFile, + StandardOpenOption.CREATE); + this.dOut = new DigestOutputStream( + tmpOut, + Constants.newMessageDigest()); + } + + public int run() throws IOException { + try { + int b = in.read(); + if (b != -1) { + dOut.write(b); + size++; + return 1; + } else { + dOut.close(); + tmpOut.close(); + LongObjectId loid = LongObjectId + .fromRaw(dOut.getMessageDigest().digest()); + Path mediaFile = lfsUtil.getMediaFile(loid); + if (Files.isRegularFile(mediaFile)) { + long fsSize = Files.size(mediaFile); + if (fsSize != size) { + throw new CorruptMediaFile(mediaFile, size, fsSize); + } + } else { + FileUtils.mkdirs(mediaFile.getParent().toFile(), true); + FileUtils.rename(tmpFile.toFile(), mediaFile.toFile()); + } + LfsPointer lfsPointer = new LfsPointer(loid, size); + lfsPointer.encode(out); + out.close(); + return -1; + } + } catch (IOException e) { + out.close(); + dOut.close(); + tmpOut.close(); + throw e; + } + } +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/Lfs.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/Lfs.java new file mode 100644 index 0000000000..f099c5a761 --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/Lfs.java @@ -0,0 +1,124 @@ +/* + * Copyright (C) 2016, Christian Halstrick <christian.halstrick@sap.com> + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.eclipse.jgit.lfs; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.eclipse.jgit.lfs.lib.LongObjectId; + +/** + * Class which represents the lfs folder hierarchy inside a .git folder + * + * @since 4.6 + */ +public class Lfs { + private Path root; + + private Path objDir; + + private Path tmpDir; + + /** + * @param root + * the path to the LFS media directory. Will be "<repo>/.git/lfs" + */ + public Lfs(Path root) { + this.root = root; + } + + /** + * @return the path to the LFS directory + */ + public Path getLfsRoot() { + return root; + } + + /** + * @return the path to the temp directory used by LFS. Will be + * "<repo>/.git/lfs/tmp" + */ + public Path getLfsTmpDir() { + if (tmpDir == null) { + tmpDir = root.resolve("tmp"); //$NON-NLS-1$ + } + return tmpDir; + } + + /** + * @return the path to the object directory used by LFS. Will be + * "<repo>/.git/lfs/objects" + */ + public Path getLfsObjDir() { + if (objDir == null) { + objDir = root.resolve("objects"); //$NON-NLS-1$ + } + return objDir; + } + + /** + * @param id + * the id of the mediafile + * @return the file which stores the original content. This will be files + * underneath + * "<repo>/.git/lfs/objects/<firstTwoLettersOfID>/<remainingLettersOfID>" + */ + public Path getMediaFile(LongObjectId id) { + String idStr = LongObjectId.toString(id); + return getLfsObjDir().resolve(idStr.substring(0, 2)) + .resolve(idStr.substring(2)); + } + + /** + * Create a new temp file in the LFS directory + * + * @return a new temporary file in the LFS directory + * @throws IOException + * when the temp file could not be created + */ + public Path createTmpFile() throws IOException { + return Files.createTempFile(getLfsTmpDir(), null, null); + } + +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsPointer.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsPointer.java new file mode 100644 index 0000000000..2521e63163 --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsPointer.java @@ -0,0 +1,121 @@ +/* + * Copyright (C) 2016, Christian Halstrick <christian.halstrick@sap.com> + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.eclipse.jgit.lfs; + +import java.io.OutputStream; +import java.io.PrintStream; + +import org.eclipse.jgit.lfs.lib.Constants; +import org.eclipse.jgit.lfs.lib.LongObjectId; + +/** + * Represents an LFS pointer file + * + * @since 4.6 + */ +public class LfsPointer { + /** + * The version of the LfsPointer file format + */ + public static final String VERSION = "https://git-lfs.github.com/spec/v1"; //$NON-NLS-1$ + + /** + * The name of the hash function as used in the pointer files. This will + * evaluate to "sha256" + */ + public static final String HASH_FUNCTION_NAME = Constants.LONG_HASH_FUNCTION + .toLowerCase().replace("-", ""); //$NON-NLS-1$ //$NON-NLS-2$ + + private LongObjectId oid; + + private long size; + + /** + * @param oid + * the id of the content + * @param size + * the size of the content + */ + public LfsPointer(LongObjectId oid, long size) { + this.oid = oid; + this.size = size; + } + + /** + * @return the id of the content + */ + public LongObjectId getOid() { + return oid; + } + + /** + * @return the size of the content + */ + public long getSize() { + return size; + } + + /** + * Encode this object into the LFS format defined by {@link #VERSION} + * + * @param out + * the {@link OutputStream} into which the encoded data should be + * written + */ + public void encode(OutputStream out) { + try (PrintStream ps = new PrintStream(out)) { + ps.print("version "); //$NON-NLS-1$ + ps.println(VERSION); + ps.print("oid " + HASH_FUNCTION_NAME + ":"); //$NON-NLS-1$ //$NON-NLS-2$ + ps.println(LongObjectId.toString(oid)); + ps.print("size "); //$NON-NLS-1$ + ps.println(size); + } + } + + @Override + public String toString() { + return "LfsPointer: oid=" + LongObjectId.toString(oid) + ", size=" //$NON-NLS-1$ //$NON-NLS-2$ + + size; + } +}
\ No newline at end of file diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/CorruptMediaFile.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/CorruptMediaFile.java new file mode 100644 index 0000000000..f2b51c0442 --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/CorruptMediaFile.java @@ -0,0 +1,100 @@ +/* + * Copyright (C) 2016, Christian Halstrick <christian.halstrick@sap.com> + * and other copyright owners as documented in the project's IP log. + * + * This program and the accompanying materials are made available + * under the terms of the Eclipse Distribution License v1.0 which + * accompanies this distribution, is reproduced below, and is + * available at http://www.eclipse.org/org/documents/edl-v10.php + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * - Neither the name of the Eclipse Foundation, Inc. nor the + * names of its contributors may be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF + * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.eclipse.jgit.lfs.errors; + +import java.io.IOException; +import java.nio.file.Path; +import java.text.MessageFormat; + +import org.eclipse.jgit.lfs.internal.LfsText; + +/** + * Thrown when a LFS mediafile is found which doesn't have the expected size + * + * @since 4.6 + */ +public class CorruptMediaFile extends IOException { + private static final long serialVersionUID = 1L; + + private Path mediaFile; + + private long expectedSize; + + private long size; + + /** + * @param mediaFile + * @param expectedSize + * @param size + */ + @SuppressWarnings("boxing") + public CorruptMediaFile(Path mediaFile, long expectedSize, + long size) { + super(MessageFormat.format(LfsText.get().inconsistentMediafileLength, + mediaFile, expectedSize, size)); + this.mediaFile = mediaFile; + this.expectedSize = expectedSize; + this.size = size; + } + + /** + * @return the media file which seems to be corrupt + */ + public Path getMediaFile() { + return mediaFile; + } + + /** + * @return the expected size of the media file + */ + public long getExpectedSize() { + return expectedSize; + } + + /** + * @return the actual size of the media file in the file system + */ + public long getSize() { + return size; + } +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsText.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsText.java index 365eaa1727..0aad5c9dac 100644 --- a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsText.java +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsText.java @@ -58,6 +58,7 @@ public class LfsText extends TranslationBundle { } // @formatter:off + /***/ public String inconsistentMediafileLength; /***/ public String incorrectLONG_OBJECT_ID_LENGTH; /***/ public String invalidLongId; /***/ public String invalidLongIdLength; diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/Constants.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/Constants.java index d2464126cc..269cbc3a83 100644 --- a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/Constants.java +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/Constants.java @@ -55,8 +55,12 @@ import org.eclipse.jgit.lfs.internal.LfsText; **/ @SuppressWarnings("nls") public final class Constants { - /** Hash function used natively by Git LFS extension for large objects. */ - private static final String LONG_HASH_FUNCTION = "SHA-256"; + /** + * Hash function used natively by Git LFS extension for large objects. + * + * @since 4.6 + */ + public static final String LONG_HASH_FUNCTION = "SHA-256"; /** * A Git LFS large object hash is 256 bits, i.e. 32 bytes. |