diff options
6 files changed, 289 insertions, 50 deletions
diff --git a/org.eclipse.jgit.lfs.test/META-INF/MANIFEST.MF b/org.eclipse.jgit.lfs.test/META-INF/MANIFEST.MF index b4c892ed97..419c4ca50c 100644 --- a/org.eclipse.jgit.lfs.test/META-INF/MANIFEST.MF +++ b/org.eclipse.jgit.lfs.test/META-INF/MANIFEST.MF @@ -7,7 +7,9 @@ Bundle-Version: 5.11.0.qualifier Bundle-Vendor: %Bundle-Vendor Bundle-Localization: plugin Bundle-RequiredExecutionEnvironment: JavaSE-1.8 -Import-Package: org.eclipse.jgit.internal.storage.dfs;version="[5.11.0,5.12.0)", +Import-Package: org.eclipse.jgit.api;version="[5.11.0,5.12.0)", + org.eclipse.jgit.attributes;version="[5.11.0,5.12.0)", + org.eclipse.jgit.internal.storage.dfs;version="[5.11.0,5.12.0)", org.eclipse.jgit.junit;version="[5.11.0,5.12.0)", org.eclipse.jgit.lfs;version="[5.11.0,5.12.0)", org.eclipse.jgit.lfs.errors;version="[5.11.0,5.12.0)", diff --git a/org.eclipse.jgit.lfs.test/tst/org/eclipse/jgit/lfs/LfsGitTest.java b/org.eclipse.jgit.lfs.test/tst/org/eclipse/jgit/lfs/LfsGitTest.java new file mode 100644 index 0000000000..8964310e41 --- /dev/null +++ b/org.eclipse.jgit.lfs.test/tst/org/eclipse/jgit/lfs/LfsGitTest.java @@ -0,0 +1,141 @@ +/* + * Copyright (C) 2021, Thomas Wolf <thomas.wolf@paranor.ch> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lfs; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; + +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.ResetCommand.ResetType; +import org.eclipse.jgit.attributes.FilterCommandRegistry; +import org.eclipse.jgit.junit.RepositoryTestCase; +import org.eclipse.jgit.lfs.lib.Constants; +import org.eclipse.jgit.lib.StoredConfig; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class LfsGitTest extends RepositoryTestCase { + + private static final String SMUDGE_NAME = org.eclipse.jgit.lib.Constants.BUILTIN_FILTER_PREFIX + + Constants.ATTR_FILTER_DRIVER_PREFIX + + org.eclipse.jgit.lib.Constants.ATTR_FILTER_TYPE_SMUDGE; + + private static final String CLEAN_NAME = org.eclipse.jgit.lib.Constants.BUILTIN_FILTER_PREFIX + + Constants.ATTR_FILTER_DRIVER_PREFIX + + org.eclipse.jgit.lib.Constants.ATTR_FILTER_TYPE_CLEAN; + + @BeforeClass + public static void installLfs() { + FilterCommandRegistry.register(SMUDGE_NAME, SmudgeFilter.FACTORY); + FilterCommandRegistry.register(CLEAN_NAME, CleanFilter.FACTORY); + } + + @AfterClass + public static void removeLfs() { + FilterCommandRegistry.unregister(SMUDGE_NAME); + FilterCommandRegistry.unregister(CLEAN_NAME); + } + + private Git git; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + git = new Git(db); + // commit something + writeTrashFile("Test.txt", "Hello world"); + git.add().addFilepattern("Test.txt").call(); + git.commit().setMessage("Initial commit").call(); + // prepare the config for LFS + StoredConfig config = git.getRepository().getConfig(); + config.setString("filter", "lfs", "clean", CLEAN_NAME); + config.setString("filter", "lfs", "smudge", SMUDGE_NAME); + config.save(); + } + + @Test + public void checkoutNonLfsPointer() throws Exception { + String content = "size_t\nsome_function(void* ptr);\n"; + File smallFile = writeTrashFile("Test.txt", content); + StringBuilder largeContent = new StringBuilder( + LfsPointer.SIZE_THRESHOLD * 4); + while (largeContent.length() < LfsPointer.SIZE_THRESHOLD * 4) { + largeContent.append(content); + } + File largeFile = writeTrashFile("large.txt", largeContent.toString()); + fsTick(largeFile); + git.add().addFilepattern("Test.txt").addFilepattern("large.txt").call(); + git.commit().setMessage("Text files").call(); + writeTrashFile(".gitattributes", "*.txt filter=lfs"); + git.add().addFilepattern(".gitattributes").call(); + git.commit().setMessage("attributes").call(); + assertTrue(smallFile.delete()); + assertTrue(largeFile.delete()); + // This reset will run the two text files through the smudge filter + git.reset().setMode(ResetType.HARD).call(); + assertTrue(smallFile.exists()); + assertTrue(largeFile.exists()); + checkFile(smallFile, content); + checkFile(largeFile, largeContent.toString()); + // Modify the large file + largeContent.append(content); + writeTrashFile("large.txt", largeContent.toString()); + // This should convert largeFile to an LFS pointer + git.add().addFilepattern("large.txt").call(); + git.commit().setMessage("Large modified").call(); + String lfsPtr = "version https://git-lfs.github.com/spec/v1\n" + + "oid sha256:d041ab19bd7edd899b3c0450d0f61819f96672f0b22d26c9753abc62e1261614\n" + + "size 858\n"; + assertEquals("[.gitattributes, mode:100644, content:*.txt filter=lfs]" + + "[Test.txt, mode:100644, content:" + content + ']' + + "[large.txt, mode:100644, content:" + lfsPtr + ']', + indexState(CONTENT)); + // Verify the file has been saved + File savedFile = new File(db.getDirectory(), "lfs"); + savedFile = new File(savedFile, "objects"); + savedFile = new File(savedFile, "d0"); + savedFile = new File(savedFile, "41"); + savedFile = new File(savedFile, + "d041ab19bd7edd899b3c0450d0f61819f96672f0b22d26c9753abc62e1261614"); + String saved = new String(Files.readAllBytes(savedFile.toPath()), + StandardCharsets.UTF_8); + assertEquals(saved, largeContent.toString()); + + assertTrue(smallFile.delete()); + assertTrue(largeFile.delete()); + git.reset().setMode(ResetType.HARD).call(); + assertTrue(smallFile.exists()); + assertTrue(largeFile.exists()); + checkFile(smallFile, content); + checkFile(largeFile, largeContent.toString()); + assertEquals("[.gitattributes, mode:100644, content:*.txt filter=lfs]" + + "[Test.txt, mode:100644, content:" + content + ']' + + "[large.txt, mode:100644, content:" + lfsPtr + ']', + indexState(CONTENT)); + git.add().addFilepattern("Test.txt").call(); + git.commit().setMessage("Small committed again").call(); + String lfsPtrSmall = "version https://git-lfs.github.com/spec/v1\n" + + "oid sha256:9110463275fb0e2f0e9fdeaf84e598e62915666161145cf08927079119cc7814\n" + + "size 33\n"; + assertEquals("[.gitattributes, mode:100644, content:*.txt filter=lfs]" + + "[Test.txt, mode:100644, content:" + lfsPtrSmall + ']' + + "[large.txt, mode:100644, content:" + lfsPtr + ']', + indexState(CONTENT)); + + assertTrue(git.status().call().isClean()); + } +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsBlobFilter.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsBlobFilter.java index 52c3001b85..032a19b5df 100644 --- a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsBlobFilter.java +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsBlobFilter.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2017, Markus Duft <markus.duft@ssi-schaefer.com> and others + * Copyright (C) 2017, 2021 Markus Duft <markus.duft@ssi-schaefer.com> and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -45,7 +45,7 @@ public class LfsBlobFilter { */ public static ObjectLoader smudgeLfsBlob(Repository db, ObjectLoader loader) throws IOException { - if (loader.getSize() > LfsPointer.SIZE_THRESHOLD) { + if (loader.getSize() > LfsPointer.FULL_SIZE_THRESHOLD) { return loader; } diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsPointer.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsPointer.java index aef4416387..0a8a3faec3 100644 --- a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsPointer.java +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsPointer.java @@ -11,7 +11,9 @@ package org.eclipse.jgit.lfs; import static java.nio.charset.StandardCharsets.UTF_8; +import java.io.BufferedInputStream; import java.io.BufferedReader; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -25,6 +27,7 @@ import org.eclipse.jgit.annotations.Nullable; import org.eclipse.jgit.lfs.lib.AnyLongObjectId; import org.eclipse.jgit.lfs.lib.Constants; import org.eclipse.jgit.lfs.lib.LongObjectId; +import org.eclipse.jgit.util.IO; /** * Represents an LFS pointer file @@ -57,6 +60,12 @@ public class LfsPointer implements Comparable<LfsPointer> { public static final String HASH_FUNCTION_NAME = Constants.LONG_HASH_FUNCTION .toLowerCase(Locale.ROOT).replace("-", ""); //$NON-NLS-1$ //$NON-NLS-2$ + /** + * {@link #SIZE_THRESHOLD} is too low; with lfs extensions a LFS pointer can + * be larger. But 8kB should be more than enough. + */ + static final int FULL_SIZE_THRESHOLD = 8 * 1024; + private final AnyLongObjectId oid; private final long size; @@ -115,64 +124,113 @@ public class LfsPointer implements Comparable<LfsPointer> { /** * Try to parse the data provided by an InputStream to the format defined by - * {@link #VERSION} + * {@link #VERSION}. If the given stream supports mark and reset as + * indicated by {@link InputStream#markSupported()}, its input position will + * be reset if the stream content is not actually a LFS pointer (i.e., when + * {@code null} is returned). If the stream content is an invalid LFS + * pointer or the given stream does not support mark/reset, the input + * position may not be reset. * * @param in * the {@link java.io.InputStream} from where to read the data - * @return an {@link org.eclipse.jgit.lfs.LfsPointer} or <code>null</code> - * if the stream was not parseable as LfsPointer + * @return an {@link org.eclipse.jgit.lfs.LfsPointer} or {@code null} if the + * stream was not parseable as LfsPointer * @throws java.io.IOException */ @Nullable public static LfsPointer parseLfsPointer(InputStream in) throws IOException { + if (in.markSupported()) { + return parse(in); + } + // Fallback; note that while parse() resets its input stream, that won't + // reset "in". + return parse(new BufferedInputStream(in)); + } + + @Nullable + private static LfsPointer parse(InputStream in) + throws IOException { + if (!in.markSupported()) { + // No translation; internal error + throw new IllegalArgumentException( + "LFS pointer parsing needs InputStream.markSupported() == true"); //$NON-NLS-1$ + } + // Try reading only a short block first. + in.mark(SIZE_THRESHOLD); + byte[] preamble = new byte[SIZE_THRESHOLD]; + int length = IO.readFully(in, preamble, 0); + if (length < preamble.length || in.read() < 0) { + // We have the whole file. Try to parse a pointer from it. + try (BufferedReader r = new BufferedReader(new InputStreamReader( + new ByteArrayInputStream(preamble, 0, length), UTF_8))) { + LfsPointer ptr = parse(r); + if (ptr == null) { + in.reset(); + } + return ptr; + } + } + // Longer than SIZE_THRESHOLD: expect "version" to be the first line. + boolean hasVersion = checkVersion(preamble); + in.reset(); + if (!hasVersion) { + return null; + } + in.mark(FULL_SIZE_THRESHOLD); + byte[] fullPointer = new byte[FULL_SIZE_THRESHOLD]; + length = IO.readFully(in, fullPointer, 0); + if (length == fullPointer.length && in.read() >= 0) { + in.reset(); + return null; // Too long. + } + try (BufferedReader r = new BufferedReader(new InputStreamReader( + new ByteArrayInputStream(fullPointer, 0, length), UTF_8))) { + LfsPointer ptr = parse(r); + if (ptr == null) { + in.reset(); + } + return ptr; + } + } + + private static LfsPointer parse(BufferedReader r) throws IOException { boolean versionLine = false; LongObjectId id = null; long sz = -1; - // This parsing is a bit too general if we go by the spec at // https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md - // Comment lines are not mentioned in the spec, and the "version" line - // MUST be the first. - try (BufferedReader br = new BufferedReader( - new InputStreamReader(in, UTF_8))) { - for (String s = br.readLine(); s != null; s = br.readLine()) { - if (s.startsWith("#") || s.length() == 0) { //$NON-NLS-1$ - continue; - } else if (s.startsWith("version")) { //$NON-NLS-1$ - if (versionLine || s.length() < 8 || s.charAt(7) != ' ') { - return null; // Not a LFS pointer - } - String rest = s.substring(8).trim(); - versionLine = VERSION.equals(rest) - || VERSION_LEGACY.equals(rest); - if (!versionLine) { - return null; // Not a LFS pointer - } - } else { - try { - if (s.startsWith("oid sha256:")) { //$NON-NLS-1$ - if (id != null) { - return null; // Not a LFS pointer - } - id = LongObjectId - .fromString(s.substring(11).trim()); - } else if (s.startsWith("size")) { //$NON-NLS-1$ - if (sz > 0 || s.length() < 5 - || s.charAt(4) != ' ') { - return null; // Not a LFS pointer - } - sz = Long.parseLong(s.substring(5).trim()); + // Comment lines are not mentioned in the spec, the "version" line + // MUST be the first, and keys are ordered alphabetically. + for (String s = r.readLine(); s != null; s = r.readLine()) { + if (s.startsWith("#") || s.length() == 0) { //$NON-NLS-1$ + continue; + } else if (s.startsWith("version")) { //$NON-NLS-1$ + if (versionLine || !checkVersionLine(s)) { + return null; // Not a LFS pointer + } + versionLine = true; + } else { + try { + if (s.startsWith("oid sha256:")) { //$NON-NLS-1$ + if (id != null) { + return null; // Not a LFS pointer } - } catch (RuntimeException e) { - // We could not parse the line. If we have a version - // already, this is a corrupt LFS pointer. Otherwise it - // is just not an LFS pointer. - if (versionLine) { - throw e; + id = LongObjectId.fromString(s.substring(11).trim()); + } else if (s.startsWith("size")) { //$NON-NLS-1$ + if (sz > 0 || s.length() < 5 || s.charAt(4) != ' ') { + return null; // Not a LFS pointer } - return null; + sz = Long.parseLong(s.substring(5).trim()); } + } catch (RuntimeException e) { + // We could not parse the line. If we have a version + // already, this is a corrupt LFS pointer. Otherwise it + // is just not an LFS pointer. + if (versionLine) { + throw e; + } + return null; } } if (versionLine && id != null && sz > -1) { @@ -182,6 +240,30 @@ public class LfsPointer implements Comparable<LfsPointer> { return null; } + private static boolean checkVersion(byte[] data) { + // According to the spec at + // https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md + // it MUST always be the first line. + try (BufferedReader r = new BufferedReader( + new InputStreamReader(new ByteArrayInputStream(data), UTF_8))) { + String s = r.readLine(); + if (s != null && s.startsWith("version")) { //$NON-NLS-1$ + return checkVersionLine(s); + } + } catch (IOException e) { + // Doesn't occur, we're reading from a byte array! + } + return false; + } + + private static boolean checkVersionLine(String s) { + if (s.length() < 8 || s.charAt(7) != ' ') { + return false; // Not a valid LFS pointer version line + } + String rest = s.substring(8).trim(); + return VERSION.equals(rest) || VERSION_LEGACY.equals(rest); + } + /** {@inheritDoc} */ @Override public String toString() { diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/SmudgeFilter.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/SmudgeFilter.java index 2f80d5b9a7..3411887567 100644 --- a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/SmudgeFilter.java +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/SmudgeFilter.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2016, Christian Halstrick <christian.halstrick@sap.com> and others + * Copyright (C) 2016, 2021 Christian Halstrick <christian.halstrick@sap.com> and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -11,6 +11,7 @@ package org.eclipse.jgit.lfs; import static java.nio.charset.StandardCharsets.UTF_8; +import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -87,20 +88,31 @@ public class SmudgeFilter extends FilterCommand { */ public SmudgeFilter(Repository db, InputStream in, OutputStream out) throws IOException { + this(in.markSupported() ? in : new BufferedInputStream(in), out, db); + } + + private SmudgeFilter(InputStream in, OutputStream out, Repository db) + throws IOException { super(in, out); + InputStream from = in; try { - Lfs lfs = new Lfs(db); - LfsPointer res = LfsPointer.parseLfsPointer(in); + LfsPointer res = LfsPointer.parseLfsPointer(from); if (res != null) { AnyLongObjectId oid = res.getOid(); + Lfs lfs = new Lfs(db); Path mediaFile = lfs.getMediaFile(oid); if (!Files.exists(mediaFile)) { downloadLfsResource(lfs, db, res); } this.in = Files.newInputStream(mediaFile); + } else { + // Not swapped; stream was reset, don't close! + from = null; } } finally { - in.close(); // make sure the swapped stream is closed properly. + if (from != null) { + from.close(); // Close the swapped-out stream + } } } diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/LfsPointerFilter.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/LfsPointerFilter.java index d84eebd226..99bae49abb 100644 --- a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/LfsPointerFilter.java +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/LfsPointerFilter.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2015, 2017, Dariusz Luksza <dariusz@luksza.org> and others + * Copyright (C) 2015, 2021 Dariusz Luksza <dariusz@luksza.org> and others * * This program and the accompanying materials are made available under the * terms of the Eclipse Distribution License v. 1.0 which is available at @@ -58,6 +58,8 @@ public class LfsPointerFilter extends TreeFilter { try (ObjectStream stream = object.openStream()) { pointer = LfsPointer.parseLfsPointer(stream); return pointer != null; + } catch (RuntimeException e) { + return false; } } |