diff options
Diffstat (limited to 'org.eclipse.jgit.lfs/src')
33 files changed, 4488 insertions, 0 deletions
diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/BuiltinLFS.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/BuiltinLFS.java new file mode 100644 index 0000000000..4ef5487347 --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/BuiltinLFS.java @@ -0,0 +1,123 @@ +/* + * Copyright (C) 2017, Markus Duft <markus.duft@ssi-schaefer.com> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lfs; + +import java.io.IOException; +import java.io.InputStream; +import java.io.PrintStream; + +import org.eclipse.jgit.annotations.Nullable; +import org.eclipse.jgit.attributes.Attribute; +import org.eclipse.jgit.hooks.PrePushHook; +import org.eclipse.jgit.lib.ConfigConstants; +import org.eclipse.jgit.lib.ObjectLoader; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.util.LfsFactory; + +/** + * Implementation of {@link LfsFactory}, using built-in (optional) LFS support. + * + * @since 4.11 + */ +public class BuiltinLFS extends LfsFactory { + + private BuiltinLFS() { + SmudgeFilter.register(); + CleanFilter.register(); + } + + /** + * Activates the built-in LFS support. + */ + public static void register() { + setInstance(new BuiltinLFS()); + } + + @Override + public boolean isAvailable() { + return true; + } + + @Override + public ObjectLoader applySmudgeFilter(Repository db, ObjectLoader loader, + Attribute attribute) throws IOException { + if (isEnabled(db) && (attribute == null || isEnabled(db, attribute))) { + return LfsBlobFilter.smudgeLfsBlob(db, loader); + } + return loader; + } + + @Override + public LfsInputStream applyCleanFilter(Repository db, InputStream input, + long length, Attribute attribute) throws IOException { + if (isEnabled(db, attribute)) { + return new LfsInputStream(LfsBlobFilter.cleanLfsBlob(db, input)); + } + return new LfsInputStream(input, length); + } + + @Override + @Nullable + public PrePushHook getPrePushHook(Repository repo, + PrintStream outputStream) { + if (isEnabled(repo)) { + return new LfsPrePushHook(repo, outputStream); + } + return null; + } + + @Override + @Nullable + public PrePushHook getPrePushHook(Repository repo, PrintStream outputStream, + PrintStream errorStream) { + if (isEnabled(repo)) { + return new LfsPrePushHook(repo, outputStream, errorStream); + } + return null; + } + + /** + * @param db + * the repository + * @return whether LFS is requested for the given repo. + */ + @Override + public boolean isEnabled(Repository db) { + if (db == null) { + return false; + } + return db.getConfig().getBoolean(ConfigConstants.CONFIG_FILTER_SECTION, + ConfigConstants.CONFIG_SECTION_LFS, + ConfigConstants.CONFIG_KEY_USEJGITBUILTIN, + false); + } + + /** + * @param db + * the repository + * @param attribute + * the attribute to check + * @return whether LFS filter is enabled for the given .gitattribute + * attribute. + */ + private boolean isEnabled(Repository db, Attribute attribute) { + if (attribute == null) { + return false; + } + return isEnabled(db) && ConfigConstants.CONFIG_SECTION_LFS + .equals(attribute.getValue()); + } + + @Override + public LfsInstallCommand getInstallCommand() { + return new InstallBuiltinLfsCommand(); + } + +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/CleanFilter.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/CleanFilter.java new file mode 100644 index 0000000000..13b74dff7d --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/CleanFilter.java @@ -0,0 +1,140 @@ +/* + * Copyright (C) 2016, Christian Halstrick <christian.halstrick@sap.com> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lfs; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; + +import org.eclipse.jgit.attributes.FilterCommand; +import org.eclipse.jgit.attributes.FilterCommandFactory; +import org.eclipse.jgit.attributes.FilterCommandRegistry; +import org.eclipse.jgit.lfs.errors.CorruptMediaFile; +import org.eclipse.jgit.lfs.internal.AtomicObjectOutputStream; +import org.eclipse.jgit.lfs.lib.AnyLongObjectId; +import org.eclipse.jgit.lfs.lib.Constants; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.util.FileUtils; + +/** + * Built-in LFS clean filter + * + * When new content is about to be added to the git repository and this filter + * is configured for that content, then this filter will replace the original + * content with content of a so-called LFS pointer file. The pointer file + * content will then be added to the git repository. Additionally this filter + * writes the original content in a so-called 'media file' to '.git/lfs/objects/ + * <first-two-characters-of-contentid>/<rest-of-contentid>' + * + * @see <a href="https://github.com/github/git-lfs/blob/master/docs/spec.md">Git + * LFS Specification</a> + * @since 4.6 + */ +public class CleanFilter extends FilterCommand { + /** + * The factory is responsible for creating instances of + * {@link org.eclipse.jgit.lfs.CleanFilter} + */ + public static final FilterCommandFactory FACTORY = CleanFilter::new; + + /** + * Registers this filter by calling + * {@link FilterCommandRegistry#register(String, FilterCommandFactory)} + */ + static void register() { + FilterCommandRegistry + .register(org.eclipse.jgit.lib.Constants.BUILTIN_FILTER_PREFIX + + Constants.ATTR_FILTER_DRIVER_PREFIX + + org.eclipse.jgit.lib.Constants.ATTR_FILTER_TYPE_CLEAN, + FACTORY); + } + + // Used to compute the hash for the original content + private AtomicObjectOutputStream aOut; + + private Lfs lfsUtil; + + // the size of the original content + private long size; + + // a temporary file into which the original content is written. When no + // errors occur this file will be renamed to the mediafile + private Path tmpFile; + + /** + * Constructor for CleanFilter. + * + * @param db + * the repository + * @param in + * an {@link java.io.InputStream} providing the original content + * @param out + * the {@link java.io.OutputStream} into which the content of the + * pointer file should be written. That's the content which will + * be added to the git repository + * @throws java.io.IOException + * when the creation of the temporary file fails or when no + * {@link java.io.OutputStream} for this file can be created + */ + public CleanFilter(Repository db, InputStream in, OutputStream out) + throws IOException { + super(in, out); + lfsUtil = new Lfs(db); + Files.createDirectories(lfsUtil.getLfsTmpDir()); + tmpFile = lfsUtil.createTmpFile(); + this.aOut = new AtomicObjectOutputStream(tmpFile.toAbsolutePath()); + } + + @Override + public int run() throws IOException { + try { + byte[] buf = new byte[8192]; + int length = in.read(buf); + if (length != -1) { + aOut.write(buf, 0, length); + size += length; + return length; + } + aOut.close(); + AnyLongObjectId loid = aOut.getId(); + aOut = null; + Path mediaFile = lfsUtil.getMediaFile(loid); + if (Files.isRegularFile(mediaFile)) { + long fsSize = Files.size(mediaFile); + if (fsSize != size) { + throw new CorruptMediaFile(mediaFile, size, fsSize); + } + FileUtils.delete(tmpFile.toFile()); + } else { + Path parent = mediaFile.getParent(); + if (parent != null) { + FileUtils.mkdirs(parent.toFile(), true); + } + FileUtils.rename(tmpFile.toFile(), mediaFile.toFile(), + StandardCopyOption.ATOMIC_MOVE); + } + LfsPointer lfsPointer = new LfsPointer(loid, size); + lfsPointer.encode(out); + in.close(); + out.close(); + return -1; + } catch (IOException e) { + if (aOut != null) { + aOut.abort(); + } + in.close(); + out.close(); + throw e; + } + } +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/InstallBuiltinLfsCommand.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/InstallBuiltinLfsCommand.java new file mode 100644 index 0000000000..4fc200e08c --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/InstallBuiltinLfsCommand.java @@ -0,0 +1,100 @@ +/* + * Copyright (C) 2018, Markus Duft <markus.duft@ssi-schaefer.com> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lfs; + +import java.io.IOException; + +import org.eclipse.jgit.api.errors.InvalidConfigurationException; +import org.eclipse.jgit.errors.ConfigInvalidException; +import org.eclipse.jgit.lib.ConfigConstants; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.lib.StoredConfig; +import org.eclipse.jgit.util.FS; +import org.eclipse.jgit.util.LfsFactory.LfsInstallCommand; +import org.eclipse.jgit.util.SystemReader; + +/** + * Installs all required LFS properties for the current user, analogous to 'git + * lfs install', but defaulting to using JGit builtin hooks. + * + * @since 4.11 + */ +public class InstallBuiltinLfsCommand implements LfsInstallCommand { + + private static final String[] ARGS_USER = new String[] { "lfs", "install" }; //$NON-NLS-1$//$NON-NLS-2$ + + private static final String[] ARGS_LOCAL = new String[] { "lfs", "install", //$NON-NLS-1$//$NON-NLS-2$ + "--local" }; //$NON-NLS-1$ + + private Repository repository; + + /** + * {@inheritDoc} + * + * @throws IOException + * if an I/O error occurs while accessing a git config or + * executing {@code git lfs install} in an external process + * @throws InvalidConfigurationException + * if a git configuration is invalid + * @throws InterruptedException + * if the current thread is interrupted while waiting for the + * {@code git lfs install} executed in an external process + */ + @Override + public Void call() throws IOException, InvalidConfigurationException, + InterruptedException { + StoredConfig cfg = null; + if (repository == null) { + try { + cfg = SystemReader.getInstance().getUserConfig(); + } catch (ConfigInvalidException e) { + throw new InvalidConfigurationException(e.getMessage(), e); + } + } else { + cfg = repository.getConfig(); + } + + cfg.setBoolean(ConfigConstants.CONFIG_FILTER_SECTION, + ConfigConstants.CONFIG_SECTION_LFS, + ConfigConstants.CONFIG_KEY_USEJGITBUILTIN, true); + cfg.setBoolean(ConfigConstants.CONFIG_FILTER_SECTION, + ConfigConstants.CONFIG_SECTION_LFS, + ConfigConstants.CONFIG_KEY_REQUIRED, true); + + cfg.save(); + + // try to run git lfs install, we really don't care if it is present + // and/or works here (yet). + ProcessBuilder builder = FS.DETECTED.runInShell("git", //$NON-NLS-1$ + repository == null ? ARGS_USER : ARGS_LOCAL); + if (repository != null) { + builder.directory(repository.isBare() ? repository.getDirectory() + : repository.getWorkTree()); + } + FS.DETECTED.runProcess(builder, null, null, (String) null); + + return null; + } + + /** + * Set the repository to install LFS for + * + * @param repo + * the repository to install LFS into locally instead of the user + * configuration + * @return this command + */ + @Override + public LfsInstallCommand setRepository(Repository repo) { + this.repository = repo; + return this; + } + +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/Lfs.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/Lfs.java new file mode 100644 index 0000000000..d2e9e02b42 --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/Lfs.java @@ -0,0 +1,107 @@ +/* + * Copyright (C) 2016, Christian Halstrick <christian.halstrick@sap.com> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lfs; + +import static org.eclipse.jgit.lib.Constants.OBJECTS; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.eclipse.jgit.lfs.lib.AnyLongObjectId; +import org.eclipse.jgit.lfs.lib.Constants; +import org.eclipse.jgit.lib.Repository; + +/** + * Class which represents the lfs folder hierarchy inside a {@code .git} folder + * + * @since 4.6 + */ +public class Lfs { + private Path root; + + private Path objDir; + + private Path tmpDir; + + /** + * Constructor for Lfs. + * + * @param db + * the associated repo + * + * @since 4.11 + */ + public Lfs(Repository db) { + this.root = db.getDirectory().toPath().resolve(Constants.LFS); + } + + /** + * Get the LFS root directory + * + * @return the path to the LFS directory + */ + public Path getLfsRoot() { + return root; + } + + /** + * Get the path to the temporary directory used by LFS. + * + * @return the path to the temporary directory used by LFS. Will be + * {@code <repo>/.git/lfs/tmp} + */ + public Path getLfsTmpDir() { + if (tmpDir == null) { + tmpDir = root.resolve("tmp"); //$NON-NLS-1$ + } + return tmpDir; + } + + /** + * Get the object directory used by LFS + * + * @return the path to the object directory used by LFS. Will be + * {@code <repo>/.git/lfs/objects} + */ + public Path getLfsObjDir() { + if (objDir == null) { + objDir = root.resolve(OBJECTS); + } + return objDir; + } + + /** + * Get the media file which stores the original content + * + * @param id + * the id of the mediafile + * @return the file which stores the original content. Its path will look + * like + * {@code "<repo>/.git/lfs/objects/<firstTwoLettersOfID>/<remainingLettersOfID>"} + */ + public Path getMediaFile(AnyLongObjectId id) { + String idStr = id.name(); + return getLfsObjDir().resolve(idStr.substring(0, 2)) + .resolve(idStr.substring(2, 4)).resolve(idStr); + } + + /** + * Create a new temp file in the LFS directory + * + * @return a new temporary file in the LFS directory + * @throws java.io.IOException + * when the temp file could not be created + */ + public Path createTmpFile() throws IOException { + return Files.createTempFile(getLfsTmpDir(), null, null); + } + +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsBlobFilter.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsBlobFilter.java new file mode 100644 index 0000000000..032a19b5df --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsBlobFilter.java @@ -0,0 +1,97 @@ +/* + * Copyright (C) 2017, 2021 Markus Duft <markus.duft@ssi-schaefer.com> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lfs; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.eclipse.jgit.lfs.lib.AnyLongObjectId; +import org.eclipse.jgit.lib.ObjectLoader; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.util.TemporaryBuffer; +import org.eclipse.jgit.util.TemporaryBuffer.LocalFile; + +/** + * Provides transparently either a stream to the blob or a LFS media file if + * managed by LFS. + * + * @since 4.11 + */ +public class LfsBlobFilter { + + /** + * In case the given {@link ObjectLoader} points to a LFS pointer file + * replace the loader with one pointing to the LFS media file contents. + * Missing LFS files are downloaded on the fly - same logic as the smudge + * filter. + * + * @param db + * the repo + * @param loader + * the loader for the blob + * @return either the original loader, or a loader for the LFS media file if + * managed by LFS. Files are downloaded on demand if required. + * @throws IOException + * in case of an error + */ + public static ObjectLoader smudgeLfsBlob(Repository db, ObjectLoader loader) + throws IOException { + if (loader.getSize() > LfsPointer.FULL_SIZE_THRESHOLD) { + return loader; + } + + try (InputStream is = loader.openStream()) { + LfsPointer ptr = LfsPointer.parseLfsPointer(is); + if (ptr != null) { + Lfs lfs = new Lfs(db); + AnyLongObjectId oid = ptr.getOid(); + Path mediaFile = lfs.getMediaFile(oid); + if (!Files.exists(mediaFile)) { + SmudgeFilter.downloadLfsResource(lfs, db, ptr); + } + + return new LfsBlobLoader(mediaFile); + } + } + + return loader; + } + + /** + * Run the LFS clean filter on the given stream and return a stream to the + * LFS pointer file buffer. Used when inserting objects. + * + * @param db + * the {@link Repository} + * @param originalContent + * the {@link InputStream} to the original content + * @return a {@link TemporaryBuffer} representing the LFS pointer. The + * caller is responsible to destroy the buffer. + * @throws IOException + * in case of any error. + */ + public static TemporaryBuffer cleanLfsBlob(Repository db, + InputStream originalContent) throws IOException { + LocalFile buffer = new TemporaryBuffer.LocalFile(null); + CleanFilter f = new CleanFilter(db, originalContent, buffer); + try { + while (f.run() != -1) { + // loop as long as f.run() tells there is work to do + } + } catch (IOException e) { + buffer.destroy(); + throw e; + } + return buffer; + } + +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsBlobLoader.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsBlobLoader.java new file mode 100644 index 0000000000..404c17fcab --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsBlobLoader.java @@ -0,0 +1,86 @@ +/* + * Copyright (C) 2017, Markus Duft <markus.duft@ssi-schaefer.com> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lfs; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; + +import org.eclipse.jgit.errors.LargeObjectException; +import org.eclipse.jgit.errors.MissingObjectException; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.ObjectLoader; +import org.eclipse.jgit.lib.ObjectStream; +import org.eclipse.jgit.storage.pack.PackConfig; +import org.eclipse.jgit.util.IO; + +/** + * An {@link ObjectLoader} implementation that reads a media file from the LFS + * storage. + * + * @since 4.11 + */ +public class LfsBlobLoader extends ObjectLoader { + + private Path mediaFile; + + private BasicFileAttributes attributes; + + private byte[] cached; + + /** + * Create a loader for the LFS media file at the given path. + * + * @param mediaFile + * path to the file + * @throws IOException + * in case of an error reading attributes + */ + public LfsBlobLoader(Path mediaFile) throws IOException { + this.mediaFile = mediaFile; + this.attributes = Files.readAttributes(mediaFile, + BasicFileAttributes.class); + } + + @Override + public int getType() { + return Constants.OBJ_BLOB; + } + + @Override + public long getSize() { + return attributes.size(); + } + + @Override + public byte[] getCachedBytes() throws LargeObjectException { + if (getSize() > PackConfig.DEFAULT_BIG_FILE_THRESHOLD) { + throw new LargeObjectException(); + } + + if (cached == null) { + try { + cached = IO.readFully(mediaFile.toFile()); + } catch (IOException ioe) { + throw new LargeObjectException(ioe); + } + } + return cached; + } + + @Override + public ObjectStream openStream() + throws MissingObjectException, IOException { + return new ObjectStream.Filter(getType(), getSize(), + Files.newInputStream(mediaFile)); + } + +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsPointer.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsPointer.java new file mode 100644 index 0000000000..72aad9b0c9 --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsPointer.java @@ -0,0 +1,304 @@ +/* + * Copyright (C) 2016, 2021 Christian Halstrick <christian.halstrick@sap.com> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lfs; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import java.io.BufferedInputStream; +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.PrintStream; +import java.io.UnsupportedEncodingException; +import java.util.Locale; +import java.util.Objects; + +import org.eclipse.jgit.annotations.Nullable; +import org.eclipse.jgit.lfs.lib.AnyLongObjectId; +import org.eclipse.jgit.lfs.lib.Constants; +import org.eclipse.jgit.lfs.lib.LongObjectId; +import org.eclipse.jgit.util.IO; + +/** + * Represents an LFS pointer file + * + * @since 4.6 + */ +public class LfsPointer implements Comparable<LfsPointer> { + /** + * The version of the LfsPointer file format + */ + public static final String VERSION = "https://git-lfs.github.com/spec/v1"; //$NON-NLS-1$ + + /** + * The version of the LfsPointer file format using legacy URL + * @since 4.7 + */ + public static final String VERSION_LEGACY = "https://hawser.github.com/spec/v1"; //$NON-NLS-1$ + + /** + * Don't inspect files that are larger than this threshold to avoid + * excessive reading. No pointer file should be larger than this. + * @since 4.11 + */ + public static final int SIZE_THRESHOLD = 200; + + /** + * The name of the hash function as used in the pointer files. This will + * evaluate to "sha256" + */ + public static final String HASH_FUNCTION_NAME = Constants.LONG_HASH_FUNCTION + .toLowerCase(Locale.ROOT).replace("-", ""); //$NON-NLS-1$ //$NON-NLS-2$ + + /** + * {@link #SIZE_THRESHOLD} is too low; with lfs extensions a LFS pointer can + * be larger. But 8kB should be more than enough. + */ + static final int FULL_SIZE_THRESHOLD = 8 * 1024; + + private final AnyLongObjectId oid; + + private final long size; + + /** + * <p>Constructor for LfsPointer.</p> + * + * @param oid + * the id of the content + * @param size + * the size of the content + */ + public LfsPointer(AnyLongObjectId oid, long size) { + this.oid = oid; + this.size = size; + } + + /** + * <p>Getter for the field <code>oid</code>.</p> + * + * @return the id of the content + */ + public AnyLongObjectId getOid() { + return oid; + } + + /** + * <p>Getter for the field <code>size</code>.</p> + * + * @return the size of the content + */ + public long getSize() { + return size; + } + + /** + * Encode this object into the LFS format defined by {@link #VERSION} + * + * @param out + * the {@link java.io.OutputStream} into which the encoded data should be + * written + */ + public void encode(OutputStream out) { + try (PrintStream ps = new PrintStream(out, false, + UTF_8.name())) { + ps.print("version "); //$NON-NLS-1$ + ps.print(VERSION + "\n"); //$NON-NLS-1$ + ps.print("oid " + HASH_FUNCTION_NAME + ":"); //$NON-NLS-1$ //$NON-NLS-2$ + ps.print(oid.name() + "\n"); //$NON-NLS-1$ + ps.print("size "); //$NON-NLS-1$ + ps.print(size + "\n"); //$NON-NLS-1$ + } catch (UnsupportedEncodingException e) { + // should not happen, we are using a standard charset + } + } + + /** + * Try to parse the data provided by an InputStream to the format defined by + * {@link #VERSION}. If the given stream supports mark and reset as + * indicated by {@link InputStream#markSupported()}, its input position will + * be reset if the stream content is not actually a LFS pointer (i.e., when + * {@code null} is returned). If the stream content is an invalid LFS + * pointer or the given stream does not support mark/reset, the input + * position may not be reset. + * + * @param in + * the {@link java.io.InputStream} from where to read the data + * @return an {@link org.eclipse.jgit.lfs.LfsPointer} or {@code null} if the + * stream was not parseable as LfsPointer + * @throws java.io.IOException + * if an IO error occurred + */ + @Nullable + public static LfsPointer parseLfsPointer(InputStream in) + throws IOException { + if (in.markSupported()) { + return parse(in); + } + // Fallback; note that while parse() resets its input stream, that won't + // reset "in". + return parse(new BufferedInputStream(in)); + } + + @Nullable + private static LfsPointer parse(InputStream in) + throws IOException { + if (!in.markSupported()) { + // No translation; internal error + throw new IllegalArgumentException( + "LFS pointer parsing needs InputStream.markSupported() == true"); //$NON-NLS-1$ + } + // Try reading only a short block first. + in.mark(SIZE_THRESHOLD); + byte[] preamble = new byte[SIZE_THRESHOLD]; + int length = IO.readFully(in, preamble, 0); + if (length < preamble.length || in.read() < 0) { + // We have the whole file. Try to parse a pointer from it. + try (BufferedReader r = new BufferedReader(new InputStreamReader( + new ByteArrayInputStream(preamble, 0, length), UTF_8))) { + LfsPointer ptr = parse(r); + if (ptr == null) { + in.reset(); + } + return ptr; + } + } + // Longer than SIZE_THRESHOLD: expect "version" to be the first line. + boolean hasVersion = checkVersion(preamble); + in.reset(); + if (!hasVersion) { + return null; + } + in.mark(FULL_SIZE_THRESHOLD); + byte[] fullPointer = new byte[FULL_SIZE_THRESHOLD]; + length = IO.readFully(in, fullPointer, 0); + if (length == fullPointer.length && in.read() >= 0) { + in.reset(); + return null; // Too long. + } + try (BufferedReader r = new BufferedReader(new InputStreamReader( + new ByteArrayInputStream(fullPointer, 0, length), UTF_8))) { + LfsPointer ptr = parse(r); + if (ptr == null) { + in.reset(); + } + return ptr; + } + } + + private static LfsPointer parse(BufferedReader r) throws IOException { + boolean versionLine = false; + LongObjectId id = null; + long sz = -1; + // This parsing is a bit too general if we go by the spec at + // https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md + // Comment lines are not mentioned in the spec, the "version" line + // MUST be the first, and keys are ordered alphabetically. + for (String s = r.readLine(); s != null; s = r.readLine()) { + if (s.startsWith("#") || s.length() == 0) { //$NON-NLS-1$ + continue; + } else if (s.startsWith("version")) { //$NON-NLS-1$ + if (versionLine || !checkVersionLine(s)) { + return null; // Not a LFS pointer + } + versionLine = true; + } else { + try { + if (s.startsWith("oid sha256:")) { //$NON-NLS-1$ + if (id != null) { + return null; // Not a LFS pointer + } + id = LongObjectId.fromString(s.substring(11).trim()); + } else if (s.startsWith("size")) { //$NON-NLS-1$ + if (sz > 0 || s.length() < 5 || s.charAt(4) != ' ') { + return null; // Not a LFS pointer + } + sz = Long.parseLong(s.substring(5).trim()); + } + } catch (RuntimeException e) { + // We could not parse the line. If we have a version + // already, this is a corrupt LFS pointer. Otherwise it + // is just not an LFS pointer. + if (versionLine) { + throw e; + } + return null; + } + } + if (versionLine && id != null && sz > -1) { + return new LfsPointer(id, sz); + } + } + return null; + } + + private static boolean checkVersion(byte[] data) { + // According to the spec at + // https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md + // it MUST always be the first line. + try (BufferedReader r = new BufferedReader( + new InputStreamReader(new ByteArrayInputStream(data), UTF_8))) { + String s = r.readLine(); + if (s != null && s.startsWith("version")) { //$NON-NLS-1$ + return checkVersionLine(s); + } + } catch (IOException e) { + // Doesn't occur, we're reading from a byte array! + } + return false; + } + + private static boolean checkVersionLine(String s) { + if (s.length() < 8 || s.charAt(7) != ' ') { + return false; // Not a valid LFS pointer version line + } + String rest = s.substring(8).trim(); + return VERSION.equals(rest) || VERSION_LEGACY.equals(rest); + } + + @Override + public String toString() { + return "LfsPointer: oid=" + oid.name() + ", size=" //$NON-NLS-1$ //$NON-NLS-2$ + + size; + } + + /** + * @since 4.11 + */ + @Override + public int compareTo(LfsPointer o) { + int x = getOid().compareTo(o.getOid()); + if (x != 0) { + return x; + } + + return Long.compare(getSize(), o.getSize()); + } + + @Override + public int hashCode() { + return Objects.hash(getOid()) * 31 + Long.hashCode(getSize()); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof LfsPointer)) { + return false; + } + LfsPointer other = (LfsPointer) obj; + return Objects.equals(getOid(), other.getOid()) + && getSize() == other.getSize(); + } +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsPrePushHook.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsPrePushHook.java new file mode 100644 index 0000000000..802835cadd --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/LfsPrePushHook.java @@ -0,0 +1,264 @@ +/* + * Copyright (C) 2017, 2022 Markus Duft <markus.duft@ssi-schaefer.com> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lfs; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.eclipse.jgit.lfs.Protocol.OPERATION_UPLOAD; +import static org.eclipse.jgit.lfs.internal.LfsConnectionFactory.toRequest; +import static org.eclipse.jgit.transport.http.HttpConnection.HTTP_OK; +import static org.eclipse.jgit.util.HttpSupport.METHOD_POST; +import static org.eclipse.jgit.util.HttpSupport.METHOD_PUT; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.PrintStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.text.MessageFormat; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; + +import org.eclipse.jgit.api.errors.AbortedByHookException; +import org.eclipse.jgit.errors.IncorrectObjectTypeException; +import org.eclipse.jgit.errors.MissingObjectException; +import org.eclipse.jgit.hooks.PrePushHook; +import org.eclipse.jgit.lfs.Protocol.ObjectInfo; +import org.eclipse.jgit.lfs.errors.CorruptMediaFile; +import org.eclipse.jgit.lfs.internal.LfsConnectionFactory; +import org.eclipse.jgit.lfs.internal.LfsText; +import org.eclipse.jgit.lib.AnyObjectId; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectReader; +import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.lib.RefDatabase; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.revwalk.ObjectWalk; +import org.eclipse.jgit.revwalk.RevObject; +import org.eclipse.jgit.transport.RemoteRefUpdate; +import org.eclipse.jgit.transport.http.HttpConnection; + +import com.google.gson.Gson; +import com.google.gson.stream.JsonReader; + +/** + * Pre-push hook that handles uploading LFS artefacts. + * + * @since 4.11 + */ +public class LfsPrePushHook extends PrePushHook { + + private static final String EMPTY = ""; //$NON-NLS-1$ + private Collection<RemoteRefUpdate> refs; + + /** + * @param repo + * the repository + * @param outputStream + * not used by this implementation + */ + public LfsPrePushHook(Repository repo, PrintStream outputStream) { + super(repo, outputStream); + } + + /** + * @param repo + * the repository + * @param outputStream + * not used by this implementation + * @param errorStream + * not used by this implementation + * @since 5.6 + */ + public LfsPrePushHook(Repository repo, PrintStream outputStream, + PrintStream errorStream) { + super(repo, outputStream, errorStream); + } + + @Override + public void setRefs(Collection<RemoteRefUpdate> toRefs) { + this.refs = toRefs; + } + + @Override + public String call() throws IOException, AbortedByHookException { + Set<LfsPointer> toPush = findObjectsToPush(); + if (toPush.isEmpty()) { + return EMPTY; + } + HttpConnection api = LfsConnectionFactory.getLfsConnection( + getRepository(), METHOD_POST, OPERATION_UPLOAD); + if (!isDryRun()) { + Map<String, LfsPointer> oid2ptr = requestBatchUpload(api, toPush); + uploadContents(api, oid2ptr); + } + return EMPTY; + + } + + private Set<LfsPointer> findObjectsToPush() throws IOException, + MissingObjectException, IncorrectObjectTypeException { + Set<LfsPointer> toPush = new TreeSet<>(); + + try (ObjectWalk walk = new ObjectWalk(getRepository())) { + for (RemoteRefUpdate up : refs) { + if (up.isDelete()) { + continue; + } + walk.setRewriteParents(false); + excludeRemoteRefs(walk); + walk.markStart(walk.parseCommit(up.getNewObjectId())); + while (walk.next() != null) { + // walk all commits to populate objects + } + findLfsPointers(toPush, walk); + } + } + return toPush; + } + + private static void findLfsPointers(Set<LfsPointer> toPush, ObjectWalk walk) + throws MissingObjectException, IncorrectObjectTypeException, + IOException { + RevObject obj; + ObjectReader r = walk.getObjectReader(); + while ((obj = walk.nextObject()) != null) { + if (obj.getType() == Constants.OBJ_BLOB + && getObjectSize(r, obj) < LfsPointer.SIZE_THRESHOLD) { + LfsPointer ptr = loadLfsPointer(r, obj); + if (ptr != null) { + toPush.add(ptr); + } + } + } + } + + private static long getObjectSize(ObjectReader r, RevObject obj) + throws IOException { + return r.getObjectSize(obj.getId(), Constants.OBJ_BLOB); + } + + private static LfsPointer loadLfsPointer(ObjectReader r, AnyObjectId obj) + throws IOException { + try (InputStream is = r.open(obj, Constants.OBJ_BLOB).openStream()) { + return LfsPointer.parseLfsPointer(is); + } + } + + private void excludeRemoteRefs(ObjectWalk walk) throws IOException { + RefDatabase refDatabase = getRepository().getRefDatabase(); + List<Ref> remoteRefs = refDatabase.getRefsByPrefix(remote()); + for (Ref r : remoteRefs) { + ObjectId oid = r.getPeeledObjectId(); + if (oid == null) { + oid = r.getObjectId(); + } + if (oid == null) { + // ignore (e.g. symbolic, ...) + continue; + } + RevObject o = walk.parseAny(oid); + if (o.getType() == Constants.OBJ_COMMIT + || o.getType() == Constants.OBJ_TAG) { + walk.markUninteresting(o); + } + } + } + + private String remote() { + String remoteName = getRemoteName() == null + ? Constants.DEFAULT_REMOTE_NAME + : getRemoteName(); + return Constants.R_REMOTES + remoteName; + } + + private Map<String, LfsPointer> requestBatchUpload(HttpConnection api, + Set<LfsPointer> toPush) throws IOException { + LfsPointer[] res = toPush.toArray(new LfsPointer[0]); + Map<String, LfsPointer> oidStr2ptr = new HashMap<>(); + for (LfsPointer p : res) { + oidStr2ptr.put(p.getOid().name(), p); + } + Gson gson = Protocol.gson(); + api.getOutputStream().write( + gson.toJson(toRequest(OPERATION_UPLOAD, res)).getBytes(UTF_8)); + int responseCode = api.getResponseCode(); + if (responseCode != HTTP_OK) { + throw new IOException( + MessageFormat.format(LfsText.get().serverFailure, + api.getURL(), Integer.valueOf(responseCode))); + } + return oidStr2ptr; + } + + private void uploadContents(HttpConnection api, + Map<String, LfsPointer> oid2ptr) throws IOException { + try (JsonReader reader = new JsonReader( + new InputStreamReader(api.getInputStream(), UTF_8))) { + for (Protocol.ObjectInfo o : parseObjects(reader)) { + if (o.actions == null) { + continue; + } + LfsPointer ptr = oid2ptr.get(o.oid); + if (ptr == null) { + // received an object we didn't request + continue; + } + Protocol.Action uploadAction = o.actions.get(OPERATION_UPLOAD); + if (uploadAction == null || uploadAction.href == null) { + continue; + } + + Lfs lfs = new Lfs(getRepository()); + Path path = lfs.getMediaFile(ptr.getOid()); + if (!Files.exists(path)) { + throw new IOException(MessageFormat + .format(LfsText.get().missingLocalObject, path)); + } + uploadFile(o, uploadAction, path); + } + } + } + + private List<ObjectInfo> parseObjects(JsonReader reader) { + Gson gson = new Gson(); + Protocol.Response resp = gson.fromJson(reader, Protocol.Response.class); + return resp.objects; + } + + private void uploadFile(Protocol.ObjectInfo o, + Protocol.Action uploadAction, Path path) + throws IOException, CorruptMediaFile { + HttpConnection contentServer = LfsConnectionFactory + .getLfsContentConnection(getRepository(), uploadAction, + METHOD_PUT); + contentServer.setDoOutput(true); + contentServer.setChunkedStreamingMode(256 << 10); + try (OutputStream out = contentServer + .getOutputStream()) { + long size = Files.copy(path, out); + if (size != o.size) { + throw new CorruptMediaFile(path, o.size, size); + } + } + int responseCode = contentServer.getResponseCode(); + if (responseCode != HTTP_OK) { + throw new IOException(MessageFormat.format( + LfsText.get().serverFailure, contentServer.getURL(), + Integer.valueOf(responseCode))); + } + } +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/Protocol.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/Protocol.java new file mode 100644 index 0000000000..bcfd6e6576 --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/Protocol.java @@ -0,0 +1,147 @@ +/* + * Copyright (C) 2016, Christian Halstrick <christian.halstrick@sap.com> + * Copyright (C) 2015, Sasa Zivkov <sasa.zivkov@sap.com> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lfs; + +import java.util.List; +import java.util.Map; + +import com.google.gson.FieldNamingPolicy; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; + +/** + * This interface describes the network protocol used between lfs client and lfs + * server + * + * @since 4.11 + */ +public interface Protocol { + /** A request sent to an LFS server */ + class Request { + /** The operation of this request */ + public String operation; + + /** The objects of this request */ + public List<ObjectSpec> objects; + } + + /** A response received from an LFS server */ + class Response { + public List<ObjectInfo> objects; + } + + /** + * MetaData of an LFS object. Needs to be specified when requesting objects + * from the LFS server and is also returned in the response + */ + class ObjectSpec { + public String oid; // the objectid + + public long size; // the size of the object + } + + /** + * Describes in a response all actions the LFS server offers for a single + * object + */ + class ObjectInfo extends ObjectSpec { + public Map<String, Action> actions; // Maps operation to action + + public Error error; + } + + /** + * Describes in a Response a single action the client can execute on a + * single object + */ + class Action { + public String href; + + public Map<String, String> header; + } + + /** + * An action with an additional expiration timestamp + * + * @since 4.11 + */ + class ExpiringAction extends Action { + /** + * Absolute date/time in format "yyyy-MM-dd'T'HH:mm:ss.SSSX" + */ + public String expiresAt; + + /** + * Validity time in milliseconds (preferred over expiresAt as specified: + * https://github.com/git-lfs/git-lfs/blob/master/docs/api/authentication.md) + */ + public String expiresIn; + } + + /** Describes an error to be returned by the LFS batch API */ + // TODO(ms): rename this class in next major release + @SuppressWarnings("JavaLangClash") + class Error { + public int code; + + public String message; + } + + /** + * The "download" operation + */ + String OPERATION_DOWNLOAD = "download"; //$NON-NLS-1$ + + /** + * The "upload" operation + */ + String OPERATION_UPLOAD = "upload"; //$NON-NLS-1$ + + /** + * The contenttype used in LFS requests + */ + String CONTENTTYPE_VND_GIT_LFS_JSON = "application/vnd.git-lfs+json; charset=utf-8"; //$NON-NLS-1$ + + /** + * Authorization header when auto-discovering via SSH. + */ + String HDR_AUTH = "Authorization"; //$NON-NLS-1$ + + /** + * Prefix of authentication token obtained through SSH. + */ + String HDR_AUTH_SSH_PREFIX = "Ssh: "; //$NON-NLS-1$ + + /** + * Path to the LFS info servlet. + */ + String INFO_LFS_ENDPOINT = "/info/lfs"; //$NON-NLS-1$ + + /** + * Path to the LFS objects servlet. + */ + String OBJECTS_LFS_ENDPOINT = "/objects/batch"; //$NON-NLS-1$ + + /** + * Gson instance for handling this protocol + * + * @return a {@link Gson} instance suitable for handling this + * {@link Protocol} + * + * @since 4.11 + */ + public static Gson gson() { + return new GsonBuilder() + .setFieldNamingPolicy( + FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES) + .disableHtmlEscaping().create(); + } +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/SmudgeFilter.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/SmudgeFilter.java new file mode 100644 index 0000000000..b6515b92e1 --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/SmudgeFilter.java @@ -0,0 +1,258 @@ +/* + * Copyright (C) 2016, 2021 Christian Halstrick <christian.halstrick@sap.com> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lfs; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import java.io.BufferedInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + +import org.eclipse.jgit.attributes.FilterCommand; +import org.eclipse.jgit.attributes.FilterCommandFactory; +import org.eclipse.jgit.attributes.FilterCommandRegistry; +import org.eclipse.jgit.lfs.internal.LfsConnectionFactory; +import org.eclipse.jgit.lfs.internal.LfsText; +import org.eclipse.jgit.lfs.lib.AnyLongObjectId; +import org.eclipse.jgit.lfs.lib.Constants; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.transport.http.HttpConnection; +import org.eclipse.jgit.util.HttpSupport; + +import com.google.gson.Gson; +import com.google.gson.stream.JsonReader; + +/** + * Built-in LFS smudge filter + * + * When content is read from git's object-database and written to the filesystem + * and this filter is configured for that content, then this filter will replace + * the content of LFS pointer files with the original content. This happens e.g. + * when a checkout needs to update a working tree file which is under LFS + * control. + * + * @since 4.6 + */ +public class SmudgeFilter extends FilterCommand { + + /** + * Max number of bytes to copy in a single {@link #run()} call. + */ + private static final int MAX_COPY_BYTES = 1024 * 1024 * 256; + + /** + * The factory is responsible for creating instances of + * {@link org.eclipse.jgit.lfs.SmudgeFilter} + */ + public static final FilterCommandFactory FACTORY = SmudgeFilter::new; + + /** + * Register this filter in JGit + */ + static void register() { + FilterCommandRegistry + .register(org.eclipse.jgit.lib.Constants.BUILTIN_FILTER_PREFIX + + Constants.ATTR_FILTER_DRIVER_PREFIX + + org.eclipse.jgit.lib.Constants.ATTR_FILTER_TYPE_SMUDGE, + FACTORY); + } + + /** + * Constructor for SmudgeFilter. + * + * @param db + * a {@link org.eclipse.jgit.lib.Repository} object. + * @param in + * a {@link java.io.InputStream} object. The stream is closed in + * any case. + * @param out + * a {@link java.io.OutputStream} object. + * @throws java.io.IOException + * in case of an error + */ + public SmudgeFilter(Repository db, InputStream in, OutputStream out) + throws IOException { + this(in.markSupported() ? in : new BufferedInputStream(in), out, db); + } + + private SmudgeFilter(InputStream in, OutputStream out, Repository db) + throws IOException { + super(in, out); + InputStream from = in; + try { + LfsPointer res = LfsPointer.parseLfsPointer(from); + if (res != null) { + AnyLongObjectId oid = res.getOid(); + Lfs lfs = new Lfs(db); + Path mediaFile = lfs.getMediaFile(oid); + if (!Files.exists(mediaFile)) { + downloadLfsResource(lfs, db, res); + } + this.in = Files.newInputStream(mediaFile); + } else { + // Not swapped; stream was reset, don't close! + from = null; + } + } finally { + if (from != null) { + from.close(); // Close the swapped-out stream + } + } + } + + /** + * Download content which is hosted on a LFS server + * + * @param lfs + * local {@link Lfs} storage. + * @param db + * the repository to work with + * @param res + * the objects to download + * @return the paths of all mediafiles which have been downloaded + * @throws IOException + * if an IO error occurred + * @since 4.11 + */ + public static Collection<Path> downloadLfsResource(Lfs lfs, Repository db, + LfsPointer... res) throws IOException { + Collection<Path> downloadedPaths = new ArrayList<>(); + Map<String, LfsPointer> oidStr2ptr = new HashMap<>(); + for (LfsPointer p : res) { + oidStr2ptr.put(p.getOid().name(), p); + } + HttpConnection lfsServerConn = LfsConnectionFactory.getLfsConnection(db, + HttpSupport.METHOD_POST, Protocol.OPERATION_DOWNLOAD); + Gson gson = Protocol.gson(); + lfsServerConn.getOutputStream() + .write(gson + .toJson(LfsConnectionFactory + .toRequest(Protocol.OPERATION_DOWNLOAD, res)) + .getBytes(UTF_8)); + int responseCode = lfsServerConn.getResponseCode(); + if (!(responseCode == HttpConnection.HTTP_OK + || responseCode == HttpConnection.HTTP_NOT_AUTHORITATIVE)) { + throw new IOException( + MessageFormat.format(LfsText.get().serverFailure, + lfsServerConn.getURL(), + Integer.valueOf(responseCode))); + } + try (JsonReader reader = new JsonReader( + new InputStreamReader(lfsServerConn.getInputStream(), + UTF_8))) { + Protocol.Response resp = gson.fromJson(reader, + Protocol.Response.class); + for (Protocol.ObjectInfo o : resp.objects) { + if (o.error != null) { + throw new IOException( + MessageFormat.format(LfsText.get().protocolError, + Integer.valueOf(o.error.code), + o.error.message)); + } + if (o.actions == null) { + continue; + } + LfsPointer ptr = oidStr2ptr.get(o.oid); + if (ptr == null) { + // received an object we didn't request + continue; + } + if (ptr.getSize() != o.size) { + throw new IOException(MessageFormat.format( + LfsText.get().inconsistentContentLength, + lfsServerConn.getURL(), Long.valueOf(ptr.getSize()), + Long.valueOf(o.size))); + } + Protocol.Action downloadAction = o.actions + .get(Protocol.OPERATION_DOWNLOAD); + if (downloadAction == null || downloadAction.href == null) { + continue; + } + + HttpConnection contentServerConn = LfsConnectionFactory + .getLfsContentConnection(db, downloadAction, + HttpSupport.METHOD_GET); + + responseCode = contentServerConn.getResponseCode(); + if (responseCode != HttpConnection.HTTP_OK) { + throw new IOException( + MessageFormat.format(LfsText.get().serverFailure, + contentServerConn.getURL(), + Integer.valueOf(responseCode))); + } + Path path = lfs.getMediaFile(ptr.getOid()); + Path parent = path.getParent(); + if (parent != null) { + parent.toFile().mkdirs(); + } + try (InputStream contentIn = contentServerConn + .getInputStream()) { + long bytesCopied = Files.copy(contentIn, path); + if (bytesCopied != o.size) { + throw new IOException(MessageFormat.format( + LfsText.get().wrongAmountOfDataReceived, + contentServerConn.getURL(), + Long.valueOf(bytesCopied), + Long.valueOf(o.size))); + } + downloadedPaths.add(path); + } + } + } + return downloadedPaths; + } + + @Override + public int run() throws IOException { + try { + int totalRead = 0; + int length = 0; + if (in != null) { + byte[] buf = new byte[8192]; + while ((length = in.read(buf)) != -1) { + out.write(buf, 0, length); + totalRead += length; + + // when threshold reached, loop back to the caller. + // otherwise we could only support files up to 2GB (int + // return type) properly. we will be called again as long as + // we don't return -1 here. + if (totalRead >= MAX_COPY_BYTES) { + // leave streams open - we need them in the next call. + return totalRead; + } + } + } + + if (totalRead == 0 && length == -1) { + // we're totally done :) cleanup all streams + in.close(); + out.close(); + return length; + } + + return totalRead; + } catch (IOException e) { + in.close(); // clean up - we swapped this stream. + out.close(); + throw e; + } + } + +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/CorruptLongObjectException.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/CorruptLongObjectException.java new file mode 100644 index 0000000000..801df4e265 --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/CorruptLongObjectException.java @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2015, Matthias Sohn <matthias.sohn@sap.com> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.lfs.errors; + +import org.eclipse.jgit.lfs.lib.AnyLongObjectId; + +/** + * Thrown when an object id is given that doesn't match the hash of the object's + * content + * + * @since 4.3 + */ +public class CorruptLongObjectException extends IllegalArgumentException { + + private static final long serialVersionUID = 1L; + + private final AnyLongObjectId id; + + private final AnyLongObjectId contentHash; + + /** + * Corrupt long object detected. + * + * @param id + * id of the long object + * @param contentHash + * hash of the long object's content + * @param message a {@link java.lang.String} object. + */ + public CorruptLongObjectException(AnyLongObjectId id, + AnyLongObjectId contentHash, + String message) { + super(message); + this.id = id; + this.contentHash = contentHash; + } + + /** + * Get the <code>id</code> of the object. + * + * @return the id of the object, i.e. the expected hash of the object's + * content + */ + public AnyLongObjectId getId() { + return id; + } + + /** + * Get the <code>contentHash</code>. + * + * @return the actual hash of the object's content which doesn't match the + * object's id when this exception is thrown which signals that the + * object has been corrupted + */ + public AnyLongObjectId getContentHash() { + return contentHash; + } +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/CorruptMediaFile.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/CorruptMediaFile.java new file mode 100644 index 0000000000..7130c6e761 --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/CorruptMediaFile.java @@ -0,0 +1,75 @@ +/* + * Copyright (C) 2016, Christian Halstrick <christian.halstrick@sap.com> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lfs.errors; + +import java.io.IOException; +import java.nio.file.Path; +import java.text.MessageFormat; + +import org.eclipse.jgit.lfs.internal.LfsText; + +/** + * Thrown when a LFS mediafile is found which doesn't have the expected size + * + * @since 4.6 + */ +public class CorruptMediaFile extends IOException { + private static final long serialVersionUID = 1L; + + private Path mediaFile; + + private long expectedSize; + + private long size; + + /** + * <p>Constructor for CorruptMediaFile.</p> + * + * @param mediaFile a {@link java.nio.file.Path} object. + * @param expectedSize a long. + * @param size a long. + */ + @SuppressWarnings("boxing") + public CorruptMediaFile(Path mediaFile, long expectedSize, + long size) { + super(MessageFormat.format(LfsText.get().inconsistentMediafileLength, + mediaFile, expectedSize, size)); + this.mediaFile = mediaFile; + this.expectedSize = expectedSize; + this.size = size; + } + + /** + * Get the <code>mediaFile</code>. + * + * @return the media file which seems to be corrupt + */ + public Path getMediaFile() { + return mediaFile; + } + + /** + * Get the <code>expectedSize</code>. + * + * @return the expected size of the media file + */ + public long getExpectedSize() { + return expectedSize; + } + + /** + * Get the <code>size</code>. + * + * @return the actual size of the media file in the file system + */ + public long getSize() { + return size; + } +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/InvalidLongObjectIdException.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/InvalidLongObjectIdException.java new file mode 100644 index 0000000000..7b8b1a3ef3 --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/InvalidLongObjectIdException.java @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2009, Jonas Fonseca <fonseca@diku.dk> + * Copyright (C) 2007, Robin Rosenberg <robin.rosenberg@dewire.com> + * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.lfs.errors; + +import static java.nio.charset.StandardCharsets.US_ASCII; + +import java.text.MessageFormat; + +import org.eclipse.jgit.lfs.internal.LfsText; + +/** + * Thrown when an invalid long object id is passed in as an argument. + * + * @since 4.3 + */ +public class InvalidLongObjectIdException extends IllegalArgumentException { + private static final long serialVersionUID = 1L; + + /** + * Create exception with bytes of the invalid object id. + * + * @param bytes containing the invalid id. + * @param offset in the byte array where the error occurred. + * @param length of the sequence of invalid bytes. + */ + public InvalidLongObjectIdException(byte[] bytes, int offset, int length) { + super(MessageFormat.format(LfsText.get().invalidLongId, + asAscii(bytes, offset, length))); + } + + /** + * <p>Constructor for InvalidLongObjectIdException.</p> + * + * @param idString + * String containing the invalid id + */ + public InvalidLongObjectIdException(String idString) { + super(MessageFormat.format(LfsText.get().invalidLongId, idString)); + } + + private static String asAscii(byte[] bytes, int offset, int length) { + try { + return new String(bytes, offset, length, US_ASCII); + } catch (StringIndexOutOfBoundsException e) { + return ""; //$NON-NLS-1$ + } + } +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/LfsBandwidthLimitExceeded.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/LfsBandwidthLimitExceeded.java new file mode 100644 index 0000000000..5afdde4097 --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/LfsBandwidthLimitExceeded.java @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2016, David Pursehouse <david.pursehouse@gmail.com> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.lfs.errors; + +/** + * Thrown when the bandwidth limit for the user or repository has been exceeded. + * + * @since 4.5 + */ +public class LfsBandwidthLimitExceeded extends LfsException { + private static final long serialVersionUID = 1L; + + /** + * <p>Constructor for LfsBandwidthLimitExceeded.</p> + * + * @param message + * error message, which may be shown to an end-user. + */ + public LfsBandwidthLimitExceeded(String message) { + super(message); + } +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/LfsConfigInvalidException.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/LfsConfigInvalidException.java new file mode 100644 index 0000000000..a520c1c0c4 --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/LfsConfigInvalidException.java @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2016, Christian Halstrick <christian.halstrick@sap.com> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lfs.errors; + +import java.io.IOException; + +/** + * Thrown when a LFS configuration problem has been detected (i.e. unable to + * find the remote LFS repository URL). + * + * @since 4.11 + */ +public class LfsConfigInvalidException extends IOException { + private static final long serialVersionUID = 1L; + + /** + * Constructor for LfsConfigInvalidException. + * + * @param msg + * the error description + */ + public LfsConfigInvalidException(String msg) { + super(msg); + } + + /** + * Constructor for LfsConfigInvalidException. + * + * @param msg + * the error description + * @param e + * cause of this exception + * @since 5.0 + */ + public LfsConfigInvalidException(String msg, Exception e) { + super(msg, e); + } + +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/LfsException.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/LfsException.java new file mode 100644 index 0000000000..6bcbf5dd8c --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/LfsException.java @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2016, David Pursehouse <david.pursehouse@gmail.com> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.lfs.errors; + +/** + * Thrown when an error occurs during LFS operation. + * + * @since 4.5 + */ +public class LfsException extends Exception { + private static final long serialVersionUID = 1L; + + /** + * <p>Constructor for LfsException.</p> + * + * @param message + * error message, which may be shown to an end-user. + */ + public LfsException(String message) { + super(message); + } +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/LfsInsufficientStorage.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/LfsInsufficientStorage.java new file mode 100644 index 0000000000..28714fa128 --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/LfsInsufficientStorage.java @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2016, David Pursehouse <david.pursehouse@gmail.com> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.lfs.errors; + +/** + * Thrown when there is insufficient storage on the server. + * + * @since 4.5 + */ +public class LfsInsufficientStorage extends LfsException { + private static final long serialVersionUID = 1L; + + /** + * <p>Constructor for LfsInsufficientStorage.</p> + * + * @param message + * error message, which may be shown to an end-user. + */ + public LfsInsufficientStorage(String message) { + super(message); + } +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/LfsRateLimitExceeded.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/LfsRateLimitExceeded.java new file mode 100644 index 0000000000..981ada3567 --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/LfsRateLimitExceeded.java @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2016, David Pursehouse <david.pursehouse@gmail.com> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.lfs.errors; + +/** + * Thrown when the user has hit a rate limit with the server. + * + * @since 4.5 + */ +public class LfsRateLimitExceeded extends LfsException { + private static final long serialVersionUID = 1L; + + /** + * <p>Constructor for LfsRateLimitExceeded.</p> + * + * @param message + * error message, which may be shown to an end-user. + */ + public LfsRateLimitExceeded(String message) { + super(message); + } +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/LfsRepositoryNotFound.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/LfsRepositoryNotFound.java new file mode 100644 index 0000000000..cdac93a0a4 --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/LfsRepositoryNotFound.java @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2016, David Pursehouse <david.pursehouse@gmail.com> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.lfs.errors; + +import java.text.MessageFormat; + +import org.eclipse.jgit.lfs.internal.LfsText; + +/** + * Thrown when the repository does not exist for the user. + * + * @since 4.5 + */ +public class LfsRepositoryNotFound extends LfsException { + private static final long serialVersionUID = 1L; + + /** + * <p>Constructor for LfsRepositoryNotFound.</p> + * + * @param name + * the repository name. + */ + public LfsRepositoryNotFound(String name) { + super(MessageFormat.format(LfsText.get().repositoryNotFound, name)); + } +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/LfsRepositoryReadOnly.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/LfsRepositoryReadOnly.java new file mode 100644 index 0000000000..1d5a8cad6e --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/LfsRepositoryReadOnly.java @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2016, David Pursehouse <david.pursehouse@gmail.com> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.lfs.errors; + +import java.text.MessageFormat; + +import org.eclipse.jgit.lfs.internal.LfsText; + +/** + * Thrown when the user has read, but not write access. Only applicable when the + * operation in the request is "upload". + * + * @since 4.5 + */ +public class LfsRepositoryReadOnly extends LfsException { + private static final long serialVersionUID = 1L; + + /** + * <p>Constructor for LfsRepositoryReadOnly.</p> + * + * @param name + * the repository name. + */ + public LfsRepositoryReadOnly(String name) { + super(MessageFormat.format(LfsText.get().repositoryReadOnly, name)); + } +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/LfsUnauthorized.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/LfsUnauthorized.java new file mode 100644 index 0000000000..0dc6aeab29 --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/LfsUnauthorized.java @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2017, David Pursehouse <david.pursehouse@gmail.com> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.lfs.errors; + +import java.text.MessageFormat; + +import org.eclipse.jgit.lfs.internal.LfsText; + +/** + * Thrown when authorization was refused for an LFS operation. + * + * @since 4.7 + */ +public class LfsUnauthorized extends LfsException { + private static final long serialVersionUID = 1L; + + /** + * <p>Constructor for LfsUnauthorized.</p> + * + * @param operation + * the operation that was attempted. + * @param name + * the repository name. + */ + public LfsUnauthorized(String operation, String name) { + super(MessageFormat.format(LfsText.get().lfsUnauthorized, operation, + name)); + } +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/LfsUnavailable.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/LfsUnavailable.java new file mode 100644 index 0000000000..b0f4c44e29 --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/LfsUnavailable.java @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2016, David Pursehouse <david.pursehouse@gmail.com> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.lfs.errors; + +import java.text.MessageFormat; + +import org.eclipse.jgit.lfs.internal.LfsText; + +/** + * Thrown when LFS is not available. + * + * @since 4.5 + */ +public class LfsUnavailable extends LfsException { + private static final long serialVersionUID = 1L; + + /** + * Constructor for LfsUnavailable. + * + * @param name + * the repository name. + */ + public LfsUnavailable(String name) { + super(MessageFormat.format(LfsText.get().lfsUnavailable, name)); + } +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/LfsValidationError.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/LfsValidationError.java new file mode 100644 index 0000000000..50afb43bc5 --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/errors/LfsValidationError.java @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2016, David Pursehouse <david.pursehouse@gmail.com> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.lfs.errors; + +/** + * Thrown when there is a validation error with one or more of the objects in + * the request. + * + * @since 4.5 + */ +public class LfsValidationError extends LfsException { + private static final long serialVersionUID = 1L; + + /** + * Constructor for LfsValidationError. + * + * @param message + * error message, which may be shown to an end-user. + */ + public LfsValidationError(String message) { + super(message); + } +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/AtomicObjectOutputStream.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/AtomicObjectOutputStream.java new file mode 100644 index 0000000000..009250294e --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/AtomicObjectOutputStream.java @@ -0,0 +1,129 @@ +/* + * Copyright (C) 2015, Matthias Sohn <matthias.sohn@sap.com> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lfs.internal; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Path; +import java.security.DigestOutputStream; +import java.text.MessageFormat; + +import org.eclipse.jgit.annotations.Nullable; +import org.eclipse.jgit.internal.storage.file.LockFile; +import org.eclipse.jgit.lfs.errors.CorruptLongObjectException; +import org.eclipse.jgit.lfs.lib.AnyLongObjectId; +import org.eclipse.jgit.lfs.lib.Constants; +import org.eclipse.jgit.lfs.lib.LongObjectId; + +/** + * Output stream writing content to a + * {@link org.eclipse.jgit.internal.storage.file.LockFile} which is committed on + * close(). The stream checks if the hash of the stream content matches the id. + */ +public class AtomicObjectOutputStream extends OutputStream { + + private LockFile locked; + + private DigestOutputStream out; + + private boolean aborted; + + private AnyLongObjectId id; + + /** + * Constructor for AtomicObjectOutputStream. + * + * @param path + * a {@link java.nio.file.Path} object. + * @param id + * a {@link org.eclipse.jgit.lfs.lib.AnyLongObjectId} object. + * @throws java.io.IOException + * if an IO error occurred + */ + public AtomicObjectOutputStream(Path path, AnyLongObjectId id) + throws IOException { + locked = new LockFile(path.toFile()); + locked.lock(); + this.id = id; + out = new DigestOutputStream(locked.getOutputStream(), + Constants.newMessageDigest()); + } + + /** + * Constructor for AtomicObjectOutputStream. + * + * @param path + * a {@link java.nio.file.Path} object. + * @throws java.io.IOException + * if an IO error occurred + */ + public AtomicObjectOutputStream(Path path) throws IOException { + this(path, null); + } + + /** + * Get the <code>id</code>. + * + * @return content hash of the object which was streamed through this + * stream. May return {@code null} if called before closing this + * stream. + */ + @Nullable + public AnyLongObjectId getId() { + return id; + } + + @Override + public void write(int b) throws IOException { + out.write(b); + } + + @Override + public void write(byte[] b) throws IOException { + out.write(b); + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + out.write(b, off, len); + } + + @Override + public void close() throws IOException { + out.close(); + if (!aborted) { + if (id != null) { + verifyHash(); + } else { + id = LongObjectId.fromRaw(out.getMessageDigest().digest()); + } + locked.commit(); + } + } + + private void verifyHash() { + AnyLongObjectId contentHash = LongObjectId + .fromRaw(out.getMessageDigest().digest()); + if (!contentHash.equals(id)) { + abort(); + throw new CorruptLongObjectException(id, contentHash, + MessageFormat.format(LfsText.get().corruptLongObject, + contentHash, id)); + } + } + + /** + * Aborts the stream. Temporary file will be deleted + */ + public void abort() { + locked.unlock(); + aborted = true; + } +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsConfig.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsConfig.java new file mode 100644 index 0000000000..0469337b1f --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsConfig.java @@ -0,0 +1,226 @@ +/* + * Copyright (C) 2022, Matthias Fromme <mfromme@dspace.de> + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lfs.internal; + +import java.io.File; +import java.io.IOException; + +import org.eclipse.jgit.annotations.Nullable; +import org.eclipse.jgit.dircache.DirCacheEntry; +import org.eclipse.jgit.errors.ConfigInvalidException; +import org.eclipse.jgit.lfs.errors.LfsConfigInvalidException; +import org.eclipse.jgit.lfs.lib.Constants; +import org.eclipse.jgit.lib.BlobBasedConfig; +import org.eclipse.jgit.lib.Config; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevTree; +import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.storage.file.FileBasedConfig; +import org.eclipse.jgit.treewalk.TreeWalk; + +import static org.eclipse.jgit.lib.Constants.HEAD; + +/** + * Encapsulate access to the {@code .lfsconfig}. + * <p> + * According to the git lfs documentation the order to find the + * {@code .lfsconfig} file is: + * </p> + * <ol> + * <li>in the root of the working tree</li> + * <li>in the index</li> + * <li>in the HEAD; for bare repositories this is the only place that is + * searched</li> + * </ol> + * <p> + * Values from the {@code .lfsconfig} are used only if not specified in another + * git config file to allow local override without modifiction of a committed + * file. + * </p> + * + * @see <a href= + * "https://github.com/git-lfs/git-lfs/blob/main/docs/man/git-lfs-config.5.ronn">Configuration + * options for git-lfs</a> + */ +public class LfsConfig { + private Repository db; + private Config delegate; + + /** + * Create a new instance of the LfsConfig. + * + * @param db + * the associated repo + */ + public LfsConfig(Repository db) { + this.db = db; + } + + /** + * Getter for the delegate to allow lazy initialization. + * + * @return the delegate {@link Config} + * @throws IOException + * if an IO error occurred + */ + private Config getDelegate() throws IOException { + if (delegate == null) { + delegate = this.load(); + } + return delegate; + } + + /** + * Read the .lfsconfig file from the repository + * + * An empty config is returned be empty if no lfs config exists. + * + * @return The loaded lfs config + * + * @throws IOException + * if an IO error occurred + */ + private Config load() throws IOException { + Config result = null; + + if (!db.isBare()) { + result = loadFromWorkingTree(); + if (result == null) { + result = loadFromIndex(); + } + } + + if (result == null) { + result = loadFromHead(); + } + + if (result == null) { + result = emptyConfig(); + } + + return result; + } + + /** + * Try to read the lfs config from a file called .lfsconfig at the top level + * of the working tree. + * + * @return the config, or <code>null</code> + * @throws IOException + * if an IO error occurred + */ + @Nullable + private Config loadFromWorkingTree() + throws IOException { + File lfsConfig = db.getFS().resolve(db.getWorkTree(), + Constants.DOT_LFS_CONFIG); + if (lfsConfig.isFile()) { + FileBasedConfig config = new FileBasedConfig(lfsConfig, db.getFS()); + try { + config.load(); + return config; + } catch (ConfigInvalidException e) { + throw new LfsConfigInvalidException( + LfsText.get().dotLfsConfigReadFailed, e); + } + } + return null; + } + + /** + * Try to read the lfs config from an entry called .lfsconfig contained in + * the index. + * + * @return the config, or <code>null</code> if the entry does not exist + * @throws IOException + * if an IO error occurred + */ + @Nullable + private Config loadFromIndex() + throws IOException { + try { + DirCacheEntry entry = db.readDirCache() + .getEntry(Constants.DOT_LFS_CONFIG); + if (entry != null) { + return new BlobBasedConfig(null, db, entry.getObjectId()); + } + } catch (ConfigInvalidException e) { + throw new LfsConfigInvalidException( + LfsText.get().dotLfsConfigReadFailed, e); + } + return null; + } + + /** + * Try to read the lfs config from an entry called .lfsconfig contained in + * the head revision. + * + * @return the config, or <code>null</code> if the file does not exist + * @throws IOException + * if an IO error occurred + */ + @Nullable + private Config loadFromHead() throws IOException { + try (RevWalk revWalk = new RevWalk(db)) { + ObjectId headCommitId = db.resolve(HEAD); + if (headCommitId == null) { + return null; + } + RevCommit commit = revWalk.parseCommit(headCommitId); + RevTree tree = commit.getTree(); + TreeWalk treewalk = TreeWalk.forPath(db, Constants.DOT_LFS_CONFIG, + tree); + if (treewalk != null) { + return new BlobBasedConfig(null, db, treewalk.getObjectId(0)); + } + } catch (ConfigInvalidException e) { + throw new LfsConfigInvalidException( + LfsText.get().dotLfsConfigReadFailed, e); + } + return null; + } + + /** + * Create an empty config as fallback to avoid null pointer checks. + * + * @return an empty config + */ + private Config emptyConfig() { + return new Config(); + } + + /** + * Get string value or null if not found. + * + * First tries to find the value in the git config files. If not found tries + * to find data in .lfsconfig. + * + * @param section + * the section + * @param subsection + * the subsection for the value + * @param name + * the key name + * @return a String value from the config, <code>null</code> if not found + * @throws IOException + * if an IO error occurred + */ + @Nullable + public String getString(final String section, final String subsection, + final String name) throws IOException { + String result = db.getConfig().getString(section, subsection, name); + if (result == null) { + result = getDelegate().getString(section, subsection, name); + } + return result; + } +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsConnectionFactory.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsConnectionFactory.java new file mode 100644 index 0000000000..1a4e85ded6 --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsConnectionFactory.java @@ -0,0 +1,321 @@ +/* + * Copyright (C) 2017, 2022 Markus Duft <markus.duft@ssi-schaefer.com> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lfs.internal; + +import static org.eclipse.jgit.lib.Constants.DEFAULT_REMOTE_NAME; +import static org.eclipse.jgit.util.HttpSupport.ENCODING_GZIP; +import static org.eclipse.jgit.util.HttpSupport.HDR_ACCEPT; +import static org.eclipse.jgit.util.HttpSupport.HDR_ACCEPT_ENCODING; +import static org.eclipse.jgit.util.HttpSupport.HDR_CONTENT_TYPE; + +import java.io.IOException; +import java.net.ProxySelector; +import java.net.URISyntaxException; +import java.net.URL; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Map; +import java.util.TreeMap; + +import org.eclipse.jgit.annotations.NonNull; +import org.eclipse.jgit.errors.CommandFailedException; +import org.eclipse.jgit.lfs.LfsPointer; +import org.eclipse.jgit.lfs.Protocol; +import org.eclipse.jgit.lfs.errors.LfsConfigInvalidException; +import org.eclipse.jgit.lib.ConfigConstants; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.lib.StoredConfig; +import org.eclipse.jgit.transport.HttpConfig; +import org.eclipse.jgit.transport.HttpTransport; +import org.eclipse.jgit.transport.URIish; +import org.eclipse.jgit.transport.http.HttpConnection; +import org.eclipse.jgit.util.HttpSupport; +import org.eclipse.jgit.util.SshSupport; +import org.eclipse.jgit.util.StringUtils; + +/** + * Provides means to get a valid LFS connection for a given repository. + */ +public class LfsConnectionFactory { + private static final int SSH_AUTH_TIMEOUT_SECONDS = 30; + private static final String SCHEME_HTTPS = "https"; //$NON-NLS-1$ + private static final String SCHEME_SSH = "ssh"; //$NON-NLS-1$ + private static final Map<String, AuthCache> sshAuthCache = new TreeMap<>(); + + /** + * Determine URL of LFS server by looking into config parameters lfs.url, + * lfs.[remote].url or remote.[remote].url. The LFS server URL is computed + * from remote.[remote].url by appending "/info/lfs". In case there is no + * URL configured, a SSH remote URI can be used to auto-detect the LFS URI + * by using the remote "git-lfs-authenticate" command. + * + * @param db + * the repository to work with + * @param method + * the method (GET,PUT,...) of the request this connection will + * be used for + * @param purpose + * the action, e.g. Protocol.OPERATION_DOWNLOAD + * @return the connection for the lfs server. e.g. + * "https://github.com/github/git-lfs.git/info/lfs" + * @throws IOException + * if an IO error occurred + */ + public static HttpConnection getLfsConnection(Repository db, String method, + String purpose) throws IOException { + StoredConfig config = db.getConfig(); + Map<String, String> additionalHeaders = new TreeMap<>(); + String lfsUrl = getLfsUrl(db, purpose, additionalHeaders); + URL url = new URL(lfsUrl + Protocol.OBJECTS_LFS_ENDPOINT); + HttpConnection connection = HttpTransport.getConnectionFactory().create( + url, HttpSupport.proxyFor(ProxySelector.getDefault(), url)); + connection.setDoOutput(true); + if (url.getProtocol().equals(SCHEME_HTTPS) + && !config.getBoolean(HttpConfig.HTTP, + HttpConfig.SSL_VERIFY_KEY, true)) { + HttpSupport.disableSslVerify(connection); + } + connection.setRequestMethod(method); + connection.setRequestProperty(HDR_ACCEPT, + Protocol.CONTENTTYPE_VND_GIT_LFS_JSON); + connection.setRequestProperty(HDR_CONTENT_TYPE, + Protocol.CONTENTTYPE_VND_GIT_LFS_JSON); + additionalHeaders + .forEach((k, v) -> connection.setRequestProperty(k, v)); + return connection; + } + + /** + * Get LFS Server URL. + * + * @param db + * the repository to work with + * @param purpose + * the action, e.g. Protocol.OPERATION_DOWNLOAD + * @param additionalHeaders + * additional headers that can be used to connect to LFS server + * @return the URL for the LFS server. e.g. + * "https://github.com/github/git-lfs.git/info/lfs" + * @throws IOException + * if the LFS config is invalid or cannot be accessed + * @see <a href= + * "https://github.com/git-lfs/git-lfs/blob/main/docs/api/server-discovery.md"> + * Server Discovery documentation</a> + */ + private static String getLfsUrl(Repository db, String purpose, + Map<String, String> additionalHeaders) + throws IOException { + LfsConfig config = new LfsConfig(db); + String lfsUrl = config.getString(ConfigConstants.CONFIG_SECTION_LFS, + null, ConfigConstants.CONFIG_KEY_URL); + + Exception ex = null; + if (lfsUrl == null) { + String remoteUrl = null; + for (String remote : db.getRemoteNames()) { + lfsUrl = config.getString(ConfigConstants.CONFIG_SECTION_LFS, + remote, + ConfigConstants.CONFIG_KEY_URL); + + // This could be done better (more precise logic), but according + // to https://github.com/git-lfs/git-lfs/issues/1759 git-lfs + // generally only supports 'origin' in an integrated workflow. + if (lfsUrl == null && remote.equals(DEFAULT_REMOTE_NAME)) { + remoteUrl = config.getString( + ConfigConstants.CONFIG_KEY_REMOTE, remote, + ConfigConstants.CONFIG_KEY_URL); + break; + } + } + if (lfsUrl == null && remoteUrl != null) { + try { + lfsUrl = discoverLfsUrl(db, purpose, additionalHeaders, + remoteUrl); + } catch (URISyntaxException | IOException + | CommandFailedException e) { + ex = e; + } + } + } + if (lfsUrl == null) { + if (ex != null) { + throw new LfsConfigInvalidException( + LfsText.get().lfsNoDownloadUrl, ex); + } + throw new LfsConfigInvalidException(LfsText.get().lfsNoDownloadUrl); + } + return lfsUrl; + } + + private static String discoverLfsUrl(Repository db, String purpose, + Map<String, String> additionalHeaders, String remoteUrl) + throws URISyntaxException, IOException, CommandFailedException { + URIish u = new URIish(remoteUrl); + if (u.getScheme() == null || SCHEME_SSH.equals(u.getScheme())) { + Protocol.ExpiringAction action = getSshAuthentication(db, purpose, + remoteUrl, u); + additionalHeaders.putAll(action.header); + return action.href; + } + return StringUtils.nameWithDotGit(remoteUrl) + + Protocol.INFO_LFS_ENDPOINT; + } + + private static Protocol.ExpiringAction getSshAuthentication( + Repository db, String purpose, String remoteUrl, URIish u) + throws IOException, CommandFailedException { + AuthCache cached = sshAuthCache.get(remoteUrl); + Protocol.ExpiringAction action = null; + if (cached != null && cached.validUntil > System.currentTimeMillis()) { + action = cached.cachedAction; + } + + if (action == null) { + // discover and authenticate; git-lfs does "ssh + // -p <port> -- <host> git-lfs-authenticate + // <project> <upload/download>" + String json = SshSupport.runSshCommand(u.setPath(""), //$NON-NLS-1$ + null, db.getFS(), + "git-lfs-authenticate " + extractProjectName(u) + " " //$NON-NLS-1$//$NON-NLS-2$ + + purpose, + SSH_AUTH_TIMEOUT_SECONDS); + + action = Protocol.gson().fromJson(json, + Protocol.ExpiringAction.class); + + // cache the result as long as possible. + AuthCache c = new AuthCache(action); + sshAuthCache.put(remoteUrl, c); + } + return action; + } + + /** + * Create a connection for the specified + * {@link org.eclipse.jgit.lfs.Protocol.Action}. + * + * @param repo + * the repo to fetch required configuration from + * @param action + * the action for which to create a connection + * @param method + * the target method (GET or PUT) + * @return a connection. output mode is not set. + * @throws IOException + * in case of any error. + */ + @NonNull + public static HttpConnection getLfsContentConnection( + Repository repo, Protocol.Action action, String method) + throws IOException { + URL contentUrl = new URL(action.href); + HttpConnection contentServerConn = HttpTransport.getConnectionFactory() + .create(contentUrl, HttpSupport + .proxyFor(ProxySelector.getDefault(), contentUrl)); + contentServerConn.setRequestMethod(method); + if (action.header != null) { + action.header.forEach( + (k, v) -> contentServerConn.setRequestProperty(k, v)); + } + if (contentUrl.getProtocol().equals(SCHEME_HTTPS) + && !repo.getConfig().getBoolean(HttpConfig.HTTP, + HttpConfig.SSL_VERIFY_KEY, true)) { + HttpSupport.disableSslVerify(contentServerConn); + } + + contentServerConn.setRequestProperty(HDR_ACCEPT_ENCODING, + ENCODING_GZIP); + + return contentServerConn; + } + + private static String extractProjectName(URIish u) { + String path = u.getPath(); + + // begins with a slash if the url contains a port (gerrit vs. github). + if (path.startsWith("/")) { //$NON-NLS-1$ + path = path.substring(1); + } + + if (path.endsWith(org.eclipse.jgit.lib.Constants.DOT_GIT)) { + return path.substring(0, path.length() - 4); + } + return path; + } + + /** + * Create request that can be serialized to JSON + * + * @param operation + * the operation to perform, e.g. Protocol.OPERATION_DOWNLOAD + * @param resources + * the LFS resources affected + * @return a request that can be serialized to JSON + */ + public static Protocol.Request toRequest(String operation, + LfsPointer... resources) { + Protocol.Request req = new Protocol.Request(); + req.operation = operation; + if (resources != null) { + req.objects = new ArrayList<>(); + for (LfsPointer res : resources) { + Protocol.ObjectSpec o = new Protocol.ObjectSpec(); + o.oid = res.getOid().getName(); + o.size = res.getSize(); + req.objects.add(o); + } + } + return req; + } + + private static final class AuthCache { + private static final long AUTH_CACHE_EAGER_TIMEOUT = 500; + + private static final DateTimeFormatter ISO_FORMAT = DateTimeFormatter + .ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSX"); //$NON-NLS-1$ + + /** + * Creates a cache entry for an authentication response. + * <p> + * The timeout of the cache token is extracted from the given action. If + * no timeout can be determined, the token will be used only once. + * + * @param action + * action with an additional expiration timestamp + */ + public AuthCache(Protocol.ExpiringAction action) { + this.cachedAction = action; + try { + if (action.expiresIn != null && !action.expiresIn.isEmpty()) { + this.validUntil = (System.currentTimeMillis() + + Long.parseLong(action.expiresIn)) + - AUTH_CACHE_EAGER_TIMEOUT; + } else if (action.expiresAt != null + && !action.expiresAt.isEmpty()) { + this.validUntil = LocalDateTime + .parse(action.expiresAt, ISO_FORMAT) + .atZone(ZoneOffset.UTC).toInstant().toEpochMilli() + - AUTH_CACHE_EAGER_TIMEOUT; + } else { + this.validUntil = System.currentTimeMillis(); + } + } catch (Exception e) { + this.validUntil = System.currentTimeMillis(); + } + } + + long validUntil; + + Protocol.ExpiringAction cachedAction; + } + +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsText.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsText.java new file mode 100644 index 0000000000..00b34ed3ea --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsText.java @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2015, Matthias Sohn <matthias.sohn@sap.com> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lfs.internal; + +import org.eclipse.jgit.nls.NLS; +import org.eclipse.jgit.nls.TranslationBundle; + +/** + * Translation bundle for JGit LFS server + */ +@SuppressWarnings("MissingSummary") +public class LfsText extends TranslationBundle { + + /** + * Get an instance of this translation bundle. + * + * @return an instance of this translation bundle + */ + public static LfsText get() { + return NLS.getBundleFor(LfsText.class); + } + + // @formatter:off + /***/ public String corruptLongObject; + /***/ public String dotLfsConfigReadFailed; + /***/ public String inconsistentContentLength; + /***/ public String inconsistentMediafileLength; + /***/ public String incorrectLONG_OBJECT_ID_LENGTH; + /***/ public String invalidLongId; + /***/ public String invalidLongIdLength; + /***/ public String lfsFailedToGetRepository; + /***/ public String lfsNoDownloadUrl; + /***/ public String lfsUnauthorized; + /***/ public String lfsUnavailable; + /***/ public String missingLocalObject; + /***/ public String protocolError; + /***/ public String repositoryNotFound; + /***/ public String repositoryReadOnly; + /***/ public String requiredHashFunctionNotAvailable; + /***/ public String serverFailure; + /***/ public String wrongAmountOfDataReceived; +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/AbbreviatedLongObjectId.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/AbbreviatedLongObjectId.java new file mode 100644 index 0000000000..7ae805c33f --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/AbbreviatedLongObjectId.java @@ -0,0 +1,361 @@ +/* + * Copyright (C) 2015, Matthias Sohn <matthias.sohn@sap.com> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.lfs.lib; + +import java.io.Serializable; +import java.text.MessageFormat; + +import org.eclipse.jgit.lfs.errors.InvalidLongObjectIdException; +import org.eclipse.jgit.lfs.internal.LfsText; +import org.eclipse.jgit.util.NB; +import org.eclipse.jgit.util.RawParseUtils; + +/** + * A prefix abbreviation of an {@link org.eclipse.jgit.lfs.lib.LongObjectId}. + * <p> + * Enable abbreviating SHA-256 strings used by Git LFS, using sufficient leading + * digits from the LongObjectId name to still be unique within the repository + * the string was generated from. These ids are likely to be unique for a useful + * period of time, especially if they contain at least 6-10 hex digits. + * <p> + * This class converts the hex string into a binary form, to make it more + * efficient for matching against an object. + * + * Ported to SHA-256 from {@link org.eclipse.jgit.lib.AbbreviatedObjectId} + * + * @since 4.3 + */ +public final class AbbreviatedLongObjectId implements Serializable { + private static final long serialVersionUID = 1L; + + /** + * Test a string of characters to verify it is a hex format. + * <p> + * If true the string can be parsed with {@link #fromString(String)}. + * + * @param id + * the string to test. + * @return true if the string can converted into an AbbreviatedObjectId. + */ + public static final boolean isId(String id) { + if (id.length() < 2 + || Constants.LONG_OBJECT_ID_STRING_LENGTH < id.length()) + return false; + try { + for (int i = 0; i < id.length(); i++) + RawParseUtils.parseHexInt4((byte) id.charAt(i)); + return true; + } catch (ArrayIndexOutOfBoundsException e) { + return false; + } + } + + /** + * Convert an AbbreviatedObjectId from hex characters (US-ASCII). + * + * @param buf + * the US-ASCII buffer to read from. + * @param offset + * position to read the first character from. + * @param end + * one past the last position to read (<code>end-offset</code> is + * the length of the string). + * @return the converted object id. + */ + public static final AbbreviatedLongObjectId fromString(final byte[] buf, + final int offset, final int end) { + if (end - offset > Constants.LONG_OBJECT_ID_STRING_LENGTH) + throw new IllegalArgumentException(MessageFormat.format( + LfsText.get().invalidLongIdLength, + Integer.valueOf(end - offset), + Integer.valueOf(Constants.LONG_OBJECT_ID_STRING_LENGTH))); + return fromHexString(buf, offset, end); + } + + /** + * Convert an AbbreviatedObjectId from an + * {@link org.eclipse.jgit.lib.AnyObjectId}. + * <p> + * This method copies over all bits of the Id, and is therefore complete + * (see {@link #isComplete()}). + * + * @param id + * the {@link org.eclipse.jgit.lib.ObjectId} to convert from. + * @return the converted object id. + */ + public static final AbbreviatedLongObjectId fromLongObjectId( + AnyLongObjectId id) { + return new AbbreviatedLongObjectId( + Constants.LONG_OBJECT_ID_STRING_LENGTH, id.w1, id.w2, id.w3, + id.w4); + } + + /** + * Convert an AbbreviatedLongObjectId from hex characters. + * + * @param str + * the string to read from. Must be <= 64 characters. + * @return the converted object id. + */ + public static final AbbreviatedLongObjectId fromString(String str) { + if (str.length() > Constants.LONG_OBJECT_ID_STRING_LENGTH) + throw new IllegalArgumentException( + MessageFormat.format(LfsText.get().invalidLongId, str)); + final byte[] b = org.eclipse.jgit.lib.Constants.encodeASCII(str); + return fromHexString(b, 0, b.length); + } + + private static final AbbreviatedLongObjectId fromHexString(final byte[] bs, + int ptr, final int end) { + try { + final long a = hexUInt64(bs, ptr, end); + final long b = hexUInt64(bs, ptr + 16, end); + final long c = hexUInt64(bs, ptr + 32, end); + final long d = hexUInt64(bs, ptr + 48, end); + return new AbbreviatedLongObjectId(end - ptr, a, b, c, d); + } catch (ArrayIndexOutOfBoundsException e) { + InvalidLongObjectIdException e1 = new InvalidLongObjectIdException( + bs, ptr, end - ptr); + e1.initCause(e); + throw e1; + } + } + + private static final long hexUInt64(final byte[] bs, int p, final int end) { + if (16 <= end - p) + return RawParseUtils.parseHexInt64(bs, p); + + long r = 0; + int n = 0; + while (n < 16 && p < end) { + r <<= 4; + r |= RawParseUtils.parseHexInt4(bs[p++]); + n++; + } + return r << ((16 - n) * 4); + } + + static long mask(int nibbles, long word, long v) { + final long b = (word - 1) * 16; + if (b + 16 <= nibbles) { + // We have all of the bits required for this word. + // + return v; + } + + if (nibbles <= b) { + // We have none of the bits required for this word. + // + return 0; + } + + final long s = 64 - (nibbles - b) * 4; + return (v >>> s) << s; + } + + /** Number of half-bytes used by this id. */ + final int nibbles; + + final long w1; + + final long w2; + + final long w3; + + final long w4; + + AbbreviatedLongObjectId(final int n, final long new_1, final long new_2, + final long new_3, final long new_4) { + nibbles = n; + w1 = new_1; + w2 = new_2; + w3 = new_3; + w4 = new_4; + } + + /** + * Get length + * + * @return number of hex digits appearing in this id. + */ + public int length() { + return nibbles; + } + + /** + * Check if this id is complete + * + * @return true if this ObjectId is actually a complete id. + */ + public boolean isComplete() { + return length() == Constants.LONG_OBJECT_ID_STRING_LENGTH; + } + + /** + * Convert to LongObjectId + * + * @return a complete ObjectId; null if {@link #isComplete()} is false. + */ + public LongObjectId toLongObjectId() { + return isComplete() ? new LongObjectId(w1, w2, w3, w4) : null; + } + + /** + * Compares this abbreviation to a full object id. + * + * @param other + * the other object id. + * @return <0 if this abbreviation names an object that is less than + * <code>other</code>; 0 if this abbreviation exactly matches the + * first {@link #length()} digits of <code>other.name()</code>; + * >0 if this abbreviation names an object that is after + * <code>other</code>. + */ + public final int prefixCompare(AnyLongObjectId other) { + int cmp; + + cmp = NB.compareUInt64(w1, mask(1, other.w1)); + if (cmp != 0) + return cmp; + + cmp = NB.compareUInt64(w2, mask(2, other.w2)); + if (cmp != 0) + return cmp; + + cmp = NB.compareUInt64(w3, mask(3, other.w3)); + if (cmp != 0) + return cmp; + + return NB.compareUInt64(w4, mask(4, other.w4)); + } + + /** + * Compare this abbreviation to a network-byte-order LongObjectId. + * + * @param bs + * array containing the other LongObjectId in network byte order. + * @param p + * position within {@code bs} to start the compare at. At least + * 32 bytes, starting at this position are required. + * @return <0 if this abbreviation names an object that is less than + * <code>other</code>; 0 if this abbreviation exactly matches the + * first {@link #length()} digits of <code>other.name()</code>; + * >0 if this abbreviation names an object that is after + * <code>other</code>. + */ + public final int prefixCompare(byte[] bs, int p) { + int cmp; + + cmp = NB.compareUInt64(w1, mask(1, NB.decodeInt64(bs, p))); + if (cmp != 0) + return cmp; + + cmp = NB.compareUInt64(w2, mask(2, NB.decodeInt64(bs, p + 8))); + if (cmp != 0) + return cmp; + + cmp = NB.compareUInt64(w3, mask(3, NB.decodeInt64(bs, p + 16))); + if (cmp != 0) + return cmp; + + return NB.compareUInt64(w4, mask(4, NB.decodeInt64(bs, p + 24))); + } + + /** + * Compare this abbreviation to a network-byte-order LongObjectId. + * + * @param bs + * array containing the other LongObjectId in network byte order. + * @param p + * position within {@code bs} to start the compare at. At least 4 + * longs, starting at this position are required. + * @return <0 if this abbreviation names an object that is less than + * <code>other</code>; 0 if this abbreviation exactly matches the + * first {@link #length()} digits of <code>other.name()</code>; + * >0 if this abbreviation names an object that is after + * <code>other</code>. + */ + public final int prefixCompare(long[] bs, int p) { + int cmp; + + cmp = NB.compareUInt64(w1, mask(1, bs[p])); + if (cmp != 0) + return cmp; + + cmp = NB.compareUInt64(w2, mask(2, bs[p + 1])); + if (cmp != 0) + return cmp; + + cmp = NB.compareUInt64(w3, mask(3, bs[p + 2])); + if (cmp != 0) + return cmp; + + return NB.compareUInt64(w4, mask(4, bs[p + 3])); + } + + /** + * Get the first byte of this id + * + * @return value for a fan-out style map, only valid of length >= 2. + */ + public final int getFirstByte() { + return (int) (w1 >>> 56); + } + + private long mask(long word, long v) { + return mask(nibbles, word, v); + } + + @Override + public int hashCode() { + return (int) (w1 >> 32); + } + + @Override + public boolean equals(Object o) { + if (o instanceof AbbreviatedLongObjectId) { + final AbbreviatedLongObjectId b = (AbbreviatedLongObjectId) o; + return nibbles == b.nibbles && w1 == b.w1 && w2 == b.w2 + && w3 == b.w3 && w4 == b.w4; + } + return false; + } + + /** + * <p>name.</p> + * + * @return string form of the abbreviation, in lower case hexadecimal. + */ + public final String name() { + final char[] b = new char[Constants.LONG_OBJECT_ID_STRING_LENGTH]; + + AnyLongObjectId.formatHexChar(b, 0, w1); + if (nibbles <= 16) + return new String(b, 0, nibbles); + + AnyLongObjectId.formatHexChar(b, 16, w2); + if (nibbles <= 32) + return new String(b, 0, nibbles); + + AnyLongObjectId.formatHexChar(b, 32, w3); + if (nibbles <= 48) + return new String(b, 0, nibbles); + + AnyLongObjectId.formatHexChar(b, 48, w4); + return new String(b, 0, nibbles); + } + + @SuppressWarnings("nls") + @Override + public String toString() { + return "AbbreviatedLongObjectId[" + name() + "]"; //$NON-NLS-1$ + } +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/AnyLongObjectId.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/AnyLongObjectId.java new file mode 100644 index 0000000000..a13a60c2b8 --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/AnyLongObjectId.java @@ -0,0 +1,533 @@ +/* + * Copyright (C) 2015, Matthias Sohn <matthias.sohn@sap.com> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.lfs.lib; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.Writer; +import java.nio.ByteBuffer; + +import org.eclipse.jgit.lib.AnyObjectId; +import org.eclipse.jgit.util.NB; +import org.eclipse.jgit.util.References; + +/** + * A (possibly mutable) SHA-256 abstraction. + * <p> + * If this is an instance of + * {@link org.eclipse.jgit.lfs.lib.MutableLongObjectId} the concept of equality + * with this instance can alter at any time, if this instance is modified to + * represent a different object name. + * + * Ported to SHA-256 from {@link org.eclipse.jgit.lib.AnyObjectId} + * + * @since 4.3 + */ +public abstract class AnyLongObjectId implements Comparable<AnyLongObjectId> { + + /** + * Compare two object identifier byte sequences for equality. + * + * @param firstObjectId + * the first identifier to compare. Must not be null. + * @param secondObjectId + * the second identifier to compare. Must not be null. + * @return true if the two identifiers are the same. + * @since 5.4 + */ + public static boolean isEqual(final AnyLongObjectId firstObjectId, + final AnyLongObjectId secondObjectId) { + if (References.isSameObject(firstObjectId, secondObjectId)) { + return true; + } + + // We test word 2 first as odds are someone already used our + // word 1 as a hash code, and applying that came up with these + // two instances we are comparing for equality. Therefore the + // first two words are very likely to be identical. We want to + // break away from collisions as quickly as possible. + // + return firstObjectId.w2 == secondObjectId.w2 + && firstObjectId.w3 == secondObjectId.w3 + && firstObjectId.w4 == secondObjectId.w4 + && firstObjectId.w1 == secondObjectId.w1; + } + + long w1; + + long w2; + + long w3; + + long w4; + + /** + * Get the first 8 bits of the LongObjectId. + * + * This is a faster version of {@code getByte(0)}. + * + * @return a discriminator usable for a fan-out style map. Returned values + * are unsigned and thus are in the range [0,255] rather than the + * signed byte range of [-128, 127]. + */ + public final int getFirstByte() { + return (int) (w1 >>> 56); + } + + /** + * Get the second 8 bits of the LongObjectId. + * + * @return a discriminator usable for a fan-out style map. Returned values + * are unsigned and thus are in the range [0,255] rather than the + * signed byte range of [-128, 127]. + */ + public final int getSecondByte() { + return (int) ((w1 >>> 48) & 0xff); + } + + /** + * Get any byte from the LongObjectId. + * + * Callers hard-coding {@code getByte(0)} should instead use the much faster + * special case variant {@link #getFirstByte()}. + * + * @param index + * index of the byte to obtain from the raw form of the + * LongObjectId. Must be in range [0, + * {@link org.eclipse.jgit.lfs.lib.Constants#LONG_OBJECT_ID_LENGTH}). + * @return the value of the requested byte at {@code index}. Returned values + * are unsigned and thus are in the range [0,255] rather than the + * signed byte range of [-128, 127]. + * @throws java.lang.ArrayIndexOutOfBoundsException + * {@code index} is less than 0, equal to + * {@link org.eclipse.jgit.lfs.lib.Constants#LONG_OBJECT_ID_LENGTH}, + * or greater than + * {@link org.eclipse.jgit.lfs.lib.Constants#LONG_OBJECT_ID_LENGTH}. + */ + public final int getByte(int index) { + long w; + switch (index >> 3) { + case 0: + w = w1; + break; + case 1: + w = w2; + break; + case 2: + w = w3; + break; + case 3: + w = w4; + break; + default: + throw new ArrayIndexOutOfBoundsException(index); + } + + return (int) ((w >>> (8 * (15 - (index & 15)))) & 0xff); + } + + /** + * {@inheritDoc} + * + * Compare this LongObjectId to another and obtain a sort ordering. + */ + @Override + public final int compareTo(AnyLongObjectId other) { + if (this == other) + return 0; + + int cmp; + + cmp = NB.compareUInt64(w1, other.w1); + if (cmp != 0) + return cmp; + + cmp = NB.compareUInt64(w2, other.w2); + if (cmp != 0) + return cmp; + + cmp = NB.compareUInt64(w3, other.w3); + if (cmp != 0) + return cmp; + + return NB.compareUInt64(w4, other.w4); + } + + /** + * Compare this LongObjectId to a network-byte-order LongObjectId. + * + * @param bs + * array containing the other LongObjectId in network byte order. + * @param p + * position within {@code bs} to start the compare at. At least + * 32 bytes, starting at this position are required. + * @return a negative integer, zero, or a positive integer as this object is + * less than, equal to, or greater than the specified object. + */ + public final int compareTo(byte[] bs, int p) { + int cmp; + + cmp = NB.compareUInt64(w1, NB.decodeInt64(bs, p)); + if (cmp != 0) + return cmp; + + cmp = NB.compareUInt64(w2, NB.decodeInt64(bs, p + 8)); + if (cmp != 0) + return cmp; + + cmp = NB.compareUInt64(w3, NB.decodeInt64(bs, p + 16)); + if (cmp != 0) + return cmp; + + return NB.compareUInt64(w4, NB.decodeInt64(bs, p + 24)); + } + + /** + * Compare this LongObjectId to a network-byte-order LongObjectId. + * + * @param bs + * array containing the other LongObjectId in network byte order. + * @param p + * position within {@code bs} to start the compare at. At least 4 + * longs, starting at this position are required. + * @return a negative integer, zero, or a positive integer as this object is + * less than, equal to, or greater than the specified object. + */ + public final int compareTo(long[] bs, int p) { + int cmp; + + cmp = NB.compareUInt64(w1, bs[p]); + if (cmp != 0) + return cmp; + + cmp = NB.compareUInt64(w2, bs[p + 1]); + if (cmp != 0) + return cmp; + + cmp = NB.compareUInt64(w3, bs[p + 2]); + if (cmp != 0) + return cmp; + + return NB.compareUInt64(w4, bs[p + 3]); + } + + /** + * Tests if this LongObjectId starts with the given abbreviation. + * + * @param abbr + * the abbreviation. + * @return true if this LongObjectId begins with the abbreviation; else + * false. + */ + public boolean startsWith(AbbreviatedLongObjectId abbr) { + return abbr.prefixCompare(this) == 0; + } + + @Override + public final int hashCode() { + return (int) (w1 >> 32); + } + + /** + * Determine if this LongObjectId has exactly the same value as another. + * + * @param other + * the other id to compare to. May be null. + * @return true only if both LongObjectIds have identical bits. + */ + @SuppressWarnings({ "NonOverridingEquals", "AmbiguousMethodReference" }) + public final boolean equals(AnyLongObjectId other) { + return other != null ? isEqual(this, other) : false; + } + + @Override + public final boolean equals(Object o) { + if (o instanceof AnyLongObjectId) { + return equals((AnyLongObjectId) o); + } + return false; + } + + /** + * Copy this LongObjectId to an output writer in raw binary. + * + * @param w + * the buffer to copy to. Must be in big endian order. + */ + public void copyRawTo(ByteBuffer w) { + w.putLong(w1); + w.putLong(w2); + w.putLong(w3); + w.putLong(w4); + } + + /** + * Copy this LongObjectId to a byte array. + * + * @param b + * the buffer to copy to. + * @param o + * the offset within b to write at. + */ + public void copyRawTo(byte[] b, int o) { + NB.encodeInt64(b, o, w1); + NB.encodeInt64(b, o + 8, w2); + NB.encodeInt64(b, o + 16, w3); + NB.encodeInt64(b, o + 24, w4); + } + + /** + * Copy this LongObjectId to an long array. + * + * @param b + * the buffer to copy to. + * @param o + * the offset within b to write at. + */ + public void copyRawTo(long[] b, int o) { + b[o] = w1; + b[o + 1] = w2; + b[o + 2] = w3; + b[o + 3] = w4; + } + + /** + * Copy this LongObjectId to an output writer in raw binary. + * + * @param w + * the stream to write to. + * @throws java.io.IOException + * the stream writing failed. + */ + public void copyRawTo(OutputStream w) throws IOException { + writeRawLong(w, w1); + writeRawLong(w, w2); + writeRawLong(w, w3); + writeRawLong(w, w4); + } + + private static void writeRawLong(OutputStream w, long v) + throws IOException { + w.write((int) (v >>> 56)); + w.write((int) (v >>> 48)); + w.write((int) (v >>> 40)); + w.write((int) (v >>> 32)); + w.write((int) (v >>> 24)); + w.write((int) (v >>> 16)); + w.write((int) (v >>> 8)); + w.write((int) v); + } + + /** + * Copy this LongObjectId to an output writer in hex format. + * + * @param w + * the stream to copy to. + * @throws java.io.IOException + * the stream writing failed. + */ + public void copyTo(OutputStream w) throws IOException { + w.write(toHexByteArray()); + } + + /** + * Copy this LongObjectId to a byte array in hex format. + * + * @param b + * the buffer to copy to. + * @param o + * the offset within b to write at. + */ + public void copyTo(byte[] b, int o) { + formatHexByte(b, o + 0, w1); + formatHexByte(b, o + 16, w2); + formatHexByte(b, o + 32, w3); + formatHexByte(b, o + 48, w4); + } + + /** + * Copy this LongObjectId to a ByteBuffer in hex format. + * + * @param b + * the buffer to copy to. + */ + public void copyTo(ByteBuffer b) { + b.put(toHexByteArray()); + } + + private byte[] toHexByteArray() { + final byte[] dst = new byte[Constants.LONG_OBJECT_ID_STRING_LENGTH]; + formatHexByte(dst, 0, w1); + formatHexByte(dst, 16, w2); + formatHexByte(dst, 32, w3); + formatHexByte(dst, 48, w4); + return dst; + } + + private static final byte[] hexbyte = { '0', '1', '2', '3', '4', '5', '6', + '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' }; + + private static void formatHexByte(byte[] dst, int p, long w) { + int o = p + 15; + while (o >= p && w != 0) { + dst[o--] = hexbyte[(int) (w & 0xf)]; + w >>>= 4; + } + while (o >= p) + dst[o--] = '0'; + } + + /** + * Copy this LongObjectId to an output writer in hex format. + * + * @param w + * the stream to copy to. + * @throws java.io.IOException + * the stream writing failed. + */ + public void copyTo(Writer w) throws IOException { + w.write(toHexCharArray()); + } + + /** + * Copy this LongObjectId to an output writer in hex format. + * + * @param tmp + * temporary char array to buffer construct into before writing. + * Must be at least large enough to hold 2 digits for each byte + * of object id (64 characters or larger). + * @param w + * the stream to copy to. + * @throws java.io.IOException + * the stream writing failed. + */ + public void copyTo(char[] tmp, Writer w) throws IOException { + toHexCharArray(tmp); + w.write(tmp, 0, Constants.LONG_OBJECT_ID_STRING_LENGTH); + } + + /** + * Copy this LongObjectId to a StringBuilder in hex format. + * + * @param tmp + * temporary char array to buffer construct into before writing. + * Must be at least large enough to hold 2 digits for each byte + * of object id (64 characters or larger). + * @param w + * the string to append onto. + */ + public void copyTo(char[] tmp, StringBuilder w) { + toHexCharArray(tmp); + w.append(tmp, 0, Constants.LONG_OBJECT_ID_STRING_LENGTH); + } + + char[] toHexCharArray() { + final char[] dst = new char[Constants.LONG_OBJECT_ID_STRING_LENGTH]; + toHexCharArray(dst); + return dst; + } + + private void toHexCharArray(char[] dst) { + formatHexChar(dst, 0, w1); + formatHexChar(dst, 16, w2); + formatHexChar(dst, 32, w3); + formatHexChar(dst, 48, w4); + } + + private static final char[] hexchar = { '0', '1', '2', '3', '4', '5', '6', + '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' }; + + static void formatHexChar(char[] dst, int p, long w) { + int o = p + 15; + while (o >= p && w != 0) { + dst[o--] = hexchar[(int) (w & 0xf)]; + w >>>= 4; + } + while (o >= p) + dst[o--] = '0'; + } + + @SuppressWarnings("nls") + @Override + public String toString() { + return "AnyLongObjectId[" + name() + "]"; + } + + /** + * Get string form of the SHA-256 + * + * @return string form of the SHA-256, in lower case hexadecimal. + */ + public final String name() { + return new String(toHexCharArray()); + } + + /** + * Get string form of the SHA-256 + * + * @return string form of the SHA-256, in lower case hexadecimal. + */ + public final String getName() { + return name(); + } + + /** + * Return an abbreviation (prefix) of this object SHA-256. + * <p> + * This implementation does not guarantee uniqueness. Callers should instead + * use + * {@link org.eclipse.jgit.lib.ObjectReader#abbreviate(AnyObjectId, int)} to + * obtain a unique abbreviation within the scope of a particular object + * database. + * + * @param len + * length of the abbreviated string. + * @return SHA-256 abbreviation. + */ + public AbbreviatedLongObjectId abbreviate(int len) { + final long a = AbbreviatedLongObjectId.mask(len, 1, w1); + final long b = AbbreviatedLongObjectId.mask(len, 2, w2); + final long c = AbbreviatedLongObjectId.mask(len, 3, w3); + final long d = AbbreviatedLongObjectId.mask(len, 4, w4); + return new AbbreviatedLongObjectId(len, a, b, c, d); + } + + /** + * Obtain an immutable copy of this current object. + * <p> + * Only returns <code>this</code> if this instance is an unsubclassed + * instance of {@link org.eclipse.jgit.lfs.lib.LongObjectId}; otherwise a + * new instance is returned holding the same value. + * <p> + * This method is useful to shed any additional memory that may be tied to + * the subclass, yet retain the unique identity of the object id for future + * lookups within maps and repositories. + * + * @return an immutable copy, using the smallest memory footprint possible. + */ + public final LongObjectId copy() { + if (getClass() == LongObjectId.class) + return (LongObjectId) this; + return new LongObjectId(this); + } + + /** + * Obtain an immutable copy of this current object. + * <p> + * See {@link #copy()} if <code>this</code> is a possibly subclassed (but + * immutable) identity and the application needs a lightweight identity + * <i>only</i> reference. + * + * @return an immutable copy. May be <code>this</code> if this is already an + * immutable instance. + */ + public abstract LongObjectId toObjectId(); +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/Constants.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/Constants.java new file mode 100644 index 0000000000..9b41ec31f1 --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/Constants.java @@ -0,0 +1,128 @@ +/* + * Copyright (C) 2015, Matthias Sohn <matthias.sohn@sap.com> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lfs.lib; + +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.text.MessageFormat; + +import org.eclipse.jgit.lfs.internal.LfsText; + +/** + * Misc. constants used throughout JGit LFS extension. + * + * @since 4.3 + */ +@SuppressWarnings("nls") +public final class Constants { + /** + * lfs folder/section/filter name + * + * @since 4.6 + */ + public static final String LFS = "lfs"; + + /** + * Hash function used natively by Git LFS extension for large objects. + * + * @since 4.6 + */ + public static final String LONG_HASH_FUNCTION = "SHA-256"; + + /** + * A Git LFS large object hash is 256 bits, i.e. 32 bytes. + * <p> + * Changing this assumption is not going to be as easy as changing this + * declaration. + */ + public static final int LONG_OBJECT_ID_LENGTH = 32; + + /** + * A Git LFS large object can be expressed as a 64 character string of + * hexadecimal digits. + * + * @see #LONG_OBJECT_ID_LENGTH + */ + public static final int LONG_OBJECT_ID_STRING_LENGTH = LONG_OBJECT_ID_LENGTH + * 2; + + /** + * LFS upload operation. + * + * @since 4.7 + */ + public static final String UPLOAD = "upload"; + + /** + * LFS download operation. + * + * @since 4.7 + */ + public static final String DOWNLOAD = "download"; + + /** + * LFS verify operation. + * + * @since 4.7 + */ + public static final String VERIFY = "verify"; + + /** + * Prefix for all LFS related filters. + * + * @since 4.11 + */ + public static final String ATTR_FILTER_DRIVER_PREFIX = "lfs/"; + + /** + * Config file name for lfs specific configuration + * + * @since 6.1 + */ + public static final String DOT_LFS_CONFIG = ".lfsconfig"; + + /** + * Create a new digest function for objects. + * + * @return a new digest object. + * @throws java.lang.RuntimeException + * this Java virtual machine does not support the required hash + * function. Very unlikely given that JGit uses a hash function + * that is in the Java reference specification. + */ + public static MessageDigest newMessageDigest() { + try { + return MessageDigest.getInstance(LONG_HASH_FUNCTION); + } catch (NoSuchAlgorithmException nsae) { + throw new RuntimeException(MessageFormat.format( + LfsText.get().requiredHashFunctionNotAvailable, + LONG_HASH_FUNCTION), nsae); + } + } + + static { + if (LONG_OBJECT_ID_LENGTH != newMessageDigest().getDigestLength()) + throw new LinkageError( + LfsText.get().incorrectLONG_OBJECT_ID_LENGTH); + } + + /** + * Content type used by LFS REST API as defined in <a href= + * "https://github.com/github/git-lfs/blob/master/docs/api/v1/http-v1-batch.md"> + * https://github.com/github/git-lfs/blob/master/docs/api/v1/http-v1-batch.md</a> + */ + public static final String CONTENT_TYPE_GIT_LFS_JSON = "application/vnd.git-lfs+json"; + + /** + * "Arbitrary binary data" as defined in + * <a href="https://www.ietf.org/rfc/rfc2046.txt">RFC 2046</a> + */ + public static final String HDR_APPLICATION_OCTET_STREAM = "application/octet-stream"; +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/LfsPointerFilter.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/LfsPointerFilter.java new file mode 100644 index 0000000000..75798ca0f1 --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/LfsPointerFilter.java @@ -0,0 +1,74 @@ +/* + * Copyright (C) 2015, 2021 Dariusz Luksza <dariusz@luksza.org> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.lfs.lib; + +import java.io.IOException; + +import org.eclipse.jgit.errors.IncorrectObjectTypeException; +import org.eclipse.jgit.errors.MissingObjectException; +import org.eclipse.jgit.lfs.LfsPointer; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.ObjectLoader; +import org.eclipse.jgit.lib.ObjectStream; +import org.eclipse.jgit.treewalk.TreeWalk; +import org.eclipse.jgit.treewalk.filter.TreeFilter; + +/** + * Detects Large File pointers, as described in [1] in Git repository. + * + * [1] https://github.com/github/git-lfs/blob/master/docs/spec.md + * + * @since 4.7 + */ +public class LfsPointerFilter extends TreeFilter { + + private LfsPointer pointer; + + /** + * Get the field <code>pointer</code>. + * + * @return {@link org.eclipse.jgit.lfs.LfsPointer} or {@code null} + */ + public LfsPointer getPointer() { + return pointer; + } + + @Override + public boolean include(TreeWalk walk) throws MissingObjectException, + IncorrectObjectTypeException, IOException { + pointer = null; + if (walk.isSubtree()) { + return walk.isRecursive(); + } + ObjectId objectId = walk.getObjectId(0); + ObjectLoader object = walk.getObjectReader().open(objectId); + if (object.getSize() > 1024) { + return false; + } + + try (ObjectStream stream = object.openStream()) { + pointer = LfsPointer.parseLfsPointer(stream); + return pointer != null; + } catch (RuntimeException e) { + return false; + } + } + + @Override + public boolean shouldBeRecursive() { + return false; + } + + @Override + public TreeFilter clone() { + return new LfsPointerFilter(); + } +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/LongObjectId.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/LongObjectId.java new file mode 100644 index 0000000000..3959115462 --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/LongObjectId.java @@ -0,0 +1,277 @@ +/* + * Copyright (C) 2015, Matthias Sohn <matthias.sohn@sap.com> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.lfs.lib; + +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.Serializable; + +import org.eclipse.jgit.lfs.errors.InvalidLongObjectIdException; +import org.eclipse.jgit.util.NB; +import org.eclipse.jgit.util.RawParseUtils; + +/** + * A SHA-256 abstraction. + * + * Ported to SHA-256 from {@link org.eclipse.jgit.lib.ObjectId} + * + * @since 4.3 + */ +public class LongObjectId extends AnyLongObjectId implements Serializable { + private static final long serialVersionUID = 1L; + + private static final LongObjectId ZEROID; + + private static final String ZEROID_STR; + + static { + ZEROID = new LongObjectId(0L, 0L, 0L, 0L); + ZEROID_STR = ZEROID.name(); + } + + /** + * Get the special all-zero LongObjectId. + * + * @return the all-zero LongObjectId, often used to stand-in for no object. + */ + public static final LongObjectId zeroId() { + return ZEROID; + } + + /** + * Test a string of characters to verify that it can be interpreted as + * LongObjectId. + * <p> + * If true the string can be parsed with {@link #fromString(String)}. + * + * @param id + * the string to test. + * @return true if the string can converted into an LongObjectId. + */ + public static final boolean isId(String id) { + if (id.length() != Constants.LONG_OBJECT_ID_STRING_LENGTH) + return false; + try { + for (int i = 0; i < Constants.LONG_OBJECT_ID_STRING_LENGTH; i++) { + RawParseUtils.parseHexInt4((byte) id.charAt(i)); + } + return true; + } catch (ArrayIndexOutOfBoundsException e) { + return false; + } + } + + /** + * Convert a LongObjectId into a hex string representation. + * + * @param i + * the id to convert. May be null. + * @return the hex string conversion of this id's content. + */ + public static final String toString(LongObjectId i) { + return i != null ? i.name() : ZEROID_STR; + } + + /** + * Compare two object identifier byte sequences for equality. + * + * @param firstBuffer + * the first buffer to compare against. Must have at least 32 + * bytes from position fi through the end of the buffer. + * @param fi + * first offset within firstBuffer to begin testing. + * @param secondBuffer + * the second buffer to compare against. Must have at least 32 + * bytes from position si through the end of the buffer. + * @param si + * first offset within secondBuffer to begin testing. + * @return true if the two identifiers are the same. + */ + public static boolean equals(final byte[] firstBuffer, final int fi, + final byte[] secondBuffer, final int si) { + return firstBuffer[fi] == secondBuffer[si] + && firstBuffer[fi + 1] == secondBuffer[si + 1] + && firstBuffer[fi + 2] == secondBuffer[si + 2] + && firstBuffer[fi + 3] == secondBuffer[si + 3] + && firstBuffer[fi + 4] == secondBuffer[si + 4] + && firstBuffer[fi + 5] == secondBuffer[si + 5] + && firstBuffer[fi + 6] == secondBuffer[si + 6] + && firstBuffer[fi + 7] == secondBuffer[si + 7] + && firstBuffer[fi + 8] == secondBuffer[si + 8] + && firstBuffer[fi + 9] == secondBuffer[si + 9] + && firstBuffer[fi + 10] == secondBuffer[si + 10] + && firstBuffer[fi + 11] == secondBuffer[si + 11] + && firstBuffer[fi + 12] == secondBuffer[si + 12] + && firstBuffer[fi + 13] == secondBuffer[si + 13] + && firstBuffer[fi + 14] == secondBuffer[si + 14] + && firstBuffer[fi + 15] == secondBuffer[si + 15] + && firstBuffer[fi + 16] == secondBuffer[si + 16] + && firstBuffer[fi + 17] == secondBuffer[si + 17] + && firstBuffer[fi + 18] == secondBuffer[si + 18] + && firstBuffer[fi + 19] == secondBuffer[si + 19] + && firstBuffer[fi + 20] == secondBuffer[si + 20] + && firstBuffer[fi + 21] == secondBuffer[si + 21] + && firstBuffer[fi + 22] == secondBuffer[si + 22] + && firstBuffer[fi + 23] == secondBuffer[si + 23] + && firstBuffer[fi + 24] == secondBuffer[si + 24] + && firstBuffer[fi + 25] == secondBuffer[si + 25] + && firstBuffer[fi + 26] == secondBuffer[si + 26] + && firstBuffer[fi + 27] == secondBuffer[si + 27] + && firstBuffer[fi + 28] == secondBuffer[si + 28] + && firstBuffer[fi + 29] == secondBuffer[si + 29] + && firstBuffer[fi + 30] == secondBuffer[si + 30] + && firstBuffer[fi + 31] == secondBuffer[si + 31]; + } + + /** + * Convert a LongObjectId from raw binary representation. + * + * @param bs + * the raw byte buffer to read from. At least 32 bytes must be + * available within this byte array. + * @return the converted object id. + */ + public static final LongObjectId fromRaw(byte[] bs) { + return fromRaw(bs, 0); + } + + /** + * Convert a LongObjectId from raw binary representation. + * + * @param bs + * the raw byte buffer to read from. At least 32 bytes after p + * must be available within this byte array. + * @param p + * position to read the first byte of data from. + * @return the converted object id. + */ + public static final LongObjectId fromRaw(byte[] bs, int p) { + final long a = NB.decodeInt64(bs, p); + final long b = NB.decodeInt64(bs, p + 8); + final long c = NB.decodeInt64(bs, p + 16); + final long d = NB.decodeInt64(bs, p + 24); + return new LongObjectId(a, b, c, d); + } + + /** + * Convert a LongObjectId from raw binary representation. + * + * @param is + * the raw long buffer to read from. At least 4 longs must be + * available within this long array. + * @return the converted object id. + */ + public static final LongObjectId fromRaw(long[] is) { + return fromRaw(is, 0); + } + + /** + * Convert a LongObjectId from raw binary representation. + * + * @param is + * the raw long buffer to read from. At least 4 longs after p + * must be available within this long array. + * @param p + * position to read the first long of data from. + * @return the converted object id. + */ + public static final LongObjectId fromRaw(long[] is, int p) { + return new LongObjectId(is[p], is[p + 1], is[p + 2], is[p + 3]); + } + + /** + * Convert a LongObjectId from hex characters (US-ASCII). + * + * @param buf + * the US-ASCII buffer to read from. At least 64 bytes after + * offset must be available within this byte array. + * @param offset + * position to read the first character from. + * @return the converted object id. + */ + public static final LongObjectId fromString(byte[] buf, int offset) { + return fromHexString(buf, offset); + } + + /** + * Convert a LongObjectId from hex characters. + * + * @param str + * the string to read from. Must be 64 characters long. + * @return the converted object id. + */ + public static LongObjectId fromString(String str) { + if (str.length() != Constants.LONG_OBJECT_ID_STRING_LENGTH) + throw new InvalidLongObjectIdException(str); + return fromHexString(org.eclipse.jgit.lib.Constants.encodeASCII(str), + 0); + } + + private static final LongObjectId fromHexString(byte[] bs, int p) { + try { + final long a = RawParseUtils.parseHexInt64(bs, p); + final long b = RawParseUtils.parseHexInt64(bs, p + 16); + final long c = RawParseUtils.parseHexInt64(bs, p + 32); + final long d = RawParseUtils.parseHexInt64(bs, p + 48); + return new LongObjectId(a, b, c, d); + } catch (ArrayIndexOutOfBoundsException e) { + InvalidLongObjectIdException e1 = new InvalidLongObjectIdException( + bs, p, Constants.LONG_OBJECT_ID_STRING_LENGTH); + e1.initCause(e); + throw e1; + } + } + + LongObjectId(final long new_1, final long new_2, final long new_3, + final long new_4) { + w1 = new_1; + w2 = new_2; + w3 = new_3; + w4 = new_4; + } + + /** + * Initialize this instance by copying another existing LongObjectId. + * <p> + * This constructor is mostly useful for subclasses which want to extend a + * LongObjectId with more properties, but initialize from an existing + * LongObjectId instance acquired by other means. + * + * @param src + * another already parsed LongObjectId to copy the value out of. + */ + protected LongObjectId(AnyLongObjectId src) { + w1 = src.w1; + w2 = src.w2; + w3 = src.w3; + w4 = src.w4; + } + + @Override + public LongObjectId toObjectId() { + return this; + } + + private void writeObject(ObjectOutputStream os) throws IOException { + os.writeLong(w1); + os.writeLong(w2); + os.writeLong(w3); + os.writeLong(w4); + } + + private void readObject(ObjectInputStream ois) throws IOException { + w1 = ois.readLong(); + w2 = ois.readLong(); + w3 = ois.readLong(); + w4 = ois.readLong(); + } +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/MutableLongObjectId.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/MutableLongObjectId.java new file mode 100644 index 0000000000..5397d8135c --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/lib/MutableLongObjectId.java @@ -0,0 +1,228 @@ +/* + * Copyright (C) 2015, Matthias Sohn <matthias.sohn@sap.com> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.lfs.lib; + +import java.text.MessageFormat; + +import org.eclipse.jgit.lfs.errors.InvalidLongObjectIdException; +import org.eclipse.jgit.lfs.internal.LfsText; +import org.eclipse.jgit.util.NB; +import org.eclipse.jgit.util.RawParseUtils; + +/** + * A mutable SHA-256 abstraction. + * + * Ported to SHA-256 from {@link org.eclipse.jgit.lib.MutableObjectId} + * + * @since 4.3 + */ +public class MutableLongObjectId extends AnyLongObjectId { + /** + * Empty constructor. Initialize object with default (zeros) value. + */ + public MutableLongObjectId() { + super(); + } + + /** + * Copying constructor. + * + * @param src + * original entry, to copy id from + */ + MutableLongObjectId(MutableLongObjectId src) { + fromObjectId(src); + } + + /** + * Set any byte in the id. + * + * @param index + * index of the byte to set in the raw form of the ObjectId. Must + * be in range [0, + * {@link org.eclipse.jgit.lfs.lib.Constants#LONG_OBJECT_ID_LENGTH}). + * @param value + * the value of the specified byte at {@code index}. Values are + * unsigned and thus are in the range [0,255] rather than the + * signed byte range of [-128, 127]. + * @throws java.lang.ArrayIndexOutOfBoundsException + * {@code index} is less than 0, equal to + * {@link org.eclipse.jgit.lfs.lib.Constants#LONG_OBJECT_ID_LENGTH}, + * or greater than + * {@link org.eclipse.jgit.lfs.lib.Constants#LONG_OBJECT_ID_LENGTH}. + */ + public void setByte(int index, int value) { + switch (index >> 3) { + case 0: + w1 = set(w1, index & 7, value); + break; + case 1: + w2 = set(w2, index & 7, value); + break; + case 2: + w3 = set(w3, index & 7, value); + break; + case 3: + w4 = set(w4, index & 7, value); + break; + default: + throw new ArrayIndexOutOfBoundsException(index); + } + } + + private static long set(long w, int index, long value) { + value &= 0xff; + + switch (index) { + case 0: + return (w & 0x00ffffffffffffffL) | (value << 56); + case 1: + return (w & 0xff00ffffffffffffL) | (value << 48); + case 2: + return (w & 0xffff00ffffffffffL) | (value << 40); + case 3: + return (w & 0xffffff00ffffffffL) | (value << 32); + case 4: + return (w & 0xffffffff00ffffffL) | (value << 24); + case 5: + return (w & 0xffffffffff00ffffL) | (value << 16); + case 6: + return (w & 0xffffffffffff00ffL) | (value << 8); + case 7: + return (w & 0xffffffffffffff00L) | value; + default: + throw new ArrayIndexOutOfBoundsException(); + } + } + + /** + * Make this id match + * {@link org.eclipse.jgit.lfs.lib.LongObjectId#zeroId()}. + */ + public void clear() { + w1 = 0; + w2 = 0; + w3 = 0; + w4 = 0; + } + + /** + * Copy a LongObjectId into this mutable buffer. + * + * @param src + * the source id to copy from. + */ + public void fromObjectId(AnyLongObjectId src) { + this.w1 = src.w1; + this.w2 = src.w2; + this.w3 = src.w3; + this.w4 = src.w4; + } + + /** + * Convert a LongObjectId from raw binary representation. + * + * @param bs + * the raw byte buffer to read from. At least 32 bytes must be + * available within this byte array. + */ + public void fromRaw(byte[] bs) { + fromRaw(bs, 0); + } + + /** + * Convert a LongObjectId from raw binary representation. + * + * @param bs + * the raw byte buffer to read from. At least 32 bytes after p + * must be available within this byte array. + * @param p + * position to read the first byte of data from. + */ + public void fromRaw(byte[] bs, int p) { + w1 = NB.decodeInt64(bs, p); + w2 = NB.decodeInt64(bs, p + 8); + w3 = NB.decodeInt64(bs, p + 16); + w4 = NB.decodeInt64(bs, p + 24); + } + + /** + * Convert a LongObjectId from binary representation expressed in integers. + * + * @param longs + * the raw long buffer to read from. At least 4 longs must be + * available within this longs array. + */ + public void fromRaw(long[] longs) { + fromRaw(longs, 0); + } + + /** + * Convert a LongObjectId from binary representation expressed in longs. + * + * @param longs + * the raw int buffer to read from. At least 4 longs after p must + * be available within this longs array. + * @param p + * position to read the first integer of data from. + */ + public void fromRaw(long[] longs, int p) { + w1 = longs[p]; + w2 = longs[p + 1]; + w3 = longs[p + 2]; + w4 = longs[p + 3]; + } + + /** + * Convert a LongObjectId from hex characters (US-ASCII). + * + * @param buf + * the US-ASCII buffer to read from. At least 32 bytes after + * offset must be available within this byte array. + * @param offset + * position to read the first character from. + */ + public void fromString(byte[] buf, int offset) { + fromHexString(buf, offset); + } + + /** + * Convert a LongObjectId from hex characters. + * + * @param str + * the string to read from. Must be 64 characters long. + */ + public void fromString(String str) { + if (str.length() != Constants.LONG_OBJECT_ID_STRING_LENGTH) + throw new IllegalArgumentException( + MessageFormat.format(LfsText.get().invalidLongId, str)); + fromHexString(org.eclipse.jgit.lib.Constants.encodeASCII(str), 0); + } + + private void fromHexString(byte[] bs, int p) { + try { + w1 = RawParseUtils.parseHexInt64(bs, p); + w2 = RawParseUtils.parseHexInt64(bs, p + 16); + w3 = RawParseUtils.parseHexInt64(bs, p + 32); + w4 = RawParseUtils.parseHexInt64(bs, p + 48); + } catch (ArrayIndexOutOfBoundsException e) { + InvalidLongObjectIdException e1 = new InvalidLongObjectIdException( + bs, p, Constants.LONG_OBJECT_ID_STRING_LENGTH); + e1.initCause(e); + throw e1; + } + } + + @Override + public LongObjectId toObjectId() { + return new LongObjectId(this); + } +} |