diff options
Diffstat (limited to 'org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal')
4 files changed, 725 insertions, 0 deletions
diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/AtomicObjectOutputStream.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/AtomicObjectOutputStream.java new file mode 100644 index 0000000000..009250294e --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/AtomicObjectOutputStream.java @@ -0,0 +1,129 @@ +/* + * Copyright (C) 2015, Matthias Sohn <matthias.sohn@sap.com> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lfs.internal; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Path; +import java.security.DigestOutputStream; +import java.text.MessageFormat; + +import org.eclipse.jgit.annotations.Nullable; +import org.eclipse.jgit.internal.storage.file.LockFile; +import org.eclipse.jgit.lfs.errors.CorruptLongObjectException; +import org.eclipse.jgit.lfs.lib.AnyLongObjectId; +import org.eclipse.jgit.lfs.lib.Constants; +import org.eclipse.jgit.lfs.lib.LongObjectId; + +/** + * Output stream writing content to a + * {@link org.eclipse.jgit.internal.storage.file.LockFile} which is committed on + * close(). The stream checks if the hash of the stream content matches the id. + */ +public class AtomicObjectOutputStream extends OutputStream { + + private LockFile locked; + + private DigestOutputStream out; + + private boolean aborted; + + private AnyLongObjectId id; + + /** + * Constructor for AtomicObjectOutputStream. + * + * @param path + * a {@link java.nio.file.Path} object. + * @param id + * a {@link org.eclipse.jgit.lfs.lib.AnyLongObjectId} object. + * @throws java.io.IOException + * if an IO error occurred + */ + public AtomicObjectOutputStream(Path path, AnyLongObjectId id) + throws IOException { + locked = new LockFile(path.toFile()); + locked.lock(); + this.id = id; + out = new DigestOutputStream(locked.getOutputStream(), + Constants.newMessageDigest()); + } + + /** + * Constructor for AtomicObjectOutputStream. + * + * @param path + * a {@link java.nio.file.Path} object. + * @throws java.io.IOException + * if an IO error occurred + */ + public AtomicObjectOutputStream(Path path) throws IOException { + this(path, null); + } + + /** + * Get the <code>id</code>. + * + * @return content hash of the object which was streamed through this + * stream. May return {@code null} if called before closing this + * stream. + */ + @Nullable + public AnyLongObjectId getId() { + return id; + } + + @Override + public void write(int b) throws IOException { + out.write(b); + } + + @Override + public void write(byte[] b) throws IOException { + out.write(b); + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + out.write(b, off, len); + } + + @Override + public void close() throws IOException { + out.close(); + if (!aborted) { + if (id != null) { + verifyHash(); + } else { + id = LongObjectId.fromRaw(out.getMessageDigest().digest()); + } + locked.commit(); + } + } + + private void verifyHash() { + AnyLongObjectId contentHash = LongObjectId + .fromRaw(out.getMessageDigest().digest()); + if (!contentHash.equals(id)) { + abort(); + throw new CorruptLongObjectException(id, contentHash, + MessageFormat.format(LfsText.get().corruptLongObject, + contentHash, id)); + } + } + + /** + * Aborts the stream. Temporary file will be deleted + */ + public void abort() { + locked.unlock(); + aborted = true; + } +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsConfig.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsConfig.java new file mode 100644 index 0000000000..0469337b1f --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsConfig.java @@ -0,0 +1,226 @@ +/* + * Copyright (C) 2022, Matthias Fromme <mfromme@dspace.de> + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lfs.internal; + +import java.io.File; +import java.io.IOException; + +import org.eclipse.jgit.annotations.Nullable; +import org.eclipse.jgit.dircache.DirCacheEntry; +import org.eclipse.jgit.errors.ConfigInvalidException; +import org.eclipse.jgit.lfs.errors.LfsConfigInvalidException; +import org.eclipse.jgit.lfs.lib.Constants; +import org.eclipse.jgit.lib.BlobBasedConfig; +import org.eclipse.jgit.lib.Config; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevTree; +import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.storage.file.FileBasedConfig; +import org.eclipse.jgit.treewalk.TreeWalk; + +import static org.eclipse.jgit.lib.Constants.HEAD; + +/** + * Encapsulate access to the {@code .lfsconfig}. + * <p> + * According to the git lfs documentation the order to find the + * {@code .lfsconfig} file is: + * </p> + * <ol> + * <li>in the root of the working tree</li> + * <li>in the index</li> + * <li>in the HEAD; for bare repositories this is the only place that is + * searched</li> + * </ol> + * <p> + * Values from the {@code .lfsconfig} are used only if not specified in another + * git config file to allow local override without modifiction of a committed + * file. + * </p> + * + * @see <a href= + * "https://github.com/git-lfs/git-lfs/blob/main/docs/man/git-lfs-config.5.ronn">Configuration + * options for git-lfs</a> + */ +public class LfsConfig { + private Repository db; + private Config delegate; + + /** + * Create a new instance of the LfsConfig. + * + * @param db + * the associated repo + */ + public LfsConfig(Repository db) { + this.db = db; + } + + /** + * Getter for the delegate to allow lazy initialization. + * + * @return the delegate {@link Config} + * @throws IOException + * if an IO error occurred + */ + private Config getDelegate() throws IOException { + if (delegate == null) { + delegate = this.load(); + } + return delegate; + } + + /** + * Read the .lfsconfig file from the repository + * + * An empty config is returned be empty if no lfs config exists. + * + * @return The loaded lfs config + * + * @throws IOException + * if an IO error occurred + */ + private Config load() throws IOException { + Config result = null; + + if (!db.isBare()) { + result = loadFromWorkingTree(); + if (result == null) { + result = loadFromIndex(); + } + } + + if (result == null) { + result = loadFromHead(); + } + + if (result == null) { + result = emptyConfig(); + } + + return result; + } + + /** + * Try to read the lfs config from a file called .lfsconfig at the top level + * of the working tree. + * + * @return the config, or <code>null</code> + * @throws IOException + * if an IO error occurred + */ + @Nullable + private Config loadFromWorkingTree() + throws IOException { + File lfsConfig = db.getFS().resolve(db.getWorkTree(), + Constants.DOT_LFS_CONFIG); + if (lfsConfig.isFile()) { + FileBasedConfig config = new FileBasedConfig(lfsConfig, db.getFS()); + try { + config.load(); + return config; + } catch (ConfigInvalidException e) { + throw new LfsConfigInvalidException( + LfsText.get().dotLfsConfigReadFailed, e); + } + } + return null; + } + + /** + * Try to read the lfs config from an entry called .lfsconfig contained in + * the index. + * + * @return the config, or <code>null</code> if the entry does not exist + * @throws IOException + * if an IO error occurred + */ + @Nullable + private Config loadFromIndex() + throws IOException { + try { + DirCacheEntry entry = db.readDirCache() + .getEntry(Constants.DOT_LFS_CONFIG); + if (entry != null) { + return new BlobBasedConfig(null, db, entry.getObjectId()); + } + } catch (ConfigInvalidException e) { + throw new LfsConfigInvalidException( + LfsText.get().dotLfsConfigReadFailed, e); + } + return null; + } + + /** + * Try to read the lfs config from an entry called .lfsconfig contained in + * the head revision. + * + * @return the config, or <code>null</code> if the file does not exist + * @throws IOException + * if an IO error occurred + */ + @Nullable + private Config loadFromHead() throws IOException { + try (RevWalk revWalk = new RevWalk(db)) { + ObjectId headCommitId = db.resolve(HEAD); + if (headCommitId == null) { + return null; + } + RevCommit commit = revWalk.parseCommit(headCommitId); + RevTree tree = commit.getTree(); + TreeWalk treewalk = TreeWalk.forPath(db, Constants.DOT_LFS_CONFIG, + tree); + if (treewalk != null) { + return new BlobBasedConfig(null, db, treewalk.getObjectId(0)); + } + } catch (ConfigInvalidException e) { + throw new LfsConfigInvalidException( + LfsText.get().dotLfsConfigReadFailed, e); + } + return null; + } + + /** + * Create an empty config as fallback to avoid null pointer checks. + * + * @return an empty config + */ + private Config emptyConfig() { + return new Config(); + } + + /** + * Get string value or null if not found. + * + * First tries to find the value in the git config files. If not found tries + * to find data in .lfsconfig. + * + * @param section + * the section + * @param subsection + * the subsection for the value + * @param name + * the key name + * @return a String value from the config, <code>null</code> if not found + * @throws IOException + * if an IO error occurred + */ + @Nullable + public String getString(final String section, final String subsection, + final String name) throws IOException { + String result = db.getConfig().getString(section, subsection, name); + if (result == null) { + result = getDelegate().getString(section, subsection, name); + } + return result; + } +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsConnectionFactory.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsConnectionFactory.java new file mode 100644 index 0000000000..1a4e85ded6 --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsConnectionFactory.java @@ -0,0 +1,321 @@ +/* + * Copyright (C) 2017, 2022 Markus Duft <markus.duft@ssi-schaefer.com> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lfs.internal; + +import static org.eclipse.jgit.lib.Constants.DEFAULT_REMOTE_NAME; +import static org.eclipse.jgit.util.HttpSupport.ENCODING_GZIP; +import static org.eclipse.jgit.util.HttpSupport.HDR_ACCEPT; +import static org.eclipse.jgit.util.HttpSupport.HDR_ACCEPT_ENCODING; +import static org.eclipse.jgit.util.HttpSupport.HDR_CONTENT_TYPE; + +import java.io.IOException; +import java.net.ProxySelector; +import java.net.URISyntaxException; +import java.net.URL; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Map; +import java.util.TreeMap; + +import org.eclipse.jgit.annotations.NonNull; +import org.eclipse.jgit.errors.CommandFailedException; +import org.eclipse.jgit.lfs.LfsPointer; +import org.eclipse.jgit.lfs.Protocol; +import org.eclipse.jgit.lfs.errors.LfsConfigInvalidException; +import org.eclipse.jgit.lib.ConfigConstants; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.lib.StoredConfig; +import org.eclipse.jgit.transport.HttpConfig; +import org.eclipse.jgit.transport.HttpTransport; +import org.eclipse.jgit.transport.URIish; +import org.eclipse.jgit.transport.http.HttpConnection; +import org.eclipse.jgit.util.HttpSupport; +import org.eclipse.jgit.util.SshSupport; +import org.eclipse.jgit.util.StringUtils; + +/** + * Provides means to get a valid LFS connection for a given repository. + */ +public class LfsConnectionFactory { + private static final int SSH_AUTH_TIMEOUT_SECONDS = 30; + private static final String SCHEME_HTTPS = "https"; //$NON-NLS-1$ + private static final String SCHEME_SSH = "ssh"; //$NON-NLS-1$ + private static final Map<String, AuthCache> sshAuthCache = new TreeMap<>(); + + /** + * Determine URL of LFS server by looking into config parameters lfs.url, + * lfs.[remote].url or remote.[remote].url. The LFS server URL is computed + * from remote.[remote].url by appending "/info/lfs". In case there is no + * URL configured, a SSH remote URI can be used to auto-detect the LFS URI + * by using the remote "git-lfs-authenticate" command. + * + * @param db + * the repository to work with + * @param method + * the method (GET,PUT,...) of the request this connection will + * be used for + * @param purpose + * the action, e.g. Protocol.OPERATION_DOWNLOAD + * @return the connection for the lfs server. e.g. + * "https://github.com/github/git-lfs.git/info/lfs" + * @throws IOException + * if an IO error occurred + */ + public static HttpConnection getLfsConnection(Repository db, String method, + String purpose) throws IOException { + StoredConfig config = db.getConfig(); + Map<String, String> additionalHeaders = new TreeMap<>(); + String lfsUrl = getLfsUrl(db, purpose, additionalHeaders); + URL url = new URL(lfsUrl + Protocol.OBJECTS_LFS_ENDPOINT); + HttpConnection connection = HttpTransport.getConnectionFactory().create( + url, HttpSupport.proxyFor(ProxySelector.getDefault(), url)); + connection.setDoOutput(true); + if (url.getProtocol().equals(SCHEME_HTTPS) + && !config.getBoolean(HttpConfig.HTTP, + HttpConfig.SSL_VERIFY_KEY, true)) { + HttpSupport.disableSslVerify(connection); + } + connection.setRequestMethod(method); + connection.setRequestProperty(HDR_ACCEPT, + Protocol.CONTENTTYPE_VND_GIT_LFS_JSON); + connection.setRequestProperty(HDR_CONTENT_TYPE, + Protocol.CONTENTTYPE_VND_GIT_LFS_JSON); + additionalHeaders + .forEach((k, v) -> connection.setRequestProperty(k, v)); + return connection; + } + + /** + * Get LFS Server URL. + * + * @param db + * the repository to work with + * @param purpose + * the action, e.g. Protocol.OPERATION_DOWNLOAD + * @param additionalHeaders + * additional headers that can be used to connect to LFS server + * @return the URL for the LFS server. e.g. + * "https://github.com/github/git-lfs.git/info/lfs" + * @throws IOException + * if the LFS config is invalid or cannot be accessed + * @see <a href= + * "https://github.com/git-lfs/git-lfs/blob/main/docs/api/server-discovery.md"> + * Server Discovery documentation</a> + */ + private static String getLfsUrl(Repository db, String purpose, + Map<String, String> additionalHeaders) + throws IOException { + LfsConfig config = new LfsConfig(db); + String lfsUrl = config.getString(ConfigConstants.CONFIG_SECTION_LFS, + null, ConfigConstants.CONFIG_KEY_URL); + + Exception ex = null; + if (lfsUrl == null) { + String remoteUrl = null; + for (String remote : db.getRemoteNames()) { + lfsUrl = config.getString(ConfigConstants.CONFIG_SECTION_LFS, + remote, + ConfigConstants.CONFIG_KEY_URL); + + // This could be done better (more precise logic), but according + // to https://github.com/git-lfs/git-lfs/issues/1759 git-lfs + // generally only supports 'origin' in an integrated workflow. + if (lfsUrl == null && remote.equals(DEFAULT_REMOTE_NAME)) { + remoteUrl = config.getString( + ConfigConstants.CONFIG_KEY_REMOTE, remote, + ConfigConstants.CONFIG_KEY_URL); + break; + } + } + if (lfsUrl == null && remoteUrl != null) { + try { + lfsUrl = discoverLfsUrl(db, purpose, additionalHeaders, + remoteUrl); + } catch (URISyntaxException | IOException + | CommandFailedException e) { + ex = e; + } + } + } + if (lfsUrl == null) { + if (ex != null) { + throw new LfsConfigInvalidException( + LfsText.get().lfsNoDownloadUrl, ex); + } + throw new LfsConfigInvalidException(LfsText.get().lfsNoDownloadUrl); + } + return lfsUrl; + } + + private static String discoverLfsUrl(Repository db, String purpose, + Map<String, String> additionalHeaders, String remoteUrl) + throws URISyntaxException, IOException, CommandFailedException { + URIish u = new URIish(remoteUrl); + if (u.getScheme() == null || SCHEME_SSH.equals(u.getScheme())) { + Protocol.ExpiringAction action = getSshAuthentication(db, purpose, + remoteUrl, u); + additionalHeaders.putAll(action.header); + return action.href; + } + return StringUtils.nameWithDotGit(remoteUrl) + + Protocol.INFO_LFS_ENDPOINT; + } + + private static Protocol.ExpiringAction getSshAuthentication( + Repository db, String purpose, String remoteUrl, URIish u) + throws IOException, CommandFailedException { + AuthCache cached = sshAuthCache.get(remoteUrl); + Protocol.ExpiringAction action = null; + if (cached != null && cached.validUntil > System.currentTimeMillis()) { + action = cached.cachedAction; + } + + if (action == null) { + // discover and authenticate; git-lfs does "ssh + // -p <port> -- <host> git-lfs-authenticate + // <project> <upload/download>" + String json = SshSupport.runSshCommand(u.setPath(""), //$NON-NLS-1$ + null, db.getFS(), + "git-lfs-authenticate " + extractProjectName(u) + " " //$NON-NLS-1$//$NON-NLS-2$ + + purpose, + SSH_AUTH_TIMEOUT_SECONDS); + + action = Protocol.gson().fromJson(json, + Protocol.ExpiringAction.class); + + // cache the result as long as possible. + AuthCache c = new AuthCache(action); + sshAuthCache.put(remoteUrl, c); + } + return action; + } + + /** + * Create a connection for the specified + * {@link org.eclipse.jgit.lfs.Protocol.Action}. + * + * @param repo + * the repo to fetch required configuration from + * @param action + * the action for which to create a connection + * @param method + * the target method (GET or PUT) + * @return a connection. output mode is not set. + * @throws IOException + * in case of any error. + */ + @NonNull + public static HttpConnection getLfsContentConnection( + Repository repo, Protocol.Action action, String method) + throws IOException { + URL contentUrl = new URL(action.href); + HttpConnection contentServerConn = HttpTransport.getConnectionFactory() + .create(contentUrl, HttpSupport + .proxyFor(ProxySelector.getDefault(), contentUrl)); + contentServerConn.setRequestMethod(method); + if (action.header != null) { + action.header.forEach( + (k, v) -> contentServerConn.setRequestProperty(k, v)); + } + if (contentUrl.getProtocol().equals(SCHEME_HTTPS) + && !repo.getConfig().getBoolean(HttpConfig.HTTP, + HttpConfig.SSL_VERIFY_KEY, true)) { + HttpSupport.disableSslVerify(contentServerConn); + } + + contentServerConn.setRequestProperty(HDR_ACCEPT_ENCODING, + ENCODING_GZIP); + + return contentServerConn; + } + + private static String extractProjectName(URIish u) { + String path = u.getPath(); + + // begins with a slash if the url contains a port (gerrit vs. github). + if (path.startsWith("/")) { //$NON-NLS-1$ + path = path.substring(1); + } + + if (path.endsWith(org.eclipse.jgit.lib.Constants.DOT_GIT)) { + return path.substring(0, path.length() - 4); + } + return path; + } + + /** + * Create request that can be serialized to JSON + * + * @param operation + * the operation to perform, e.g. Protocol.OPERATION_DOWNLOAD + * @param resources + * the LFS resources affected + * @return a request that can be serialized to JSON + */ + public static Protocol.Request toRequest(String operation, + LfsPointer... resources) { + Protocol.Request req = new Protocol.Request(); + req.operation = operation; + if (resources != null) { + req.objects = new ArrayList<>(); + for (LfsPointer res : resources) { + Protocol.ObjectSpec o = new Protocol.ObjectSpec(); + o.oid = res.getOid().getName(); + o.size = res.getSize(); + req.objects.add(o); + } + } + return req; + } + + private static final class AuthCache { + private static final long AUTH_CACHE_EAGER_TIMEOUT = 500; + + private static final DateTimeFormatter ISO_FORMAT = DateTimeFormatter + .ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSX"); //$NON-NLS-1$ + + /** + * Creates a cache entry for an authentication response. + * <p> + * The timeout of the cache token is extracted from the given action. If + * no timeout can be determined, the token will be used only once. + * + * @param action + * action with an additional expiration timestamp + */ + public AuthCache(Protocol.ExpiringAction action) { + this.cachedAction = action; + try { + if (action.expiresIn != null && !action.expiresIn.isEmpty()) { + this.validUntil = (System.currentTimeMillis() + + Long.parseLong(action.expiresIn)) + - AUTH_CACHE_EAGER_TIMEOUT; + } else if (action.expiresAt != null + && !action.expiresAt.isEmpty()) { + this.validUntil = LocalDateTime + .parse(action.expiresAt, ISO_FORMAT) + .atZone(ZoneOffset.UTC).toInstant().toEpochMilli() + - AUTH_CACHE_EAGER_TIMEOUT; + } else { + this.validUntil = System.currentTimeMillis(); + } + } catch (Exception e) { + this.validUntil = System.currentTimeMillis(); + } + } + + long validUntil; + + Protocol.ExpiringAction cachedAction; + } + +} diff --git a/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsText.java b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsText.java new file mode 100644 index 0000000000..00b34ed3ea --- /dev/null +++ b/org.eclipse.jgit.lfs/src/org/eclipse/jgit/lfs/internal/LfsText.java @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2015, Matthias Sohn <matthias.sohn@sap.com> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lfs.internal; + +import org.eclipse.jgit.nls.NLS; +import org.eclipse.jgit.nls.TranslationBundle; + +/** + * Translation bundle for JGit LFS server + */ +@SuppressWarnings("MissingSummary") +public class LfsText extends TranslationBundle { + + /** + * Get an instance of this translation bundle. + * + * @return an instance of this translation bundle + */ + public static LfsText get() { + return NLS.getBundleFor(LfsText.class); + } + + // @formatter:off + /***/ public String corruptLongObject; + /***/ public String dotLfsConfigReadFailed; + /***/ public String inconsistentContentLength; + /***/ public String inconsistentMediafileLength; + /***/ public String incorrectLONG_OBJECT_ID_LENGTH; + /***/ public String invalidLongId; + /***/ public String invalidLongIdLength; + /***/ public String lfsFailedToGetRepository; + /***/ public String lfsNoDownloadUrl; + /***/ public String lfsUnauthorized; + /***/ public String lfsUnavailable; + /***/ public String missingLocalObject; + /***/ public String protocolError; + /***/ public String repositoryNotFound; + /***/ public String repositoryReadOnly; + /***/ public String requiredHashFunctionNotAvailable; + /***/ public String serverFailure; + /***/ public String wrongAmountOfDataReceived; +} |