123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246 |
- /*
- * Copyright (C) 2016, Christian Halstrick <christian.halstrick@sap.com> and others
- *
- * This program and the accompanying materials are made available under the
- * terms of the Eclipse Distribution License v. 1.0 which is available at
- * https://www.eclipse.org/org/documents/edl-v10.php.
- *
- * SPDX-License-Identifier: BSD-3-Clause
- */
- package org.eclipse.jgit.lfs;
-
- import static java.nio.charset.StandardCharsets.UTF_8;
-
- import java.io.IOException;
- import java.io.InputStream;
- import java.io.InputStreamReader;
- import java.io.OutputStream;
- import java.nio.file.Files;
- import java.nio.file.Path;
- import java.text.MessageFormat;
- import java.util.ArrayList;
- import java.util.Collection;
- import java.util.HashMap;
- import java.util.Map;
-
- import org.eclipse.jgit.attributes.FilterCommand;
- import org.eclipse.jgit.attributes.FilterCommandFactory;
- import org.eclipse.jgit.attributes.FilterCommandRegistry;
- import org.eclipse.jgit.lfs.internal.LfsConnectionFactory;
- import org.eclipse.jgit.lfs.internal.LfsText;
- import org.eclipse.jgit.lfs.lib.AnyLongObjectId;
- import org.eclipse.jgit.lfs.lib.Constants;
- import org.eclipse.jgit.lib.Repository;
- import org.eclipse.jgit.transport.http.HttpConnection;
- import org.eclipse.jgit.util.HttpSupport;
-
- import com.google.gson.Gson;
- import com.google.gson.stream.JsonReader;
-
- /**
- * Built-in LFS smudge filter
- *
- * When content is read from git's object-database and written to the filesystem
- * and this filter is configured for that content, then this filter will replace
- * the content of LFS pointer files with the original content. This happens e.g.
- * when a checkout needs to update a working tree file which is under LFS
- * control.
- *
- * @since 4.6
- */
- public class SmudgeFilter extends FilterCommand {
-
- /**
- * Max number of bytes to copy in a single {@link #run()} call.
- */
- private static final int MAX_COPY_BYTES = 1024 * 1024 * 256;
-
- /**
- * The factory is responsible for creating instances of
- * {@link org.eclipse.jgit.lfs.SmudgeFilter}
- */
- public static final FilterCommandFactory FACTORY = SmudgeFilter::new;
-
- /**
- * Register this filter in JGit
- */
- static void register() {
- FilterCommandRegistry
- .register(org.eclipse.jgit.lib.Constants.BUILTIN_FILTER_PREFIX
- + Constants.ATTR_FILTER_DRIVER_PREFIX
- + org.eclipse.jgit.lib.Constants.ATTR_FILTER_TYPE_SMUDGE,
- FACTORY);
- }
-
- /**
- * Constructor for SmudgeFilter.
- *
- * @param db
- * a {@link org.eclipse.jgit.lib.Repository} object.
- * @param in
- * a {@link java.io.InputStream} object. The stream is closed in
- * any case.
- * @param out
- * a {@link java.io.OutputStream} object.
- * @throws java.io.IOException
- * in case of an error
- */
- public SmudgeFilter(Repository db, InputStream in, OutputStream out)
- throws IOException {
- super(in, out);
- try {
- Lfs lfs = new Lfs(db);
- LfsPointer res = LfsPointer.parseLfsPointer(in);
- if (res != null) {
- AnyLongObjectId oid = res.getOid();
- Path mediaFile = lfs.getMediaFile(oid);
- if (!Files.exists(mediaFile)) {
- downloadLfsResource(lfs, db, res);
- }
- this.in = Files.newInputStream(mediaFile);
- }
- } finally {
- in.close(); // make sure the swapped stream is closed properly.
- }
- }
-
- /**
- * Download content which is hosted on a LFS server
- *
- * @param lfs
- * local {@link Lfs} storage.
- * @param db
- * the repository to work with
- * @param res
- * the objects to download
- * @return the paths of all mediafiles which have been downloaded
- * @throws IOException
- * @since 4.11
- */
- public static Collection<Path> downloadLfsResource(Lfs lfs, Repository db,
- LfsPointer... res) throws IOException {
- Collection<Path> downloadedPaths = new ArrayList<>();
- Map<String, LfsPointer> oidStr2ptr = new HashMap<>();
- for (LfsPointer p : res) {
- oidStr2ptr.put(p.getOid().name(), p);
- }
- HttpConnection lfsServerConn = LfsConnectionFactory.getLfsConnection(db,
- HttpSupport.METHOD_POST, Protocol.OPERATION_DOWNLOAD);
- Gson gson = Protocol.gson();
- lfsServerConn.getOutputStream()
- .write(gson
- .toJson(LfsConnectionFactory
- .toRequest(Protocol.OPERATION_DOWNLOAD, res))
- .getBytes(UTF_8));
- int responseCode = lfsServerConn.getResponseCode();
- if (!(responseCode == HttpConnection.HTTP_OK
- || responseCode == HttpConnection.HTTP_NOT_AUTHORITATIVE)) {
- throw new IOException(
- MessageFormat.format(LfsText.get().serverFailure,
- lfsServerConn.getURL(),
- Integer.valueOf(responseCode)));
- }
- try (JsonReader reader = new JsonReader(
- new InputStreamReader(lfsServerConn.getInputStream(),
- UTF_8))) {
- Protocol.Response resp = gson.fromJson(reader,
- Protocol.Response.class);
- for (Protocol.ObjectInfo o : resp.objects) {
- if (o.error != null) {
- throw new IOException(
- MessageFormat.format(LfsText.get().protocolError,
- Integer.valueOf(o.error.code),
- o.error.message));
- }
- if (o.actions == null) {
- continue;
- }
- LfsPointer ptr = oidStr2ptr.get(o.oid);
- if (ptr == null) {
- // received an object we didn't request
- continue;
- }
- if (ptr.getSize() != o.size) {
- throw new IOException(MessageFormat.format(
- LfsText.get().inconsistentContentLength,
- lfsServerConn.getURL(), Long.valueOf(ptr.getSize()),
- Long.valueOf(o.size)));
- }
- Protocol.Action downloadAction = o.actions
- .get(Protocol.OPERATION_DOWNLOAD);
- if (downloadAction == null || downloadAction.href == null) {
- continue;
- }
-
- HttpConnection contentServerConn = LfsConnectionFactory
- .getLfsContentConnection(db, downloadAction,
- HttpSupport.METHOD_GET);
-
- responseCode = contentServerConn.getResponseCode();
- if (responseCode != HttpConnection.HTTP_OK) {
- throw new IOException(
- MessageFormat.format(LfsText.get().serverFailure,
- contentServerConn.getURL(),
- Integer.valueOf(responseCode)));
- }
- Path path = lfs.getMediaFile(ptr.getOid());
- Path parent = path.getParent();
- if (parent != null) {
- parent.toFile().mkdirs();
- }
- try (InputStream contentIn = contentServerConn
- .getInputStream()) {
- long bytesCopied = Files.copy(contentIn, path);
- if (bytesCopied != o.size) {
- throw new IOException(MessageFormat.format(
- LfsText.get().wrongAmoutOfDataReceived,
- contentServerConn.getURL(),
- Long.valueOf(bytesCopied),
- Long.valueOf(o.size)));
- }
- downloadedPaths.add(path);
- }
- }
- }
- return downloadedPaths;
- }
-
- /** {@inheritDoc} */
- @Override
- public int run() throws IOException {
- try {
- int totalRead = 0;
- int length = 0;
- if (in != null) {
- byte[] buf = new byte[8192];
- while ((length = in.read(buf)) != -1) {
- out.write(buf, 0, length);
- totalRead += length;
-
- // when threshold reached, loop back to the caller.
- // otherwise we could only support files up to 2GB (int
- // return type) properly. we will be called again as long as
- // we don't return -1 here.
- if (totalRead >= MAX_COPY_BYTES) {
- // leave streams open - we need them in the next call.
- return totalRead;
- }
- }
- }
-
- if (totalRead == 0 && length == -1) {
- // we're totally done :) cleanup all streams
- in.close();
- out.close();
- return length;
- }
-
- return totalRead;
- } catch (IOException e) {
- in.close(); // clean up - we swapped this stream.
- out.close();
- throw e;
- }
- }
-
- }
|