diff options
Diffstat (limited to 'org.eclipse.jgit.lfs.server/src/org')
14 files changed, 2058 insertions, 0 deletions
diff --git a/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/LargeFileRepository.java b/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/LargeFileRepository.java new file mode 100644 index 0000000000..4c81baf838 --- /dev/null +++ b/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/LargeFileRepository.java @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2015, Matthias Sohn <matthias.sohn@sap.com> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lfs.server; + +import java.io.IOException; + +import org.eclipse.jgit.annotations.Nullable; +import org.eclipse.jgit.lfs.lib.AnyLongObjectId; + +/** + * Abstraction of a repository for storing large objects + * + * @since 4.3 + */ +public interface LargeFileRepository { + + /** + * Get download action + * + * @param id + * id of the object to download + * @return Action for downloading the object + */ + Response.Action getDownloadAction(AnyLongObjectId id); + + /** + * Get upload action + * + * @param id + * id of the object to upload + * @param size + * size of the object to be uploaded + * @return Action for uploading the object + */ + Response.Action getUploadAction(AnyLongObjectId id, long size); + + /** + * Get verify action + * + * @param id + * id of the object to be verified + * @return Action for verifying the object, or {@code null} if the server + * doesn't support or require verification + */ + @Nullable + Response.Action getVerifyAction(AnyLongObjectId id); + + /** + * Get size of an object + * + * @param id + * id of the object + * @return length of the object content in bytes, -1 if the object doesn't + * exist + * @throws java.io.IOException + * if an IO error occurred + */ + long getSize(AnyLongObjectId id) throws IOException; +} diff --git a/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/LfsObject.java b/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/LfsObject.java new file mode 100644 index 0000000000..ad4726efbf --- /dev/null +++ b/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/LfsObject.java @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2015, Sasa Zivkov <sasa.zivkov@sap.com> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lfs.server; + +/** + * LFS object. + * + * @since 4.5 + */ +public class LfsObject { + String oid; + long size; + + /** + * Get the <code>oid</code> of this object. + * + * @return the object ID. + */ + public String getOid() { + return oid; + } + + /** + * Get the <code>size</code> of this object. + * + * @return the object size. + */ + public long getSize() { + return size; + } +} diff --git a/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/LfsProtocolServlet.java b/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/LfsProtocolServlet.java new file mode 100644 index 0000000000..1d245a0a99 --- /dev/null +++ b/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/LfsProtocolServlet.java @@ -0,0 +1,227 @@ +/* + * Copyright (C) 2015, Sasa Zivkov <sasa.zivkov@sap.com> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lfs.server; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.http.HttpStatus.SC_FORBIDDEN; +import static org.apache.http.HttpStatus.SC_INSUFFICIENT_STORAGE; +import static org.apache.http.HttpStatus.SC_INTERNAL_SERVER_ERROR; +import static org.apache.http.HttpStatus.SC_NOT_FOUND; +import static org.apache.http.HttpStatus.SC_OK; +import static org.apache.http.HttpStatus.SC_SERVICE_UNAVAILABLE; +import static org.apache.http.HttpStatus.SC_UNAUTHORIZED; +import static org.apache.http.HttpStatus.SC_UNPROCESSABLE_ENTITY; +import static org.eclipse.jgit.lfs.lib.Constants.DOWNLOAD; +import static org.eclipse.jgit.lfs.lib.Constants.UPLOAD; +import static org.eclipse.jgit.lfs.lib.Constants.VERIFY; +import static org.eclipse.jgit.util.HttpSupport.HDR_AUTHORIZATION; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.io.Reader; +import java.io.Writer; +import java.text.MessageFormat; +import java.util.List; + +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; + +import org.eclipse.jgit.lfs.errors.LfsBandwidthLimitExceeded; +import org.eclipse.jgit.lfs.errors.LfsException; +import org.eclipse.jgit.lfs.errors.LfsInsufficientStorage; +import org.eclipse.jgit.lfs.errors.LfsRateLimitExceeded; +import org.eclipse.jgit.lfs.errors.LfsRepositoryNotFound; +import org.eclipse.jgit.lfs.errors.LfsRepositoryReadOnly; +import org.eclipse.jgit.lfs.errors.LfsUnauthorized; +import org.eclipse.jgit.lfs.errors.LfsUnavailable; +import org.eclipse.jgit.lfs.errors.LfsValidationError; +import org.eclipse.jgit.lfs.internal.LfsText; +import org.eclipse.jgit.lfs.server.internal.LfsGson; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * LFS protocol handler implementing the LFS batch API [1] + * + * [1] https://github.com/github/git-lfs/blob/master/docs/api/v1/http-v1-batch.md + * + * @since 4.3 + */ +public abstract class LfsProtocolServlet extends HttpServlet { + private static final Logger LOG = LoggerFactory + .getLogger(LfsProtocolServlet.class); + + private static final long serialVersionUID = 1L; + + private static final String CONTENTTYPE_VND_GIT_LFS_JSON = + "application/vnd.git-lfs+json; charset=utf-8"; //$NON-NLS-1$ + + private static final int SC_RATE_LIMIT_EXCEEDED = 429; + + private static final int SC_BANDWIDTH_LIMIT_EXCEEDED = 509; + + /** + * Get the large file repository for the given request and path. + * + * @param request + * the request + * @param path + * the path + * @param auth + * the Authorization HTTP header + * @return the large file repository storing large files. + * @throws org.eclipse.jgit.lfs.errors.LfsException + * implementations should throw more specific exceptions to + * signal which type of error occurred: + * <dl> + * <dt>{@link org.eclipse.jgit.lfs.errors.LfsValidationError}</dt> + * <dd>when there is a validation error with one or more of the + * objects in the request</dd> + * <dt>{@link org.eclipse.jgit.lfs.errors.LfsRepositoryNotFound}</dt> + * <dd>when the repository does not exist for the user</dd> + * <dt>{@link org.eclipse.jgit.lfs.errors.LfsRepositoryReadOnly}</dt> + * <dd>when the user has read, but not write access. Only + * applicable when the operation in the request is "upload"</dd> + * <dt>{@link org.eclipse.jgit.lfs.errors.LfsRateLimitExceeded}</dt> + * <dd>when the user has hit a rate limit with the server</dd> + * <dt>{@link org.eclipse.jgit.lfs.errors.LfsBandwidthLimitExceeded}</dt> + * <dd>when the bandwidth limit for the user or repository has + * been exceeded</dd> + * <dt>{@link org.eclipse.jgit.lfs.errors.LfsInsufficientStorage}</dt> + * <dd>when there is insufficient storage on the server</dd> + * <dt>{@link org.eclipse.jgit.lfs.errors.LfsUnavailable}</dt> + * <dd>when LFS is not available</dd> + * <dt>{@link org.eclipse.jgit.lfs.errors.LfsException}</dt> + * <dd>when an unexpected internal server error occurred</dd> + * </dl> + * @since 4.7 + */ + protected abstract LargeFileRepository getLargeFileRepository( + LfsRequest request, String path, String auth) throws LfsException; + + /** + * LFS request. + * + * @since 4.5 + */ + protected static class LfsRequest { + private String operation; + + private List<LfsObject> objects; + + /** + * Get the LFS operation. + * + * @return the operation + */ + public String getOperation() { + return operation; + } + + /** + * Get the LFS objects. + * + * @return the objects + */ + public List<LfsObject> getObjects() { + return objects; + } + + /** + * Whether operation is upload + * + * @return true if the operation is upload. + * @since 4.7 + */ + public boolean isUpload() { + return operation.equals(UPLOAD); + } + + /** + * Whether the operation is download + * + * @return true if the operation is download. + * @since 4.7 + */ + public boolean isDownload() { + return operation.equals(DOWNLOAD); + } + + /** + * Whether the operation is verify + * + * @return true if the operation is verify. + * @since 4.7 + */ + public boolean isVerify() { + return operation.equals(VERIFY); + } + } + + @Override + protected void doPost(HttpServletRequest req, HttpServletResponse res) + throws ServletException, IOException { + Writer w = new BufferedWriter( + new OutputStreamWriter(res.getOutputStream(), UTF_8)); + + Reader r = new BufferedReader( + new InputStreamReader(req.getInputStream(), UTF_8)); + LfsRequest request = LfsGson.fromJson(r, LfsRequest.class); + String path = req.getPathInfo(); + + res.setContentType(CONTENTTYPE_VND_GIT_LFS_JSON); + LargeFileRepository repo = null; + try { + repo = getLargeFileRepository(request, path, + req.getHeader(HDR_AUTHORIZATION)); + if (repo == null) { + String error = MessageFormat + .format(LfsText.get().lfsFailedToGetRepository, path); + LOG.error(error); + throw new LfsException(error); + } + res.setStatus(SC_OK); + TransferHandler handler = TransferHandler + .forOperation(request.operation, repo, request.objects); + LfsGson.toJson(handler.process(), w); + } catch (LfsValidationError e) { + sendError(res, w, SC_UNPROCESSABLE_ENTITY, e.getMessage()); + } catch (LfsRepositoryNotFound e) { + sendError(res, w, SC_NOT_FOUND, e.getMessage()); + } catch (LfsRepositoryReadOnly e) { + sendError(res, w, SC_FORBIDDEN, e.getMessage()); + } catch (LfsRateLimitExceeded e) { + sendError(res, w, SC_RATE_LIMIT_EXCEEDED, e.getMessage()); + } catch (LfsBandwidthLimitExceeded e) { + sendError(res, w, SC_BANDWIDTH_LIMIT_EXCEEDED, e.getMessage()); + } catch (LfsInsufficientStorage e) { + sendError(res, w, SC_INSUFFICIENT_STORAGE, e.getMessage()); + } catch (LfsUnavailable e) { + sendError(res, w, SC_SERVICE_UNAVAILABLE, e.getMessage()); + } catch (LfsUnauthorized e) { + sendError(res, w, SC_UNAUTHORIZED, e.getMessage()); + } catch (LfsException e) { + sendError(res, w, SC_INTERNAL_SERVER_ERROR, e.getMessage()); + } finally { + w.flush(); + } + } + + private void sendError(HttpServletResponse rsp, Writer writer, int status, + String message) { + rsp.setStatus(status); + LfsGson.toJson(message, writer); + } +} diff --git a/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/Response.java b/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/Response.java new file mode 100644 index 0000000000..1605a786a5 --- /dev/null +++ b/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/Response.java @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2015, Sasa Zivkov <sasa.zivkov@sap.com> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lfs.server; + +import java.util.List; +import java.util.Map; + +/** + * POJOs for Gson serialization/de-serialization. + * + * See the <a href="https://github.com/github/git-lfs/tree/master/docs/api">LFS + * API specification</a> + * + * @since 4.3 + */ +public interface Response { + /** Describes an action the client can execute on a single object */ + class Action { + public String href; + public Map<String, String> header; + } + + // TODO(ms): rename this class in next major release + @SuppressWarnings("JavaLangClash") + /** Describes an error to be returned by the LFS batch API */ + class Error { + public int code; + public String message; + } + + /** Describes the actions the LFS server offers for a single object */ + class ObjectInfo { + public String oid; + public long size; + public Map<String, Action> actions; + public Error error; + } + + /** Describes the body of a LFS batch API response */ + class Body { + public List<ObjectInfo> objects; + } +} diff --git a/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/TransferHandler.java b/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/TransferHandler.java new file mode 100644 index 0000000000..4662830689 --- /dev/null +++ b/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/TransferHandler.java @@ -0,0 +1,134 @@ +/* + * Copyright (C) 2015, Sasa Zivkov <sasa.zivkov@sap.com> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.lfs.server; + +import static jakarta.servlet.http.HttpServletResponse.SC_NOT_FOUND; +import static org.eclipse.jgit.lfs.lib.Constants.DOWNLOAD; +import static org.eclipse.jgit.lfs.lib.Constants.UPLOAD; +import static org.eclipse.jgit.lfs.lib.Constants.VERIFY; + +import java.io.IOException; +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import org.eclipse.jgit.lfs.lib.LongObjectId; +import org.eclipse.jgit.lfs.server.Response.Action; +import org.eclipse.jgit.lfs.server.Response.Body; +import org.eclipse.jgit.lfs.server.internal.LfsServerText; + +abstract class TransferHandler { + + static TransferHandler forOperation(String operation, + LargeFileRepository repository, List<LfsObject> objects) { + switch (operation) { + case UPLOAD: + return new Upload(repository, objects); + case DOWNLOAD: + return new Download(repository, objects); + case VERIFY: + default: + throw new UnsupportedOperationException(MessageFormat.format( + LfsServerText.get().unsupportedOperation, operation)); + } + } + + private static class Upload extends TransferHandler { + Upload(LargeFileRepository repository, + List<LfsObject> objects) { + super(repository, objects); + } + + @Override + Body process() throws IOException { + Response.Body body = new Response.Body(); + if (!objects.isEmpty()) { + body.objects = new ArrayList<>(); + for (LfsObject o : objects) { + addObjectInfo(body, o); + } + } + return body; + } + + private void addObjectInfo(Response.Body body, LfsObject o) + throws IOException { + Response.ObjectInfo info = new Response.ObjectInfo(); + body.objects.add(info); + info.oid = o.oid; + info.size = o.size; + + LongObjectId oid = LongObjectId.fromString(o.oid); + if (repository.getSize(oid) == -1) { + info.actions = new HashMap<>(); + info.actions.put(UPLOAD, + repository.getUploadAction(oid, o.size)); + Action verify = repository.getVerifyAction(oid); + if (verify != null) { + info.actions.put(VERIFY, verify); + } + } + } + } + + private static class Download extends TransferHandler { + Download(LargeFileRepository repository, + List<LfsObject> objects) { + super(repository, objects); + } + + @Override + Body process() throws IOException { + Response.Body body = new Response.Body(); + if (!objects.isEmpty()) { + body.objects = new ArrayList<>(); + for (LfsObject o : objects) { + addObjectInfo(body, o); + } + } + return body; + } + + private void addObjectInfo(Response.Body body, LfsObject o) + throws IOException { + Response.ObjectInfo info = new Response.ObjectInfo(); + body.objects.add(info); + info.oid = o.oid; + info.size = o.size; + + LongObjectId oid = LongObjectId.fromString(o.oid); + if (repository.getSize(oid) >= 0) { + info.actions = new HashMap<>(); + info.actions.put(DOWNLOAD, + repository.getDownloadAction(oid)); + } else { + info.error = new Response.Error(); + info.error.code = SC_NOT_FOUND; + info.error.message = MessageFormat.format( + LfsServerText.get().objectNotFound, + oid.getName()); + } + } + } + + final LargeFileRepository repository; + + final List<LfsObject> objects; + + TransferHandler(LargeFileRepository repository, + List<LfsObject> objects) { + this.repository = repository; + this.objects = objects; + } + + abstract Response.Body process() throws IOException; +} diff --git a/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/fs/FileLfsRepository.java b/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/fs/FileLfsRepository.java new file mode 100644 index 0000000000..ff648aaebf --- /dev/null +++ b/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/fs/FileLfsRepository.java @@ -0,0 +1,169 @@ +/* + * Copyright (C) 2015, Matthias Sohn <matthias.sohn@sap.com> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lfs.server.fs; + +import static org.eclipse.jgit.util.HttpSupport.HDR_AUTHORIZATION; + +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.channels.ReadableByteChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Collections; + +import org.eclipse.jgit.annotations.Nullable; +import org.eclipse.jgit.lfs.internal.AtomicObjectOutputStream; +import org.eclipse.jgit.lfs.lib.AnyLongObjectId; +import org.eclipse.jgit.lfs.lib.Constants; +import org.eclipse.jgit.lfs.server.LargeFileRepository; +import org.eclipse.jgit.lfs.server.Response; +import org.eclipse.jgit.lfs.server.Response.Action; + +/** + * Repository storing large objects in the file system + * + * @since 4.3 + */ +public class FileLfsRepository implements LargeFileRepository { + + private String url; + private final Path dir; + + /** + * <p> + * Constructor for FileLfsRepository. + * </p> + * + * @param url + * external URL of this repository + * @param dir + * storage directory + * @throws java.io.IOException + * if an IO error occurred + */ + public FileLfsRepository(String url, Path dir) throws IOException { + this.url = url; + this.dir = dir; + Files.createDirectories(dir); + } + + @Override + public Response.Action getDownloadAction(AnyLongObjectId id) { + return getAction(id); + } + + @Override + public Action getUploadAction(AnyLongObjectId id, long size) { + return getAction(id); + } + + @Override + @Nullable + public Action getVerifyAction(AnyLongObjectId id) { + return null; + } + + @Override + public long getSize(AnyLongObjectId id) throws IOException { + Path p = getPath(id); + if (Files.exists(p)) { + return Files.size(p); + } + return -1; + } + + /** + * Get the storage directory + * + * @return the path of the storage directory + */ + public Path getDir() { + return dir; + } + + /** + * Get the path where the given object is stored + * + * @param id + * id of a large object + * @return path the object's storage path + */ + protected Path getPath(AnyLongObjectId id) { + StringBuilder s = new StringBuilder( + Constants.LONG_OBJECT_ID_STRING_LENGTH + 6); + s.append(toHexCharArray(id.getFirstByte())).append('/'); + s.append(toHexCharArray(id.getSecondByte())).append('/'); + s.append(id.name()); + return dir.resolve(s.toString()); + } + + private Response.Action getAction(AnyLongObjectId id) { + Response.Action a = new Response.Action(); + a.href = url + id.getName(); + a.header = Collections.singletonMap(HDR_AUTHORIZATION, "not:required"); //$NON-NLS-1$ + return a; + } + + ReadableByteChannel getReadChannel(AnyLongObjectId id) + throws IOException { + return FileChannel.open(getPath(id), StandardOpenOption.READ); + } + + AtomicObjectOutputStream getOutputStream(AnyLongObjectId id) + throws IOException { + Path path = getPath(id); + Path parent = path.getParent(); + if (parent != null) { + Files.createDirectories(parent); + } + return new AtomicObjectOutputStream(path, id); + } + + private static char[] toHexCharArray(int b) { + final char[] dst = new char[2]; + formatHexChar(dst, 0, b); + return dst; + } + + private static final char[] hexchar = { '0', '1', '2', '3', '4', '5', '6', + '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' }; + + private static void formatHexChar(char[] dst, int p, int b) { + int o = p + 1; + while (o >= p && b != 0) { + dst[o--] = hexchar[b & 0xf]; + b >>>= 4; + } + while (o >= p) + dst[o--] = '0'; + } + + /** + * Get URL of content server + * + * @return the url of the content server + * @since 4.11 + */ + public String getUrl() { + return url; + } + + /** + * Set the URL of the content server + * + * @param url + * the url of the content server + * @since 4.11 + */ + public void setUrl(String url) { + this.url = url; + } +} diff --git a/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/fs/FileLfsServlet.java b/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/fs/FileLfsServlet.java new file mode 100644 index 0000000000..e95122da59 --- /dev/null +++ b/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/fs/FileLfsServlet.java @@ -0,0 +1,159 @@ +/* + * Copyright (C) 2015, Matthias Sohn <matthias.sohn@sap.com> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lfs.server.fs; + +import java.io.IOException; +import java.io.PrintWriter; +import java.text.MessageFormat; + +import jakarta.servlet.AsyncContext; +import jakarta.servlet.ServletException; +import jakarta.servlet.annotation.WebServlet; +import jakarta.servlet.http.HttpServlet; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; + +import org.apache.http.HttpStatus; +import org.eclipse.jgit.lfs.errors.InvalidLongObjectIdException; +import org.eclipse.jgit.lfs.lib.AnyLongObjectId; +import org.eclipse.jgit.lfs.lib.Constants; +import org.eclipse.jgit.lfs.lib.LongObjectId; +import org.eclipse.jgit.lfs.server.internal.LfsGson; +import org.eclipse.jgit.lfs.server.internal.LfsServerText; + +/** + * Servlet supporting upload and download of large objects as defined by the + * GitHub Large File Storage extension API extending git to allow separate + * storage of large files + * (https://github.com/github/git-lfs/tree/master/docs/api). + * + * @since 4.3 + */ +@WebServlet(asyncSupported = true) +public class FileLfsServlet extends HttpServlet { + + private static final long serialVersionUID = 1L; + + private final FileLfsRepository repository; + + private final long timeout; + + /** + * <p>Constructor for FileLfsServlet.</p> + * + * @param repository + * the repository storing the large objects + * @param timeout + * timeout for object upload / download in milliseconds + */ + public FileLfsServlet(FileLfsRepository repository, long timeout) { + this.repository = repository; + this.timeout = timeout; + } + + /** + * {@inheritDoc} + * + * Handle object downloads + */ + @Override + protected void doGet(HttpServletRequest req, + HttpServletResponse rsp) throws ServletException, IOException { + AnyLongObjectId obj = getObjectToTransfer(req, rsp); + if (obj != null) { + if (repository.getSize(obj) == -1) { + sendError(rsp, HttpStatus.SC_NOT_FOUND, MessageFormat + .format(LfsServerText.get().objectNotFound, + obj.getName())); + return; + } + AsyncContext context = req.startAsync(); + context.setTimeout(timeout); + rsp.getOutputStream() + .setWriteListener(new ObjectDownloadListener(repository, + context, rsp, obj)); + } + } + + /** + * Retrieve object id from request + * + * @param req + * servlet request + * @param rsp + * servlet response + * @return object id, or <code>null</code> if the object id could not be + * retrieved + * @throws java.io.IOException + * if an I/O error occurs + * @since 7.0 + */ + protected AnyLongObjectId getObjectToTransfer(HttpServletRequest req, + HttpServletResponse rsp) throws IOException { + String info = req.getPathInfo(); + int length = 1 + Constants.LONG_OBJECT_ID_STRING_LENGTH; + if (info.length() != length) { + sendError(rsp, HttpStatus.SC_UNPROCESSABLE_ENTITY, MessageFormat + .format(LfsServerText.get().invalidPathInfo, info)); + return null; + } + try { + return LongObjectId.fromString(info.substring(1, length)); + } catch (InvalidLongObjectIdException e) { + sendError(rsp, HttpStatus.SC_UNPROCESSABLE_ENTITY, e.getMessage()); + return null; + } + } + + /** + * {@inheritDoc} + * + * Handle object uploads + */ + @Override + protected void doPut(HttpServletRequest req, + HttpServletResponse rsp) throws ServletException, IOException { + AnyLongObjectId id = getObjectToTransfer(req, rsp); + if (id != null) { + AsyncContext context = req.startAsync(); + context.setTimeout(timeout); + req.getInputStream().setReadListener(new ObjectUploadListener( + repository, context, req, rsp, id)); + } + } + + /** + * Send an error response. + * + * @param rsp + * the servlet response + * @param status + * HTTP status code + * @param message + * error message + * @throws java.io.IOException + * on failure to send the response + * @since 7.0 + */ + protected static void sendError(HttpServletResponse rsp, int status, String message) + throws IOException { + if (rsp.isCommitted()) { + rsp.getOutputStream().close(); + return; + } + rsp.reset(); + rsp.setStatus(status); + try (PrintWriter writer = rsp.getWriter()) { + LfsGson.toJson(message, writer); + writer.flush(); + } + rsp.flushBuffer(); + } +} diff --git a/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/fs/ObjectDownloadListener.java b/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/fs/ObjectDownloadListener.java new file mode 100644 index 0000000000..c826aa66eb --- /dev/null +++ b/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/fs/ObjectDownloadListener.java @@ -0,0 +1,142 @@ +/* + * Copyright (C) 2015, Matthias Sohn <matthias.sohn@sap.com> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lfs.server.fs; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.nio.channels.ReadableByteChannel; +import java.nio.channels.WritableByteChannel; +import java.util.logging.Level; +import java.util.logging.Logger; + +import jakarta.servlet.AsyncContext; +import jakarta.servlet.ServletOutputStream; +import jakarta.servlet.WriteListener; +import jakarta.servlet.http.HttpServletResponse; + +import org.apache.http.HttpStatus; +import org.eclipse.jgit.lfs.lib.AnyLongObjectId; +import org.eclipse.jgit.lfs.lib.Constants; +import org.eclipse.jgit.util.HttpSupport; + +/** + * Handle asynchronous large object download. + * + * @since 4.7 + */ +public class ObjectDownloadListener implements WriteListener { + + private static final Logger LOG = Logger + .getLogger(ObjectDownloadListener.class.getName()); + + private final AsyncContext context; + + private final HttpServletResponse response; + + private final ServletOutputStream out; + + private final ReadableByteChannel in; + + private final WritableByteChannel outChannel; + + private ByteBuffer buffer = ByteBuffer.allocateDirect(8192); + + /** + * <p> + * Constructor for ObjectDownloadListener. + * </p> + * + * @param repository + * the repository storing large objects + * @param context + * the servlet asynchronous context + * @param response + * the servlet response + * @param id + * id of the object to be downloaded + * @throws java.io.IOException + * if an IO error occurred + * @since 7.0 + */ + public ObjectDownloadListener(FileLfsRepository repository, + AsyncContext context, HttpServletResponse response, + AnyLongObjectId id) throws IOException { + this.context = context; + this.response = response; + this.in = repository.getReadChannel(id); + this.out = response.getOutputStream(); + this.outChannel = Channels.newChannel(out); + + response.addHeader(HttpSupport.HDR_CONTENT_LENGTH, + String.valueOf(repository.getSize(id))); + response.setContentType(Constants.HDR_APPLICATION_OCTET_STREAM); + } + + /** + * {@inheritDoc} + * + * Write file content + */ + @SuppressWarnings("Finally") + @Override + public void onWritePossible() throws IOException { + while (out.isReady()) { + try { + buffer.clear(); + if (in.read(buffer) < 0) { + buffer = null; + } else { + buffer.flip(); + } + } catch (Throwable t) { + LOG.log(Level.SEVERE, t.getMessage(), t); + buffer = null; + } finally { + if (buffer != null) { + outChannel.write(buffer); + } else { + try { + in.close(); + } catch (IOException e) { + LOG.log(Level.SEVERE, e.getMessage(), e); + } + try { + out.close(); + } finally { + context.complete(); + } + // This is need to avoid endless loop in recent Jetty versions. + // That's because out.isReady() is returning true for already + // closed streams and because out.close() doesn't throw any + // exception any more when trying to close already closed stream. + return; + } + } + } + } + + /** + * {@inheritDoc} + * + * Handle errors + */ + @Override + public void onError(Throwable e) { + try { + FileLfsServlet.sendError(response, + HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getMessage()); + context.complete(); + in.close(); + } catch (IOException ex) { + LOG.log(Level.SEVERE, ex.getMessage(), ex); + } + } +} diff --git a/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/fs/ObjectUploadListener.java b/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/fs/ObjectUploadListener.java new file mode 100644 index 0000000000..d0c07fb25a --- /dev/null +++ b/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/fs/ObjectUploadListener.java @@ -0,0 +1,199 @@ +/* + * Copyright (C) 2015, Matthias Sohn <matthias.sohn@sap.com> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lfs.server.fs; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.nio.channels.ReadableByteChannel; +import java.nio.channels.WritableByteChannel; +import java.nio.file.Path; +import java.util.logging.Level; +import java.util.logging.Logger; + +import jakarta.servlet.AsyncContext; +import jakarta.servlet.ReadListener; +import jakarta.servlet.ServletInputStream; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; + +import org.apache.http.HttpStatus; +import org.eclipse.jgit.lfs.errors.CorruptLongObjectException; +import org.eclipse.jgit.lfs.internal.AtomicObjectOutputStream; +import org.eclipse.jgit.lfs.lib.AnyLongObjectId; +import org.eclipse.jgit.lfs.lib.Constants; + +/** + * Handle asynchronous object upload. + * + * @since 4.6 + */ +public class ObjectUploadListener implements ReadListener { + + private static final Logger LOG = Logger + .getLogger(ObjectUploadListener.class.getName()); + + private final AsyncContext context; + + private final HttpServletResponse response; + + private final ServletInputStream in; + + private final ReadableByteChannel inChannel; + + private final AtomicObjectOutputStream out; + + private WritableByteChannel channel; + + private final ByteBuffer buffer = ByteBuffer.allocateDirect(8192); + + private final Path path; + + private long uploaded; + + private Callback callback; + + /** + * Callback invoked after object upload completed. + * + * @since 5.1.7 + */ + public interface Callback { + /** + * Notified after object upload completed. + * + * @param path + * path to the object on the backend + * @param size + * uploaded size in bytes + */ + void uploadCompleted(String path, long size); + } + + /** + * Constructor for ObjectUploadListener. + * + * @param repository + * the repository storing large objects + * @param context + * a {@link jakarta.servlet.AsyncContext} object. + * @param request + * a {@link jakarta.servlet.http.HttpServletRequest} object. + * @param response + * a {@link jakarta.servlet.http.HttpServletResponse} object. + * @param id + * a {@link org.eclipse.jgit.lfs.lib.AnyLongObjectId} object. + * @throws java.io.FileNotFoundException + * if file wasn't found + * @throws java.io.IOException + * if an IO error occurred + * @since 7.0 + */ + public ObjectUploadListener(FileLfsRepository repository, + AsyncContext context, HttpServletRequest request, + HttpServletResponse response, AnyLongObjectId id) + throws FileNotFoundException, IOException { + this.context = context; + this.response = response; + this.in = request.getInputStream(); + this.inChannel = Channels.newChannel(in); + this.out = repository.getOutputStream(id); + this.channel = Channels.newChannel(out); + this.path = repository.getPath(id); + this.uploaded = 0L; + response.setContentType(Constants.CONTENT_TYPE_GIT_LFS_JSON); + } + + /** + * Set the callback to invoke after upload completed. + * + * @param callback + * the callback + * @return {@code this}. + * @since 5.1.7 + */ + public ObjectUploadListener setCallback(Callback callback) { + this.callback = callback; + return this; + } + + /** + * {@inheritDoc} + * + * Writes all the received data to the output channel + */ + @Override + public void onDataAvailable() throws IOException { + while (in.isReady()) { + if (inChannel.read(buffer) > 0) { + buffer.flip(); + uploaded += Integer.valueOf(channel.write(buffer)).longValue(); + buffer.compact(); + } else { + buffer.flip(); + while (buffer.hasRemaining()) { + uploaded += Integer.valueOf(channel.write(buffer)) + .longValue(); + } + close(); + return; + } + } + } + + @Override + public void onAllDataRead() throws IOException { + close(); + } + + /** + * Close resources held by this listener + * + * @throws java.io.IOException + * if an IO error occurred + */ + protected void close() throws IOException { + try { + inChannel.close(); + channel.close(); + // TODO check if status 200 is ok for PUT request, HTTP foresees 204 + // for successful PUT without response body + if (!response.isCommitted()) { + response.setStatus(HttpServletResponse.SC_OK); + } + if (callback != null) { + callback.uploadCompleted(path.toString(), uploaded); + } + } finally { + context.complete(); + } + } + + @Override + public void onError(Throwable e) { + try { + out.abort(); + inChannel.close(); + channel.close(); + int status; + if (e instanceof CorruptLongObjectException) { + status = HttpStatus.SC_BAD_REQUEST; + LOG.log(Level.WARNING, e.getMessage(), e); + } else { + status = HttpStatus.SC_INTERNAL_SERVER_ERROR; + LOG.log(Level.SEVERE, e.getMessage(), e); + } + FileLfsServlet.sendError(response, status, e.getMessage()); + } catch (IOException ex) { + LOG.log(Level.SEVERE, ex.getMessage(), ex); + } + } +} diff --git a/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/internal/LfsGson.java b/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/internal/LfsGson.java new file mode 100644 index 0000000000..c7e45043de --- /dev/null +++ b/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/internal/LfsGson.java @@ -0,0 +1,87 @@ +/* + * Copyright (C) 2017, David Pursehouse <david.pursehouse@gmail.com> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +package org.eclipse.jgit.lfs.server.internal; + +import java.io.Reader; + +import com.google.gson.FieldNamingPolicy; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonIOException; +import com.google.gson.JsonSyntaxException; + +/** + * Wrapper for {@link com.google.gson.Gson} used by LFS servlets. + */ +public class LfsGson { + private static final Gson gson = new GsonBuilder() + .setFieldNamingPolicy(FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES) + .disableHtmlEscaping() + .create(); + + /** + * Wrapper class only used for serialization of error messages. + */ + // TODO(ms): rename this class in next major release + @SuppressWarnings("JavaLangClash") + static class Error { + String message; + + Error(String m) { + this.message = m; + } + } + + /** + * Serializes the specified object into its equivalent Json representation. + * + * @param src + * the object for which Json representation is to be created. If + * this is a String, it is wrapped in an instance of + * {@link org.eclipse.jgit.lfs.server.internal.LfsGson.Error}. + * @param writer + * Writer to which the Json representation needs to be written + * @throws com.google.gson.JsonIOException + * if there was a problem writing to the writer + * @see Gson#toJson(Object, Appendable) + */ + public static void toJson(Object src, Appendable writer) + throws JsonIOException { + if (src instanceof String) { + gson.toJson(new Error((String) src), writer); + } else { + gson.toJson(src, writer); + } + } + + /** + * Deserializes the Json read from the specified reader into an object of + * the specified type. + * + * @param json + * reader producing json from which the object is to be + * deserialized + * @param classOfT + * specified type to deserialize + * @return an Object of type T + * @throws com.google.gson.JsonIOException + * if there was a problem reading from the Reader + * @throws com.google.gson.JsonSyntaxException + * if json is not a valid representation for an object of type + * @see Gson#fromJson(Reader, java.lang.reflect.Type) + * @param <T> + * a T object. + */ + public static <T> T fromJson(Reader json, Class<T> classOfT) + throws JsonSyntaxException, JsonIOException { + return gson.fromJson(json, classOfT); + } +} diff --git a/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/internal/LfsServerText.java b/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/internal/LfsServerText.java new file mode 100644 index 0000000000..47c778d13b --- /dev/null +++ b/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/internal/LfsServerText.java @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2015, Matthias Sohn <matthias.sohn@sap.com> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lfs.server.internal; + +import org.eclipse.jgit.nls.NLS; +import org.eclipse.jgit.nls.TranslationBundle; + +/** + * Translation bundle for JGit LFS server + */ +@SuppressWarnings("MissingSummary") +public class LfsServerText extends TranslationBundle { + + /** + * Get an instance of this translation bundle + * + * @return an instance of this translation bundle + */ + public static LfsServerText get() { + return NLS.getBundleFor(LfsServerText.class); + } + + // @formatter:off + /***/ public String failedToCalcSignature; + /***/ public String invalidPathInfo; + /***/ public String objectNotFound; + /***/ public String undefinedS3AccessKey; + /***/ public String undefinedS3Bucket; + /***/ public String undefinedS3Region; + /***/ public String undefinedS3Hostname; + /***/ public String undefinedS3SecretKey; + /***/ public String undefinedS3StorageClass; + /***/ public String unparsableEndpoint; + /***/ public String unsupportedOperation; + /***/ public String unsupportedUtf8; +} diff --git a/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/s3/S3Config.java b/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/s3/S3Config.java new file mode 100644 index 0000000000..9b44aebe2a --- /dev/null +++ b/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/s3/S3Config.java @@ -0,0 +1,169 @@ +/* + * Copyright (C) 2015, Matthias Sohn <matthias.sohn@sap.com> + * Copyright (C) 2015, Sasa Zivkov <sasa.zivkov@sap.com> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lfs.server.s3; + +/** + * Configuration for an Amazon AWS S3 bucket + * + * @since 4.3 + */ +public class S3Config { + private final String hostname; + private final String region; + private final String bucket; + private final String storageClass; + private final String accessKey; + private final String secretKey; + private final int expirationSeconds; + private final boolean disableSslVerify; + + /** + * <p> + * Constructor for S3Config. + * </p> + * + * @param hostname + * S3 API host + * @param region + * AWS region + * @param bucket + * S3 storage bucket + * @param storageClass + * S3 storage class + * @param accessKey + * access key for authenticating to AWS + * @param secretKey + * secret key for authenticating to AWS + * @param expirationSeconds + * period in seconds after which requests signed for this bucket + * will expire + * @param disableSslVerify + * if {@code true} disable Amazon server certificate and hostname + * verification + * @since 5.8 + */ + public S3Config(String hostname, String region, String bucket, String storageClass, + String accessKey, String secretKey, int expirationSeconds, + boolean disableSslVerify) { + this.hostname = hostname; + this.region = region; + this.bucket = bucket; + this.storageClass = storageClass; + this.accessKey = accessKey; + this.secretKey = secretKey; + this.expirationSeconds = expirationSeconds; + this.disableSslVerify = disableSslVerify; + } + + /** + * <p>Constructor for S3Config.</p> + * + * @param region + * AWS region + * @param bucket + * S3 storage bucket + * @param storageClass + * S3 storage class + * @param accessKey + * access key for authenticating to AWS + * @param secretKey + * secret key for authenticating to AWS + * @param expirationSeconds + * period in seconds after which requests signed for this bucket + * will expire + * @param disableSslVerify + * if {@code true} disable Amazon server certificate and hostname + * verification + */ + public S3Config(String region, String bucket, String storageClass, + String accessKey, String secretKey, int expirationSeconds, + boolean disableSslVerify) { + this(String.format("s3-%s.amazonaws.com", region), region, bucket, //$NON-NLS-1$ + storageClass, accessKey, secretKey, expirationSeconds, + disableSslVerify); + } + + /** + * Get the <code>hostname</code>. + * + * @return Get the S3 API host + * @since 5.8 + */ + public String getHostname() { + return hostname; + } + + /** + * Get the <code>region</code>. + * + * @return Get name of AWS region this bucket resides in + */ + public String getRegion() { + return region; + } + + /** + * Get the <code>bucket</code>. + * + * @return Get S3 storage bucket name + */ + public String getBucket() { + return bucket; + } + + /** + * Get the <code>storageClass</code>. + * + * @return S3 storage class to use for objects stored in this bucket + */ + public String getStorageClass() { + return storageClass; + } + + /** + * Get the <code>accessKey</code>. + * + * @return access key for authenticating to AWS + */ + public String getAccessKey() { + return accessKey; + } + + /** + * Get the <code>secretKey</code>. + * + * @return secret key for authenticating to AWS + */ + public String getSecretKey() { + return secretKey; + } + + /** + * Get the <code>expirationSeconds</code>. + * + * @return period in seconds after which requests signed for this bucket + * will expire + */ + public int getExpirationSeconds() { + return expirationSeconds; + } + + /** + * Whether SSL verification is disabled + * + * @return {@code true} if Amazon server certificate and hostname + * verification is disabled + */ + boolean isDisableSslVerify() { + return disableSslVerify; + } + +} diff --git a/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/s3/S3Repository.java b/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/s3/S3Repository.java new file mode 100644 index 0000000000..01ddc95310 --- /dev/null +++ b/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/s3/S3Repository.java @@ -0,0 +1,186 @@ +/* + * Copyright (C) 2015, Matthias Sohn <matthias.sohn@sap.com> + * Copyright (C) 2015, Sasa Zivkov <sasa.zivkov@sap.com> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lfs.server.s3; + +import static jakarta.servlet.http.HttpServletResponse.SC_OK; +import static org.eclipse.jgit.lfs.server.s3.SignerV4.UNSIGNED_PAYLOAD; +import static org.eclipse.jgit.lfs.server.s3.SignerV4.X_AMZ_CONTENT_SHA256; +import static org.eclipse.jgit.lfs.server.s3.SignerV4.X_AMZ_EXPIRES; +import static org.eclipse.jgit.lfs.server.s3.SignerV4.X_AMZ_STORAGE_CLASS; +import static org.eclipse.jgit.util.HttpSupport.HDR_CONTENT_LENGTH; +import static org.eclipse.jgit.util.HttpSupport.METHOD_GET; +import static org.eclipse.jgit.util.HttpSupport.METHOD_HEAD; +import static org.eclipse.jgit.util.HttpSupport.METHOD_PUT; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.Proxy; +import java.net.ProxySelector; +import java.net.URL; +import java.text.MessageFormat; +import java.util.HashMap; +import java.util.Map; + +import org.eclipse.jgit.lfs.lib.AnyLongObjectId; +import org.eclipse.jgit.lfs.server.LargeFileRepository; +import org.eclipse.jgit.lfs.server.Response; +import org.eclipse.jgit.lfs.server.Response.Action; +import org.eclipse.jgit.lfs.server.internal.LfsServerText; +import org.eclipse.jgit.transport.http.HttpConnection; +import org.eclipse.jgit.transport.http.apache.HttpClientConnectionFactory; +import org.eclipse.jgit.util.HttpSupport; + +/** + * Repository storing LFS objects in Amazon S3 + * + * @since 4.3 + */ +public class S3Repository implements LargeFileRepository { + + private S3Config s3Config; + + /** + * Construct a LFS repository storing large objects in Amazon S3 + * + * @param config + * AWS S3 storage bucket configuration + */ + public S3Repository(S3Config config) { + validateConfig(config); + this.s3Config = config; + } + + @Override + public Response.Action getDownloadAction(AnyLongObjectId oid) { + URL endpointUrl = getObjectUrl(oid); + Map<String, String> queryParams = new HashMap<>(); + queryParams.put(X_AMZ_EXPIRES, + Integer.toString(s3Config.getExpirationSeconds())); + Map<String, String> headers = new HashMap<>(); + String authorizationQueryParameters = SignerV4.createAuthorizationQuery( + s3Config, endpointUrl, METHOD_GET, headers, queryParams, + UNSIGNED_PAYLOAD); + + Response.Action a = new Response.Action(); + a.href = endpointUrl.toString() + "?" + authorizationQueryParameters; //$NON-NLS-1$ + return a; + } + + @Override + public Response.Action getUploadAction(AnyLongObjectId oid, long size) { + cacheObjectMetaData(oid, size); + URL objectUrl = getObjectUrl(oid); + Map<String, String> headers = new HashMap<>(); + headers.put(X_AMZ_CONTENT_SHA256, oid.getName()); + headers.put(HDR_CONTENT_LENGTH, Long.toString(size)); + headers.put(X_AMZ_STORAGE_CLASS, s3Config.getStorageClass()); + headers.put(HttpSupport.HDR_CONTENT_TYPE, "application/octet-stream"); //$NON-NLS-1$ + headers = SignerV4.createHeaderAuthorization(s3Config, objectUrl, + METHOD_PUT, headers, oid.getName()); + + Response.Action a = new Response.Action(); + a.href = objectUrl.toString(); + a.header = new HashMap<>(); + a.header.putAll(headers); + return a; + } + + @Override + public Action getVerifyAction(AnyLongObjectId id) { + return null; // TODO(ms) implement this + } + + @Override + public long getSize(AnyLongObjectId oid) throws IOException { + URL endpointUrl = getObjectUrl(oid); + Map<String, String> queryParams = new HashMap<>(); + queryParams.put(X_AMZ_EXPIRES, + Integer.toString(s3Config.getExpirationSeconds())); + Map<String, String> headers = new HashMap<>(); + + String authorizationQueryParameters = SignerV4.createAuthorizationQuery( + s3Config, endpointUrl, METHOD_HEAD, headers, queryParams, + UNSIGNED_PAYLOAD); + String href = endpointUrl.toString() + "?" //$NON-NLS-1$ + + authorizationQueryParameters; + + Proxy proxy = HttpSupport.proxyFor(ProxySelector.getDefault(), + endpointUrl); + HttpClientConnectionFactory f = new HttpClientConnectionFactory(); + HttpConnection conn = f.create(new URL(href), proxy); + if (s3Config.isDisableSslVerify()) { + HttpSupport.disableSslVerify(conn); + } + conn.setRequestMethod(METHOD_HEAD); + conn.connect(); + int status = conn.getResponseCode(); + if (status == SC_OK) { + String contentLengthHeader = conn + .getHeaderField(HDR_CONTENT_LENGTH); + if (contentLengthHeader != null) { + return Long.parseLong(contentLengthHeader); + } + } + return -1; + } + + /** + * Cache metadata (size) for an object to avoid extra roundtrip to S3 in + * order to retrieve this metadata for a given object. Subclasses can + * implement a local cache and override {{@link #getSize(AnyLongObjectId)} + * to retrieve the object size from the local cache to eliminate the need + * for another roundtrip to S3 + * + * @param oid + * the object id identifying the object to be cached + * @param size + * the object's size (in bytes) + */ + protected void cacheObjectMetaData(AnyLongObjectId oid, long size) { + // no caching + } + + private void validateConfig(S3Config config) { + assertNotEmpty(LfsServerText.get().undefinedS3AccessKey, + config.getAccessKey()); + assertNotEmpty(LfsServerText.get().undefinedS3Bucket, + config.getBucket()); + assertNotEmpty(LfsServerText.get().undefinedS3Region, + config.getRegion()); + assertNotEmpty(LfsServerText.get().undefinedS3Hostname, + config.getHostname()); + assertNotEmpty(LfsServerText.get().undefinedS3SecretKey, + config.getSecretKey()); + assertNotEmpty(LfsServerText.get().undefinedS3StorageClass, + config.getStorageClass()); + } + + private void assertNotEmpty(String message, String value) { + if (value == null || value.trim().length() == 0) { + throw new IllegalArgumentException(message); + } + } + + private URL getObjectUrl(AnyLongObjectId oid) { + try { + return new URL(String.format("https://%s/%s/%s", //$NON-NLS-1$ + s3Config.getHostname(), s3Config.getBucket(), + getPath(oid))); + } catch (MalformedURLException e) { + throw new IllegalArgumentException(MessageFormat.format( + LfsServerText.get().unparsableEndpoint, e.getMessage())); + } + } + + private String getPath(AnyLongObjectId oid) { + return oid.getName(); + } +} diff --git a/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/s3/SignerV4.java b/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/s3/SignerV4.java new file mode 100644 index 0000000000..d88cf13615 --- /dev/null +++ b/org.eclipse.jgit.lfs.server/src/org/eclipse/jgit/lfs/server/s3/SignerV4.java @@ -0,0 +1,389 @@ +/* + * Copyright (C) 2015, Matthias Sohn <matthias.sohn@sap.com> + * Copyright (C) 2015, Sasa Zivkov <sasa.zivkov@sap.com> and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.lfs.server.s3; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.eclipse.jgit.util.HttpSupport.HDR_AUTHORIZATION; + +import java.io.UnsupportedEncodingException; +import java.net.URL; +import java.net.URLEncoder; +import java.security.MessageDigest; +import java.text.MessageFormat; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.SimpleTimeZone; +import java.util.SortedMap; +import java.util.TreeMap; + +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; + +import org.eclipse.jgit.lfs.lib.Constants; +import org.eclipse.jgit.lfs.server.internal.LfsServerText; + +/** + * Signing support for Amazon AWS signing V4 + * <p> + * See + * http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html + */ +class SignerV4 { + static final String UNSIGNED_PAYLOAD = "UNSIGNED-PAYLOAD"; //$NON-NLS-1$ + + private static final String ALGORITHM = "HMAC-SHA256"; //$NON-NLS-1$ + private static final String DATE_STRING_FORMAT = "yyyyMMdd"; //$NON-NLS-1$ + private static final String HEX = "0123456789abcdef"; //$NON-NLS-1$ + private static final String HMACSHA256 = "HmacSHA256"; //$NON-NLS-1$ + private static final String ISO8601_BASIC_FORMAT = "yyyyMMdd'T'HHmmss'Z'"; //$NON-NLS-1$ + private static final String S3 = "s3"; //$NON-NLS-1$ + private static final String SCHEME = "AWS4"; //$NON-NLS-1$ + private static final String TERMINATOR = "aws4_request"; //$NON-NLS-1$ + private static final String UTC = "UTC"; //$NON-NLS-1$ + private static final String X_AMZ_ALGORITHM = "X-Amz-Algorithm"; //$NON-NLS-1$ + private static final String X_AMZ_CREDENTIAL = "X-Amz-Credential"; //$NON-NLS-1$ + private static final String X_AMZ_DATE = "X-Amz-Date"; //$NON-NLS-1$ + private static final String X_AMZ_SIGNATURE = "X-Amz-Signature"; //$NON-NLS-1$ + private static final String X_AMZ_SIGNED_HEADERS = "X-Amz-SignedHeaders"; //$NON-NLS-1$ + + static final String X_AMZ_CONTENT_SHA256 = "x-amz-content-sha256"; //$NON-NLS-1$ + static final String X_AMZ_EXPIRES = "X-Amz-Expires"; //$NON-NLS-1$ + static final String X_AMZ_STORAGE_CLASS = "x-amz-storage-class"; //$NON-NLS-1$ + + /** + * Create an AWSV4 authorization for a request, suitable for embedding in + * query parameters. + * + * @param bucketConfig + * configuration of S3 storage bucket this request should be + * signed for + * @param url + * HTTP request URL + * @param httpMethod + * HTTP method + * @param headers + * The HTTP request headers; 'Host' and 'X-Amz-Date' will be + * added to this set. + * @param queryParameters + * Any query parameters that will be added to the endpoint. The + * parameters should be specified in canonical format. + * @param bodyHash + * Pre-computed SHA256 hash of the request body content; this + * value should also be set as the header 'X-Amz-Content-SHA256' + * for non-streaming uploads. + * @return The computed authorization string for the request. This value + * needs to be set as the header 'Authorization' on the subsequent + * HTTP request. + */ + static String createAuthorizationQuery(S3Config bucketConfig, URL url, + String httpMethod, Map<String, String> headers, + Map<String, String> queryParameters, String bodyHash) { + addHostHeader(url, headers); + + queryParameters.put(X_AMZ_ALGORITHM, SCHEME + "-" + ALGORITHM); //$NON-NLS-1$ + + Date now = new Date(); + String dateStamp = dateStamp(now); + String scope = scope(bucketConfig.getRegion(), dateStamp); + queryParameters.put(X_AMZ_CREDENTIAL, + bucketConfig.getAccessKey() + "/" + scope); //$NON-NLS-1$ + + String dateTimeStampISO8601 = dateTimeStampISO8601(now); + queryParameters.put(X_AMZ_DATE, dateTimeStampISO8601); + + String canonicalizedHeaderNames = canonicalizeHeaderNames(headers); + queryParameters.put(X_AMZ_SIGNED_HEADERS, canonicalizedHeaderNames); + + String canonicalizedQueryParameters = canonicalizeQueryString( + queryParameters); + String canonicalizedHeaders = canonicalizeHeaderString(headers); + String canonicalRequest = canonicalRequest(url, httpMethod, + canonicalizedQueryParameters, canonicalizedHeaderNames, + canonicalizedHeaders, bodyHash); + byte[] signature = createSignature(bucketConfig, dateTimeStampISO8601, + dateStamp, scope, canonicalRequest); + queryParameters.put(X_AMZ_SIGNATURE, toHex(signature)); + + return formatAuthorizationQuery(queryParameters); + } + + private static String formatAuthorizationQuery( + Map<String, String> queryParameters) { + StringBuilder s = new StringBuilder(); + for (String key : queryParameters.keySet()) { + appendQuery(s, key, queryParameters.get(key)); + } + return s.toString(); + } + + private static void appendQuery(StringBuilder s, String key, + String value) { + if (s.length() != 0) { + s.append("&"); //$NON-NLS-1$ + } + s.append(key).append("=").append(value); //$NON-NLS-1$ + } + + /** + * Sign headers for given bucket, url and HTTP method and add signature in + * Authorization header. + * + * @param bucketConfig + * configuration of S3 storage bucket this request should be + * signed for + * @param url + * HTTP request URL + * @param httpMethod + * HTTP method + * @param headers + * HTTP headers to sign + * @param bodyHash + * Pre-computed SHA256 hash of the request body content; this + * value should also be set as the header 'X-Amz-Content-SHA256' + * for non-streaming uploads. + * @return HTTP headers signd by an Authorization header added to the + * headers + */ + static Map<String, String> createHeaderAuthorization( + S3Config bucketConfig, URL url, String httpMethod, + Map<String, String> headers, String bodyHash) { + addHostHeader(url, headers); + + Date now = new Date(); + String dateTimeStamp = dateTimeStampISO8601(now); + headers.put(X_AMZ_DATE, dateTimeStamp); + + String canonicalizedHeaderNames = canonicalizeHeaderNames(headers); + String canonicalizedHeaders = canonicalizeHeaderString(headers); + String canonicalRequest = canonicalRequest(url, httpMethod, "", //$NON-NLS-1$ + canonicalizedHeaderNames, canonicalizedHeaders, bodyHash); + String dateStamp = dateStamp(now); + String scope = scope(bucketConfig.getRegion(), dateStamp); + + byte[] signature = createSignature(bucketConfig, dateTimeStamp, + dateStamp, scope, canonicalRequest); + + headers.put(HDR_AUTHORIZATION, formatAuthorizationHeader(bucketConfig, + canonicalizedHeaderNames, scope, signature)); // $NON-NLS-1$ + + return headers; + } + + private static String formatAuthorizationHeader( + S3Config bucketConfig, String canonicalizedHeaderNames, + String scope, byte[] signature) { + StringBuilder s = new StringBuilder(); + s.append(SCHEME).append("-").append(ALGORITHM).append(" "); //$NON-NLS-1$ //$NON-NLS-2$ + s.append("Credential=").append(bucketConfig.getAccessKey()).append("/") //$NON-NLS-1$//$NON-NLS-2$ + .append(scope).append(","); //$NON-NLS-1$ + s.append("SignedHeaders=").append(canonicalizedHeaderNames).append(","); //$NON-NLS-1$ //$NON-NLS-2$ + s.append("Signature=").append(toHex(signature)); //$NON-NLS-1$ + return s.toString(); + } + + private static void addHostHeader(URL url, + Map<String, String> headers) { + StringBuilder hostHeader = new StringBuilder(url.getHost()); + int port = url.getPort(); + if (port > -1) { + hostHeader.append(":").append(port); //$NON-NLS-1$ + } + headers.put("Host", hostHeader.toString()); //$NON-NLS-1$ + } + + private static String canonicalizeHeaderNames( + Map<String, String> headers) { + List<String> sortedHeaders = new ArrayList<>(); + sortedHeaders.addAll(headers.keySet()); + Collections.sort(sortedHeaders, String.CASE_INSENSITIVE_ORDER); + + StringBuilder buffer = new StringBuilder(); + for (String header : sortedHeaders) { + if (buffer.length() > 0) + buffer.append(";"); //$NON-NLS-1$ + buffer.append(header.toLowerCase(Locale.ROOT)); + } + + return buffer.toString(); + } + + private static String canonicalizeHeaderString( + Map<String, String> headers) { + if (headers == null || headers.isEmpty()) { + return ""; //$NON-NLS-1$ + } + + List<String> sortedHeaders = new ArrayList<>(); + sortedHeaders.addAll(headers.keySet()); + Collections.sort(sortedHeaders, String.CASE_INSENSITIVE_ORDER); + + StringBuilder buffer = new StringBuilder(); + for (String key : sortedHeaders) { + buffer.append( + key.toLowerCase(Locale.ROOT).replaceAll("\\s+", " ") + ":" //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ + + headers.get(key).replaceAll("\\s+", " ")); //$NON-NLS-1$//$NON-NLS-2$ + buffer.append("\n"); //$NON-NLS-1$ + } + + return buffer.toString(); + } + + private static String dateStamp(Date now) { + // TODO(ms) cache and reuse DateFormat instances + SimpleDateFormat dateStampFormat = new SimpleDateFormat( + DATE_STRING_FORMAT); + dateStampFormat.setTimeZone(new SimpleTimeZone(0, UTC)); + String dateStamp = dateStampFormat.format(now); + return dateStamp; + } + + private static String dateTimeStampISO8601(Date now) { + // TODO(ms) cache and reuse DateFormat instances + SimpleDateFormat dateTimeFormat = new SimpleDateFormat( + ISO8601_BASIC_FORMAT); + dateTimeFormat.setTimeZone(new SimpleTimeZone(0, UTC)); + String dateTimeStamp = dateTimeFormat.format(now); + return dateTimeStamp; + } + + private static String scope(String region, String dateStamp) { + String scope = String.format("%s/%s/%s/%s", dateStamp, region, S3, //$NON-NLS-1$ + TERMINATOR); + return scope; + } + + private static String canonicalizeQueryString( + Map<String, String> parameters) { + if (parameters == null || parameters.isEmpty()) { + return ""; //$NON-NLS-1$ + } + + SortedMap<String, String> sorted = new TreeMap<>(); + + Iterator<Map.Entry<String, String>> pairs = parameters.entrySet() + .iterator(); + while (pairs.hasNext()) { + Map.Entry<String, String> pair = pairs.next(); + String key = pair.getKey(); + String value = pair.getValue(); + sorted.put(urlEncode(key, false), urlEncode(value, false)); + } + + StringBuilder builder = new StringBuilder(); + pairs = sorted.entrySet().iterator(); + while (pairs.hasNext()) { + Map.Entry<String, String> pair = pairs.next(); + builder.append(pair.getKey()); + builder.append("="); //$NON-NLS-1$ + builder.append(pair.getValue()); + if (pairs.hasNext()) { + builder.append("&"); //$NON-NLS-1$ + } + } + + return builder.toString(); + } + + private static String canonicalRequest(URL endpoint, String httpMethod, + String queryParameters, String canonicalizedHeaderNames, + String canonicalizedHeaders, String bodyHash) { + return String.format("%s\n%s\n%s\n%s\n%s\n%s", //$NON-NLS-1$ + httpMethod, canonicalizeResourcePath(endpoint), + queryParameters, canonicalizedHeaders, canonicalizedHeaderNames, + bodyHash); + } + + private static String canonicalizeResourcePath(URL endpoint) { + if (endpoint == null) { + return "/"; //$NON-NLS-1$ + } + String path = endpoint.getPath(); + if (path == null || path.isEmpty()) { + return "/"; //$NON-NLS-1$ + } + + String encodedPath = urlEncode(path, true); + if (encodedPath.startsWith("/")) { //$NON-NLS-1$ + return encodedPath; + } + return "/" + encodedPath; //$NON-NLS-1$ + } + + private static byte[] hash(String s) { + MessageDigest md = Constants.newMessageDigest(); + md.update(s.getBytes(UTF_8)); + return md.digest(); + } + + private static byte[] sign(String stringData, byte[] key) { + try { + byte[] data = stringData.getBytes(UTF_8); + Mac mac = Mac.getInstance(HMACSHA256); + mac.init(new SecretKeySpec(key, HMACSHA256)); + return mac.doFinal(data); + } catch (Exception e) { + throw new RuntimeException(MessageFormat.format( + LfsServerText.get().failedToCalcSignature, e.getMessage()), + e); + } + } + + private static String stringToSign(String scheme, String algorithm, + String dateTime, String scope, String canonicalRequest) { + return String.format("%s-%s\n%s\n%s\n%s", //$NON-NLS-1$ + scheme, algorithm, dateTime, scope, + toHex(hash(canonicalRequest))); + } + + private static String toHex(byte[] bytes) { + StringBuilder builder = new StringBuilder(2 * bytes.length); + for (byte b : bytes) { + builder.append(HEX.charAt((b & 0xF0) >> 4)); + builder.append(HEX.charAt(b & 0xF)); + } + return builder.toString(); + } + + private static String urlEncode(String url, boolean keepPathSlash) { + String encoded; + try { + encoded = URLEncoder.encode(url, UTF_8.name()); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(LfsServerText.get().unsupportedUtf8, e); + } + if (keepPathSlash) { + encoded = encoded.replace("%2F", "/"); //$NON-NLS-1$ //$NON-NLS-2$ + } + return encoded; + } + + private static byte[] createSignature(S3Config bucketConfig, + String dateTimeStamp, String dateStamp, + String scope, String canonicalRequest) { + String stringToSign = stringToSign(SCHEME, ALGORITHM, dateTimeStamp, + scope, canonicalRequest); + + byte[] signature = (SCHEME + bucketConfig.getSecretKey()) + .getBytes(UTF_8); + signature = sign(dateStamp, signature); + signature = sign(bucketConfig.getRegion(), signature); + signature = sign(S3, signature); + signature = sign(TERMINATOR, signature); + signature = sign(stringToSign, signature); + return signature; + } +} |