Support LFS protocol and a file system based LFS storage

Implement LfsProtocolServlet handling the "Git LFS v1 Batch API"
protocol [1]. Add a simple file system based LFS content store and the
debug-lfs-store command to simplify testing.

Introduce a LargeFileRepository interface to enable additional storage
implementation while reusing the same protocol implementation.

At the client side we have to configure the lfs.url, specify that
we use the batch API and we don't use authentication:

  [lfs]
	  url = http://host:port/lfs
	  batch = true
  [lfs "http://host:port/lfs"]
	  access = none

the git-lfs client appends the "objects/batch" to the lfs.url.

Hard code an Authorization header in the FileLfsRepository.getAction
because then git-lfs client will skip asking for credentials. It will
just forward the Authorization header from the response to the
download/upload request.

The FileLfsServlet supports file content storage for "Large File
Storage" (LFS) server as defined by the Github LFS API [2].

- upload and download of large files is probably network bound hence use
  an asynchronous servlet for good scalability
- simple object storage in file system with 2 level fan-out
- use LockFile to protect writing large objects against multiple
  concurrent uploads of the same object
- to prevent corrupt uploads the uploaded file is rejected if its hash
  doesn't match id given in URL

The debug-lfs-store command is used to run the LfsProtocolServlet and,
optionally, the FileLfsServlet which makes it easier to setup a
local test server.

[1]
https://github.com/github/git-lfs/blob/master/docs/api/http-v1-batch.md
[2] https://github.com/github/git-lfs/tree/master/docs/api

Bug: 472961
Change-Id: I7378da5575159d2195138d799704880c5c82d5f3
Signed-off-by: Matthias Sohn <matthias.sohn@sap.com>
Signed-off-by: Sasa Zivkov <sasa.zivkov@sap.com>

3 files changed, 485 insertions, 0 deletions

diff --git a/org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit/lfs/server/fs/DownloadTest.java b/org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit/lfs/server/fs/DownloadTest.java
new file mode 100644
index 0000000000..6c4f3cbb99
--- /dev/null
+++ b/org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit/lfs/server/fs/DownloadTest.java
@@ -0,0 +1,132 @@
+/*
+ * Copyright (C) 2015, Matthias Sohn <matthias.sohnk@sap.com>
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.eclipse.jgit.lfs.server.fs;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.text.MessageFormat;
+
+import org.apache.http.client.ClientProtocolException;
+import org.eclipse.jgit.lfs.lib.AnyLongObjectId;
+import org.eclipse.jgit.lfs.test.LongObjectIdTestUtils;
+import org.eclipse.jgit.util.FileUtils;
+import org.junit.Test;
+
+public class DownloadTest extends LfsServerTest {
+
+	@Test
+	public void testDownload() throws Exception {
+		String TEXT = "test";
+		AnyLongObjectId id = putContent(TEXT);
+		Path f = Paths.get(getTempDirectory().toString(), "download");
+		long len = getContent(id, f);
+		assertEquals(TEXT.length(), len);
+		FileUtils.delete(f.toFile(), FileUtils.RETRY);
+	}
+
+	@Test
+	public void testDownloadInvalidPathInfo()
+			throws ClientProtocolException, IOException {
+		String TEXT = "test";
+		AnyLongObjectId id = putContent(TEXT);
+		Path f = Paths.get(getTempDirectory().toString(), "download");
+		try {
+			getContent(id.name().substring(0, 60), f);
+			fail("expected RuntimeException");
+		} catch (RuntimeException e) {
+			assertEquals("Status: 400 Bad Request",
+					e.getMessage());
+		}
+	}
+
+	@Test
+	public void testDownloadInvalidId()
+			throws ClientProtocolException, IOException {
+		String TEXT = "test";
+		AnyLongObjectId id = putContent(TEXT);
+		Path f = Paths.get(getTempDirectory().toString(), "download");
+		try {
+			getContent(id.name().replace('f', 'z'), f);
+			fail("expected RuntimeException");
+		} catch (RuntimeException e) {
+			assertEquals("Status: 400 Bad Request",
+					e.getMessage());
+		}
+	}
+
+	@Test
+	public void testDownloadNotFound()
+			throws ClientProtocolException, IOException {
+		String TEXT = "test";
+		AnyLongObjectId id = LongObjectIdTestUtils.hash(TEXT);
+		Path f = Paths.get(getTempDirectory().toString(), "download");
+		try {
+			getContent(id, f);
+			fail("expected RuntimeException");
+		} catch (RuntimeException e) {
+			assertEquals("Status: 404 Not Found",
+					e.getMessage());
+		}
+	}
+
+	@SuppressWarnings("boxing")
+	@Test
+	public void testLargeFileDownload() throws Exception {
+		Path f = Paths.get(getTempDirectory().toString(), "largeRandomFile");
+		long expectedLen = createPseudoRandomContentFile(f, 5 * MiB);
+		AnyLongObjectId id = putContent(f);
+		Path f2 = Paths.get(getTempDirectory().toString(), "download");
+		long start = System.nanoTime();
+		long len = getContent(id, f2);
+		System.out.println(
+				MessageFormat.format("dowloaded 10 MiB random data in {0}ms",
+						(System.nanoTime() - start) / 1e6));
+		assertEquals(expectedLen, len);
+		FileUtils.delete(f.toFile(), FileUtils.RETRY);
+
+	}
+}
diff --git a/org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit/lfs/server/fs/LfsServerTest.java b/org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit/lfs/server/fs/LfsServerTest.java
new file mode 100644
index 0000000000..8c266d4830
--- /dev/null
+++ b/org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit/lfs/server/fs/LfsServerTest.java
@@ -0,0 +1,252 @@
+/*
+ * Copyright (C) 2015, Matthias Sohn <matthias.sohnk@sap.com>
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.eclipse.jgit.lfs.server.fs;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.BufferedInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.nio.channels.Channels;
+import java.nio.channels.FileChannel;
+import java.nio.channels.ReadableByteChannel;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.security.DigestInputStream;
+import java.security.SecureRandom;
+
+import org.apache.http.HttpEntity;
+import org.apache.http.HttpResponse;
+import org.apache.http.StatusLine;
+import org.apache.http.client.ClientProtocolException;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPut;
+import org.apache.http.entity.ContentType;
+import org.apache.http.entity.InputStreamEntity;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.eclipse.jetty.servlet.ServletContextHandler;
+import org.eclipse.jetty.servlet.ServletHolder;
+import org.eclipse.jgit.junit.http.AppServer;
+import org.eclipse.jgit.lfs.lib.AnyLongObjectId;
+import org.eclipse.jgit.lfs.lib.Constants;
+import org.eclipse.jgit.lfs.lib.LongObjectId;
+import org.eclipse.jgit.lfs.test.LongObjectIdTestUtils;
+import org.eclipse.jgit.util.FileUtils;
+import org.junit.After;
+import org.junit.Before;
+
+@SuppressWarnings("restriction")
+public abstract class LfsServerTest {
+
+	private static final long timeout = /* 10 sec */ 10 * 1000;
+
+	protected static final int MiB = 1024 * 1024;
+
+	/** In-memory application server; subclass must start. */
+	protected AppServer server;
+
+	private Path tmp;
+
+	private Path dir;
+
+	protected FileLfsRepository repository;
+
+	protected FileLfsServlet servlet;
+
+	public LfsServerTest() {
+		super();
+	}
+
+	public Path getTempDirectory() {
+		return tmp;
+	}
+
+	public Path getDir() {
+		return dir;
+	}
+
+	@Before
+	public void setup() throws Exception {
+		tmp = Files.createTempDirectory("jgit_test_");
+		server = new AppServer();
+		ServletContextHandler app = server.addContext("/lfs");
+		dir = Paths.get(tmp.toString(), "lfs");
+		this.repository = new FileLfsRepository(null, dir);
+		servlet = new FileLfsServlet(repository, timeout);
+		app.addServlet(new ServletHolder(servlet), "/objects/*");
+		server.setUp();
+	}
+
+	@After
+	public void tearDown() throws Exception {
+		server.tearDown();
+		FileUtils.delete(tmp.toFile(), FileUtils.RECURSIVE | FileUtils.RETRY);
+	}
+
+	protected AnyLongObjectId putContent(String s)
+			throws IOException, ClientProtocolException {
+		AnyLongObjectId id = LongObjectIdTestUtils.hash(s);
+		return putContent(id, s);
+	}
+
+	protected AnyLongObjectId putContent(AnyLongObjectId id, String s)
+			throws ClientProtocolException, IOException {
+		try (CloseableHttpClient client = HttpClientBuilder.create().build()) {
+			HttpEntity entity = new StringEntity(s,
+					ContentType.APPLICATION_OCTET_STREAM);
+			String hexId = id.name();
+			HttpPut request = new HttpPut(
+					server.getURI() + "/lfs/objects/" + hexId);
+			request.setEntity(entity);
+			try (CloseableHttpResponse response = client.execute(request)) {
+				StatusLine statusLine = response.getStatusLine();
+				int status = statusLine.getStatusCode();
+				if (status >= 400) {
+					throw new RuntimeException("Status: " + status + ". "
+							+ statusLine.getReasonPhrase());
+				}
+			}
+			return id;
+		}
+	}
+
+	protected LongObjectId putContent(Path f)
+			throws FileNotFoundException, IOException {
+		try (CloseableHttpClient client = HttpClientBuilder.create().build()) {
+			LongObjectId id1, id2;
+			String hexId1, hexId2;
+			try (DigestInputStream in = new DigestInputStream(
+					new BufferedInputStream(Files.newInputStream(f)),
+					Constants.newMessageDigest())) {
+				InputStreamEntity entity = new InputStreamEntity(in,
+						Files.size(f), ContentType.APPLICATION_OCTET_STREAM);
+				id1 = LongObjectIdTestUtils.hash(f);
+				hexId1 = id1.name();
+				HttpPut request = new HttpPut(
+						server.getURI() + "/lfs/objects/" + hexId1);
+				request.setEntity(entity);
+				HttpResponse response = client.execute(request);
+				checkResponseStatus(response);
+				id2 = LongObjectId.fromRaw(in.getMessageDigest().digest());
+				hexId2 = id2.name();
+				assertEquals(hexId1, hexId2);
+			}
+			return id1;
+		}
+	}
+
+	private void checkResponseStatus(HttpResponse response) {
+		StatusLine statusLine = response.getStatusLine();
+		int status = statusLine.getStatusCode();
+		if (statusLine.getStatusCode() >= 400) {
+			throw new RuntimeException("Status: " + status + " "
+					+ statusLine.getReasonPhrase());
+		}
+		assertEquals(200, status);
+	}
+
+	protected long getContent(AnyLongObjectId id, Path f) throws IOException {
+		String hexId = id.name();
+		return getContent(hexId, f);
+	}
+
+	protected long getContent(String hexId, Path f) throws IOException {
+		try (CloseableHttpClient client = HttpClientBuilder.create().build()) {
+			HttpGet request = new HttpGet(
+					server.getURI() + "/lfs/objects/" + hexId);
+			HttpResponse response = client.execute(request);
+			checkResponseStatus(response);
+			HttpEntity entity = response.getEntity();
+			long pos = 0;
+			try (InputStream in = entity.getContent();
+					ReadableByteChannel inChannel = Channels.newChannel(in);
+					FileChannel outChannel = FileChannel.open(f,
+							StandardOpenOption.CREATE_NEW,
+							StandardOpenOption.WRITE)) {
+				long transferred;
+				do {
+					transferred = outChannel.transferFrom(inChannel, pos, MiB);
+					pos += transferred;
+				} while (transferred > 0);
+			}
+			return pos;
+		}
+	}
+
+	/**
+	 * Creates a file with random content, repeatedly writing a random string of
+	 * 4k length to the file until the file has at least the specified length.
+	 *
+	 * @param f
+	 *            file to fill
+	 * @param size
+	 *            size of the file to generate
+	 * @return length of the generated file in bytes
+	 * @throws IOException
+	 */
+	protected long createPseudoRandomContentFile(Path f, long size)
+			throws IOException {
+		SecureRandom rnd = new SecureRandom();
+		byte[] buf = new byte[4096];
+		rnd.nextBytes(buf);
+		ByteBuffer bytebuf = ByteBuffer.wrap(buf);
+		try (FileChannel outChannel = FileChannel.open(f,
+				StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) {
+			long len = 0;
+			do {
+				len += outChannel.write(bytebuf);
+				if (bytebuf.position() == 4096) {
+					bytebuf.rewind();
+				}
+			} while (len < size);
+		}
+		return Files.size(f);
+	}
+}
\ No newline at end of file
diff --git a/org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit/lfs/server/fs/UploadTest.java b/org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit/lfs/server/fs/UploadTest.java
new file mode 100644
index 0000000000..35bf09b0c1
--- /dev/null
+++ b/org.eclipse.jgit.lfs.server.test/tst/org/eclipse/jgit/lfs/server/fs/UploadTest.java
@@ -0,0 +1,101 @@
+/*
+ * Copyright (C) 2015, Matthias Sohn <matthias.sohnk@sap.com>
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ *   copyright notice, this list of conditions and the following
+ *   disclaimer in the documentation and/or other materials provided
+ *   with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ *   names of its contributors may be used to endorse or promote
+ *   products derived from this software without specific prior
+ *   written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package org.eclipse.jgit.lfs.server.fs;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.text.MessageFormat;
+
+import org.eclipse.jgit.lfs.lib.AnyLongObjectId;
+import org.eclipse.jgit.lfs.lib.LongObjectId;
+import org.eclipse.jgit.lfs.test.LongObjectIdTestUtils;
+import org.junit.Test;
+
+public class UploadTest extends LfsServerTest {
+
+	@Test
+	public void testUpload() throws Exception {
+		String TEXT = "test";
+		AnyLongObjectId id = putContent(TEXT);
+		assertTrue("expect object " + id.name() + " to exist",
+				repository.getSize(id) >= 0);
+		assertEquals("expected object length " + TEXT.length(), TEXT.length(),
+				repository.getSize(id));
+	}
+
+	@Test
+	public void testCorruptUpload() throws Exception {
+		String TEXT = "test";
+		AnyLongObjectId id = LongObjectIdTestUtils.hash("wrongHash");
+		try {
+			putContent(id, TEXT);
+			fail("expected RuntimeException(\"Status 400\")");
+		} catch (RuntimeException e) {
+			assertEquals("Status: 400. Bad Request", e.getMessage());
+		}
+		assertFalse("expect object " + id.name() + " not to exist",
+				repository.getSize(id) >= 0);
+	}
+
+	@SuppressWarnings("boxing")
+	@Test
+	public void testLargeFileUpload() throws Exception {
+		Path f = Paths.get(getTempDirectory().toString(), "largeRandomFile");
+		createPseudoRandomContentFile(f, 5 * MiB);
+		long start = System.nanoTime();
+		LongObjectId id = putContent(f);
+		System.out.println(
+				MessageFormat.format("uploaded 10 MiB random data in {0}ms",
+						(System.nanoTime() - start) / 1e6));
+		assertTrue("expect object " + id.name() + " to exist",
+				repository.getSize(id) >= 0);
+		assertEquals("expected object length " + Files.size(f), Files.size(f),
+				repository.getSize(id));
+	}
+}