path: root/src/main/java/com/gitblit/servlet/FilestoreServlet.java

/*
 * Copyright 2015 gitblit.com.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.gitblit.servlet;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.Serializable;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import com.google.inject.Inject;
import com.google.inject.Singleton;

import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.gitblit.Constants;
import com.gitblit.IStoredSettings;
import com.gitblit.models.FilestoreModel;
import com.gitblit.models.RepositoryModel;
import com.gitblit.models.FilestoreModel.Status;
import com.gitblit.manager.FilestoreManager;
import com.gitblit.manager.IGitblit;
import com.gitblit.models.UserModel;
import com.gitblit.utils.JsonUtils;


/**
 * Handles large file storage as per the Git LFS v1 Batch API
 * 
 * Further details can be found at https://github.com/github/git-lfs
 * 
 * @author Paul Martin
 */
@Singleton
public class FilestoreServlet extends HttpServlet {

	private static final long serialVersionUID = 1L;
	public static final int PROTOCOL_VERSION = 1;
	
	public static final String GIT_LFS_META_MIME = "application/vnd.git-lfs+json";
	
	public static final String REGEX_PATH = "^(.*?)/(r)/(.*?)/info/lfs/objects/(batch|" + Constants.REGEX_SHA256 + ")";
	public static final int REGEX_GROUP_BASE_URI = 1;
	public static final int REGEX_GROUP_PREFIX = 2;
	public static final int REGEX_GROUP_REPOSITORY = 3;
	public static final int REGEX_GROUP_ENDPOINT = 4;
	
	protected final Logger logger;
	
	private static IGitblit gitblit;

	@Inject
	public FilestoreServlet(IStoredSettings settings, IGitblit gitblit) {
		
		super();
		logger = LoggerFactory.getLogger(getClass());
		
		FilestoreServlet.gitblit = gitblit;
	}

		
	/**
	 * Handles batch upload request (metadata)
	 *
	 * @param request
	 * @param response
	 * @throws javax.servlet.ServletException
	 * @throws java.io.IOException
	 */
	@Override
	protected void doPost(HttpServletRequest request, 
			HttpServletResponse response) throws ServletException ,IOException {
		
		UrlInfo info = getInfoFromRequest(request);
		if (info == null) {
			sendError(response, HttpServletResponse.SC_NOT_FOUND);
        	return;
		}

		//Post is for batch operations so no oid should be defined
		if (info.oid != null) {
			sendError(response, HttpServletResponse.SC_BAD_REQUEST);
			return;
		}
		
		IGitLFS.Batch batch = deserialize(request, response, IGitLFS.Batch.class);
		
		if (batch == null) { 
			sendError(response, HttpServletResponse.SC_BAD_REQUEST);
			return;
		}

		UserModel user = getUserOrAnonymous(request);
		
		IGitLFS.BatchResponse batchResponse = new IGitLFS.BatchResponse();
		
		if (batch.operation.equalsIgnoreCase("upload")) {
			for (IGitLFS.Request item : batch.objects) {
				
				Status state = gitblit.addObject(item.oid, item.size, user, info.repository);

				batchResponse.objects.add(getResponseForUpload(info.baseUrl, item.oid, item.size, user.getName(), info.repository.name, state));
			}
		} else if (batch.operation.equalsIgnoreCase("download")) {
			for (IGitLFS.Request item : batch.objects) {
				
				Status state = gitblit.downloadBlob(item.oid, user, info.repository, null);
				batchResponse.objects.add(getResponseForDownload(info.baseUrl, item.oid, item.size, user.getName(), info.repository.name, state));
			}
		} else {
			sendError(response, HttpServletResponse.SC_NOT_IMPLEMENTED);
			return;
		}
		
		response.setStatus(HttpServletResponse.SC_OK);
		serialize(response, batchResponse);
	}
	
	/**
	 * Handles the actual upload (BLOB)
	 * 
	 * @param request
	 * @param response
	 * @throws javax.servlet.ServletException
	 * @throws java.io.IOException
	 */
	@Override
	protected void doPut(HttpServletRequest request, 
			HttpServletResponse response) throws ServletException ,IOException {
		
		UrlInfo info = getInfoFromRequest(request);
		
		if (info == null) {
			sendError(response, HttpServletResponse.SC_NOT_FOUND);
        	return;
		}

		//Put is a singular operation so must have oid
		if (info.oid == null) {
			sendError(response, HttpServletResponse.SC_BAD_REQUEST);
			return;
		}
		
		UserModel user = getUserOrAnonymous(request);
		long size = FilestoreManager.UNDEFINED_SIZE;
		
		
		
		FilestoreModel.Status status = gitblit.uploadBlob(info.oid, size, user, info.repository, request.getInputStream());
		IGitLFS.Response responseObject = getResponseForUpload(info.baseUrl, info.oid, size, user.getName(), info.repository.name, status);
		
		logger.info(MessageFormat.format("FILESTORE-AUDIT {0}:{4} {1} {2}@{3}", 
				"PUT", info.oid, user.getName(), info.repository.name, status.toString() ));
		
		if (responseObject.error == null) {
			response.setStatus(responseObject.successCode);
		} else {
			serialize(response, responseObject.error);
		}
	};
	
	/**
	 * Handles a download
	 * Treated as hypermedia request if accept header contains Git-LFS MIME
	 * otherwise treated as a download of the blob
	 * @param request
	 * @param response
	 * @throws javax.servlet.ServletException
	 * @throws java.io.IOException
	 */
	@Override
	protected void doGet(HttpServletRequest request, 
			HttpServletResponse response) throws ServletException ,IOException {
		
		UrlInfo info = getInfoFromRequest(request);
		
		if (info == null || info.oid == null) {
			sendError(response, HttpServletResponse.SC_NOT_FOUND);
        	return;
		}
		
		UserModel user = getUserOrAnonymous(request);
		
		FilestoreModel model = gitblit.getObject(info.oid, user, info.repository);
		long size = FilestoreManager.UNDEFINED_SIZE;
		
		boolean isMetaRequest = AccessRestrictionFilter.hasContentInRequestHeader(request, "Accept", GIT_LFS_META_MIME);
		FilestoreModel.Status status = Status.Unavailable;
		
		if (model != null) {
			size = model.getSize();
			status = model.getStatus();
		}
		
		if (!isMetaRequest) {
			status = gitblit.downloadBlob(info.oid, user, info.repository, response.getOutputStream());
			
			logger.info(MessageFormat.format("FILESTORE-AUDIT {0}:{4} {1} {2}@{3}", 
					"GET", info.oid, user.getName(), info.repository.name, status.toString() ));
		}
		
		if (status == Status.Error_Unexpected_Stream_End) {
			return;
		}

		IGitLFS.Response responseObject = getResponseForDownload(info.baseUrl, 
				info.oid, size, user.getName(), info.repository.name, status);
		
		if (responseObject.error == null) {
			response.setStatus(responseObject.successCode);
			
			if (isMetaRequest) {
				serialize(response, responseObject);
			}
		} else {
			response.setStatus(responseObject.error.code);
			serialize(response, responseObject.error);
		}
	};
	
	private void sendError(HttpServletResponse response, int code) throws IOException {
		
		String msg = "";
		
		switch (code)
		{
			case HttpServletResponse.SC_NOT_FOUND: msg = "Not Found"; break;
			case HttpServletResponse.SC_NOT_IMPLEMENTED: msg = "Not Implemented"; break;
			case HttpServletResponse.SC_BAD_REQUEST: msg = "Malformed Git-LFS request"; break;
			
			default: msg = "Unknown Error";
		}
		
		response.setStatus(code);
		serialize(response, new IGitLFS.ObjectError(code, msg));
	}
	
	@SuppressWarnings("incomplete-switch")
	private IGitLFS.Response getResponseForUpload(String baseUrl, String oid, long size, String user, String repo, FilestoreModel.Status state) {

		switch (state) {
			case AuthenticationRequired:
				return new IGitLFS.Response(oid, size, 401, MessageFormat.format("Authentication required to write to repository {0}", repo));
			case Error_Unauthorized: 
				return new IGitLFS.Response(oid, size, 403, MessageFormat.format("User {0}, does not have write permissions to repository {1}", user, repo));
			case Error_Exceeds_Size_Limit: 
				return new IGitLFS.Response(oid, size, 509, MessageFormat.format("Object is larger than allowed limit of {1}",  gitblit.getMaxUploadSize()));
			case Error_Hash_Mismatch: 
				return new IGitLFS.Response(oid, size, 422, "Hash mismatch");
			case Error_Invalid_Oid: 
				return new IGitLFS.Response(oid, size, 422, MessageFormat.format("{0} is not a valid oid", oid));
			case Error_Invalid_Size: 
				return new IGitLFS.Response(oid, size, 422, MessageFormat.format("{0} is not a valid size", size));
			case Error_Size_Mismatch: 
				return new IGitLFS.Response(oid, size, 422, "Object size mismatch");
			case Deleted: 
				return new IGitLFS.Response(oid, size, 410, "Object was deleted : ".concat("TBD Reason") );
			case Upload_In_Progress:
				return new IGitLFS.Response(oid, size, 503, "File currently being uploaded by another user");
			case Unavailable: 
				return new IGitLFS.Response(oid, size, 404, MessageFormat.format("Repository {0}, does not exist for user {1}", repo, user));
			case Upload_Pending: 
				return new IGitLFS.Response(oid, size, 202, "upload", getObjectUri(baseUrl, repo, oid) );
			case Available: 
				return new IGitLFS.Response(oid, size, 200, "upload", getObjectUri(baseUrl, repo, oid) );
		}
		
		return new IGitLFS.Response(oid, size, 500, "Unknown Error");
	}

	@SuppressWarnings("incomplete-switch")
	private IGitLFS.Response getResponseForDownload(String baseUrl, String oid, long size, String user, String repo, FilestoreModel.Status state) {

		switch (state) {
			case Error_Unauthorized: 
				return new IGitLFS.Response(oid, size, 403, MessageFormat.format("User {0}, does not have read permissions to repository {1}", user, repo));
			case Error_Invalid_Oid: 
				return new IGitLFS.Response(oid, size, 422, MessageFormat.format("{0} is not a valid oid", oid));
			case Error_Unknown:
				return new IGitLFS.Response(oid, size, 500, "Unknown Error");
			case Deleted: 
				return new IGitLFS.Response(oid, size, 410, "Object was deleted : ".concat("TBD Reason") );
			case Available: 
				return new IGitLFS.Response(oid, size, 200, "download", getObjectUri(baseUrl, repo, oid) );
		}
		
		return new IGitLFS.Response(oid, size, 404, "Object not available");
	}

	
	private String getObjectUri(String baseUrl, String repo, String oid) {
		return baseUrl + "/" + repo + "/" + Constants.R_LFS + "objects/" + oid;
	}
	
	
	protected void serialize(HttpServletResponse response, Object o) throws IOException {
		if (o != null) {
			// Send JSON response
			String json = JsonUtils.toJsonString(o);
			response.setCharacterEncoding(Constants.ENCODING);
			response.setContentType(GIT_LFS_META_MIME);
			response.getWriter().append(json);
		}
	}
	
	protected <X> X deserialize(HttpServletRequest request, HttpServletResponse response,
			Class<X> clazz) {
		
		String json = "";
		try {
			
			json = readJson(request, response);
			
			return JsonUtils.fromJsonString(json.toString(), clazz);
			
		} catch (Exception e) {
			//Intentional silent fail
		}
		
		return null;
	}
	
	private String readJson(HttpServletRequest request, HttpServletResponse response)
			throws IOException {
		BufferedReader reader = request.getReader();
		StringBuilder json = new StringBuilder();
		String line = null;
		while ((line = reader.readLine()) != null) {
			json.append(line);
		}
		reader.close();

		if (json.length() == 0) {
			logger.error(MessageFormat.format("Failed to receive json data from {0}",
					request.getRemoteAddr()));
			response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
			return null;
		}
		return json.toString();
	}
	
	private UserModel getUserOrAnonymous(HttpServletRequest r) {
		UserModel user = (UserModel) r.getUserPrincipal();
		if (user != null) { return user; }
		return UserModel.ANONYMOUS;
	}
	
	private static class UrlInfo {
		public RepositoryModel repository;
		public String oid;
		public String baseUrl;
		
		public UrlInfo(RepositoryModel repo, String oid, String baseUrl) {
			this.repository = repo;
			this.oid = oid;
			this.baseUrl = baseUrl;
		}
	}
	
	public static UrlInfo getInfoFromRequest(HttpServletRequest httpRequest) {
		
		String url = httpRequest.getRequestURL().toString();
		Pattern p = Pattern.compile(REGEX_PATH);
        Matcher m = p.matcher(url);
		
        
        if (m.find()) {
        	RepositoryModel repo = gitblit.getRepositoryModel(m.group(REGEX_GROUP_REPOSITORY));
        	String baseUrl = m.group(REGEX_GROUP_BASE_URI) + "/" + m.group(REGEX_GROUP_PREFIX);
        	
        	if (m.group(REGEX_GROUP_ENDPOINT).equals("batch")) {
        		return new UrlInfo(repo, null, baseUrl);
        	} else {
        		return new UrlInfo(repo, m.group(REGEX_GROUP_ENDPOINT), baseUrl);
        	}
        }
		
		return null;
	}
	
	
	public interface IGitLFS {
	
		@SuppressWarnings("serial")
		public class Request implements Serializable
		{
			public String oid;
			public long size;
		}
		
		
		@SuppressWarnings("serial")
		public class Batch implements Serializable
		{
			public String operation;
			public List<Request> objects;
		}
		
		
		@SuppressWarnings("serial")
		public class Response implements Serializable
		{
			public String oid;
			public long size;
			public Map<String, HyperMediaLink> actions;
			public ObjectError error;
			public transient int successCode; 
			
			public Response(String id, long itemSize, int errorCode, String errorText) {
				oid = id;
				size = itemSize;
				actions = null;
				successCode = 0;
				error = new ObjectError(errorCode, errorText);
			}
			
			public Response(String id, long itemSize, int actionCode, String action, String uri) {
				oid = id;
				size = itemSize;
				error = null;
				successCode = actionCode;
				actions = new HashMap<String, HyperMediaLink>();
				actions.put(action, new HyperMediaLink(action, uri));
			}
			
		}
		
		@SuppressWarnings("serial")
		public class BatchResponse implements Serializable {
			public List<Response> objects;
			
			public BatchResponse() {
				objects = new ArrayList<Response>();
			}
		}
		
		
		@SuppressWarnings("serial")
		public class ObjectError implements Serializable
		{
			public String message;
			public int code;
			public String documentation_url;
			public Integer request_id;
			
			public ObjectError(int errorCode, String errorText) {
				code = errorCode;
				message = errorText;
				request_id = null;
			}
		}
		
		@SuppressWarnings("serial")
		public class HyperMediaLink implements Serializable
		{
			public String href;
			public transient String header;
			//public Date expires_at;
			
			public HyperMediaLink(String action, String uri) {
				header = action;
				href = uri;
			}
		}
	}


	
}