1 files changed, 439 insertions, 0 deletions
diff --git a/src/main/java/com/gitblit/manager/FilestoreManager.java b/src/main/java/com/gitblit/manager/FilestoreManager.java
new file mode 100644
index 00000000..33672e4a
--- /dev/null
+++ b/src/main/java/com/gitblit/manager/FilestoreManager.java
@@ -0,0 +1,439 @@
+/*
+ * Copyright 2015 gitblit.com.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.gitblit.manager;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.RandomAccessFile;
+import java.lang.reflect.Type;
+import java.nio.file.Files;
+import java.text.MessageFormat;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.regex.Pattern;
+
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.bouncycastle.util.io.StreamOverflowException;
+import org.eclipse.jetty.io.EofException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.gitblit.IStoredSettings;
+import com.gitblit.Keys;
+import com.gitblit.models.FilestoreModel.Status;
+import com.gitblit.models.FilestoreModel;
+import com.gitblit.models.RepositoryModel;
+import com.gitblit.models.UserModel;
+import com.gitblit.utils.ArrayUtils;
+import com.gitblit.utils.JsonUtils.GmtDateTypeAdapter;
+import com.google.gson.ExclusionStrategy;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.reflect.TypeToken;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+/**
+ * FilestoreManager handles files uploaded via:
+ * 	+ git-lfs
+ *  + ticket attachment (TBD)
+ * 
+ * Files are stored using their SHA256 hash (as per git-lfs)
+ * If the same file is uploaded through different repositories no additional space is used
+ * Access is controlled through the current repository permissions.
+ *
+ * TODO: Identify what and how the actual BLOBs should work with federation
+ *
+ * @author Paul Martin
+ *
+ */
+@Singleton
+public class FilestoreManager implements IFilestoreManager {
+
+	private final Logger logger = LoggerFactory.getLogger(getClass());
+	
+	private final IRuntimeManager runtimeManager;
+	
+	private final IStoredSettings settings;
+	
+	public static final int UNDEFINED_SIZE = -1;
+	
+	private static final String METAFILE = "filestore.json";
+	
+	private static final String METAFILE_TMP = "filestore.json.tmp";
+	
+	protected static final Type METAFILE_TYPE = new TypeToken<Collection<FilestoreModel>>() {}.getType();
+	
+	private Map<String, FilestoreModel > fileCache = new ConcurrentHashMap<String, FilestoreModel>();
+	
+	
+	@Inject
+	FilestoreManager(
+			IRuntimeManager runtimeManager) {
+		this.runtimeManager = runtimeManager;
+		this.settings = runtimeManager.getSettings();
+	}
+	
+	@Override
+	public IManager start() {
+
+		//Try to load any existing metadata
+		File metadata = new File(getStorageFolder(), METAFILE);
+		
+		if (metadata.exists()) {
+			Collection<FilestoreModel> items = null;
+			
+			Gson gson = gson();
+			try (FileReader file = new FileReader(metadata)) {
+				items = gson.fromJson(file, METAFILE_TYPE);
+				file.close();
+				
+			} catch (IOException e) {
+				e.printStackTrace();
+			}
+			
+			for(Iterator<FilestoreModel> itr = items.iterator(); itr.hasNext(); ) {
+			    FilestoreModel model = itr.next();
+			    fileCache.put(model.oid, model);
+			}
+
+			logger.info("Loaded {} items from filestore metadata file", fileCache.size());
+		}
+		else
+		{
+			logger.info("No filestore metadata file found");
+		}
+		
+		return this;
+	}
+
+	@Override
+	public IManager stop() {
+		return this;
+	}
+
+
+	@Override
+	public boolean isValidOid(String oid) {
+		//NOTE: Assuming SHA256 support only as per git-lfs
+		return Pattern.matches("[a-fA-F0-9]{64}", oid);
+	}
+	
+	@Override
+	public FilestoreModel.Status addObject(String oid, long size, UserModel user, RepositoryModel repo) {
+
+		//Handle access control
+		if (!user.canPush(repo)) {
+			if (user == UserModel.ANONYMOUS) {
+				return Status.AuthenticationRequired;
+			} else {
+				return Status.Error_Unauthorized;
+			}
+		}
+		
+		//Handle object details
+		if (!isValidOid(oid)) { return Status.Error_Invalid_Oid; }
+		
+		if (fileCache.containsKey(oid)) {
+			FilestoreModel item = fileCache.get(oid);
+			
+			if (!item.isInErrorState() && (size != UNDEFINED_SIZE) && (item.getSize() != size)) {
+				return Status.Error_Size_Mismatch;
+			}
+			
+			item.addRepository(repo.name);
+			
+			if (item.isInErrorState()) {
+				item.reset(user, size);
+			}
+		} else {
+			
+			if (size  < 0) {return Status.Error_Invalid_Size; }
+			if ((getMaxUploadSize() != UNDEFINED_SIZE) && (size > getMaxUploadSize())) { return Status.Error_Exceeds_Size_Limit; }
+			
+			FilestoreModel model = new FilestoreModel(oid, size, user, repo.name); 
+			fileCache.put(oid, model);
+			saveFilestoreModel(model);
+		}
+		
+		return fileCache.get(oid).getStatus();
+	}
+
+	@Override
+	public FilestoreModel.Status uploadBlob(String oid, long size, UserModel user, RepositoryModel repo, InputStream streamIn) {
+
+		//Access control and object logic
+		Status state = addObject(oid, size, user, repo); 
+		
+		if (state != Status.Upload_Pending) { 
+			return state;
+		}
+		
+		FilestoreModel model = fileCache.get(oid);
+		
+		if (!model.actionUpload(user)) {
+			return Status.Upload_In_Progress;
+		} else {
+			long actualSize = 0;
+			File file = getStoragePath(oid);
+
+			try {
+				file.getParentFile().mkdirs();
+				file.createNewFile();
+					
+				try (FileOutputStream streamOut = new FileOutputStream(file)) {
+					
+					actualSize = IOUtils.copyLarge(streamIn, streamOut);
+					
+					streamOut.flush();
+					streamOut.close();
+					
+					if (model.getSize() != actualSize) {
+						model.setStatus(Status.Error_Size_Mismatch, user);
+						
+						logger.warn(MessageFormat.format("Failed to upload blob {0} due to size mismatch, expected {1} got {2}", 
+								oid, model.getSize(), actualSize));
+					} else {
+						String actualOid = "";
+						
+						try (FileInputStream fileForHash = new FileInputStream(file)) {
+							actualOid = DigestUtils.sha256Hex(fileForHash);
+							fileForHash.close();
+						}
+						
+						if (oid.equalsIgnoreCase(actualOid)) {
+							model.setStatus(Status.Available, user);
+						} else {
+							model.setStatus(Status.Error_Hash_Mismatch, user);
+							
+							logger.warn(MessageFormat.format("Failed to upload blob {0} due to hash mismatch, got {1}", oid, actualOid));
+						}
+					}
+				}
+			} catch (Exception e) {
+				
+				model.setStatus(Status.Error_Unknown, user);
+				logger.warn(MessageFormat.format("Failed to upload blob {0}", oid), e);
+			} finally {
+				saveFilestoreModel(model);
+			}
+			
+			if (model.isInErrorState()) {
+				file.delete();
+				model.removeRepository(repo.name);
+			}
+		}
+		
+		return model.getStatus();
+	}
+	
+	private FilestoreModel.Status canGetObject(String oid, UserModel user, RepositoryModel repo) {
+	
+		//Access Control
+		if (!user.canView(repo)) {
+			if (user == UserModel.ANONYMOUS) {
+				return Status.AuthenticationRequired;
+			} else {
+				return Status.Error_Unauthorized;
+			}
+		}
+
+		//Object Logic
+		if (!isValidOid(oid)) { 
+			return Status.Error_Invalid_Oid;
+		}
+		
+		if (!fileCache.containsKey(oid)) { 
+			return Status.Unavailable;
+		}
+		
+		FilestoreModel item = fileCache.get(oid);
+		
+		if (item.getStatus() == Status.Available) {
+			return Status.Available;
+		}
+		
+		return Status.Unavailable;
+	}
+	
+	@Override
+	public FilestoreModel getObject(String oid, UserModel user, RepositoryModel repo) {
+		
+		if (canGetObject(oid, user, repo) == Status.Available) {
+			return fileCache.get(oid);
+		}
+		
+		return null;
+	}
+
+	@Override
+	public FilestoreModel.Status downloadBlob(String oid, UserModel user, RepositoryModel repo, OutputStream streamOut) {
+		
+		//Access control and object logic
+		Status status = canGetObject(oid, user, repo);
+				
+		if (status != Status.Available) { 
+			return status;
+		}
+				
+		FilestoreModel item = fileCache.get(oid);
+		
+		if (streamOut != null) {
+			try (FileInputStream streamIn = new FileInputStream(getStoragePath(oid))) {
+				
+				IOUtils.copyLarge(streamIn, streamOut);
+				
+				streamOut.flush();
+				streamIn.close();
+			} catch (EofException e) {
+				logger.error(MessageFormat.format("Client aborted connection for {0}", oid), e);
+				return Status.Error_Unexpected_Stream_End;
+			} catch (Exception e) {
+				logger.error(MessageFormat.format("Failed to download blob {0}", oid), e);
+				return Status.Error_Unknown;
+			}
+		}
+		
+		return item.getStatus();
+	}
+
+	@Override
+	public List<FilestoreModel> getAllObjects() {
+		return new ArrayList<FilestoreModel>(fileCache.values());
+	}
+
+	@Override
+	public File getStorageFolder() {
+		return runtimeManager.getFileOrFolder(Keys.filestore.storageFolder, "${baseFolder}/lfs");
+	}
+	
+	@Override
+	public File getStoragePath(String oid) {
+		 return new File(getStorageFolder(), oid.substring(0, 2).concat("/").concat(oid.substring(2)));
+	}
+
+	@Override
+	public long getMaxUploadSize() {
+		return settings.getLong(Keys.filestore.maxUploadSize, -1);
+	}
+	
+	@Override
+	public long getFilestoreUsedByteCount() {
+		Iterator<FilestoreModel> iterator = fileCache.values().iterator();
+		long total = 0;
+		
+		while (iterator.hasNext()) {
+			
+			FilestoreModel item = iterator.next();
+			if (item.getStatus() == Status.Available) {
+				total += item.getSize();
+			}
+		}
+		
+		return total;
+	}
+	
+	@Override
+	public long getFilestoreAvailableByteCount() {
+		
+		try {
+			return Files.getFileStore(getStorageFolder().toPath()).getUsableSpace();
+		} catch (IOException e) {
+			logger.error(MessageFormat.format("Failed to retrive available space in Filestore {0}", e));
+		}
+		
+		return UNDEFINED_SIZE;
+	};
+	
+	private synchronized void saveFilestoreModel(FilestoreModel model) {
+		
+		File metaFile = new File(getStorageFolder(), METAFILE);
+		File metaFileTmp = new File(getStorageFolder(), METAFILE_TMP);
+		boolean isNewFile = false;
+		
+		try {
+			if (!metaFile.exists()) {
+				metaFile.getParentFile().mkdirs();
+				metaFile.createNewFile();
+				isNewFile = true;
+			}
+			FileUtils.copyFile(metaFile, metaFileTmp);
+			
+		} catch (IOException e) {
+			logger.error("Writing filestore model to file {0}, {1}", METAFILE, e);
+		}
+		
+		try (RandomAccessFile fs = new RandomAccessFile(metaFileTmp, "rw")) {
+		
+			if (isNewFile) {
+				fs.writeBytes("[");
+			} else {
+				fs.seek(fs.length() - 1);
+				fs.writeBytes(",");
+			}
+			
+			fs.writeBytes(gson().toJson(model));
+			fs.writeBytes("]");
+			
+			fs.close();
+			
+		} catch (IOException e) {
+			logger.error("Writing filestore model to file {0}, {1}", METAFILE_TMP, e);
+		}
+		
+		try {
+			if (metaFileTmp.exists()) {
+				FileUtils.copyFile(metaFileTmp, metaFile);
+				
+				metaFileTmp.delete();
+			} else {
+				logger.error("Writing filestore model to file {0}", METAFILE);
+			}
+		}
+		catch (IOException e) {
+			logger.error("Writing filestore model to file {0}, {1}", METAFILE, e);
+		}
+	}
+	
+	/*
+	 * Intended for testing purposes only
+	 */
+	public void clearFilestoreCache() {
+		fileCache.clear();
+	}
+	
+	private static Gson gson(ExclusionStrategy... strategies) {
+		GsonBuilder builder = new GsonBuilder();
+		builder.registerTypeAdapter(Date.class, new GmtDateTypeAdapter());
+		if (!ArrayUtils.isEmpty(strategies)) {
+			builder.setExclusionStrategies(strategies);
+		}
+		return builder.create();
+	}
+	
+}