gitblit/src/main/java/com/gitblit/manager/FilestoreManager.java

/*
 * Copyright 2015 gitblit.com.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.gitblit.manager;

import java.io.EOFException;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.lang.reflect.Type;
import java.nio.file.Files;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Pattern;

import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.gitblit.IStoredSettings;
import com.gitblit.Keys;
import com.gitblit.models.FilestoreModel;
import com.gitblit.models.FilestoreModel.Status;
import com.gitblit.models.RepositoryModel;
import com.gitblit.models.UserModel;
import com.gitblit.utils.ArrayUtils;
import com.gitblit.utils.JsonUtils.GmtDateTypeAdapter;
import com.google.gson.ExclusionStrategy;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.reflect.TypeToken;
import com.google.inject.Inject;
import com.google.inject.Singleton;

/**
 * FilestoreManager handles files uploaded via:
 * 	+ git-lfs
 *  + ticket attachment (TBD)
 *
 * Files are stored using their SHA256 hash (as per git-lfs)
 * If the same file is uploaded through different repositories no additional space is used
 * Access is controlled through the current repository permissions.
 *
 * TODO: Identify what and how the actual BLOBs should work with federation
 *
 * @author Paul Martin
 *
 */
@Singleton
public class FilestoreManager implements IFilestoreManager {

	private final Logger logger = LoggerFactory.getLogger(getClass());

	private final IRuntimeManager runtimeManager;
	
	private final IStoredSettings settings;

	public static final int UNDEFINED_SIZE = -1;

	private static final String METAFILE = "filestore.json";

	private static final String METAFILE_TMP = "filestore.json.tmp";

	protected static final Type METAFILE_TYPE = new TypeToken<Collection<FilestoreModel>>() {}.getType();

	private Map<String, FilestoreModel > fileCache = new ConcurrentHashMap<String, FilestoreModel>();


	@Inject
	public FilestoreManager(IRuntimeManager runtimeManager) {
		this.runtimeManager = runtimeManager;
		this.settings = runtimeManager.getSettings();
	}

	@Override
	public IManager start() {

		// Try to load any existing metadata
		File dir = getStorageFolder();
		dir.mkdirs();
		File metadata = new File(dir, METAFILE);

		if (metadata.exists()) {
			Collection<FilestoreModel> items = null;

			Gson gson = gson();
			try (FileReader file = new FileReader(metadata)) {
				items = gson.fromJson(file, METAFILE_TYPE);
				file.close();

			} catch (IOException e) {
				e.printStackTrace();
			}

			for(Iterator<FilestoreModel> itr = items.iterator(); itr.hasNext(); ) {
			    FilestoreModel model = itr.next();
			    fileCache.put(model.oid, model);
			}

			logger.info("Loaded {} items from filestore metadata file", fileCache.size());
		}
		else
		{
			logger.info("No filestore metadata file found");
		}

		return this;
	}

	@Override
	public IManager stop() {
		return this;
	}


	@Override
	public boolean isValidOid(String oid) {
		//NOTE: Assuming SHA256 support only as per git-lfs
		return Pattern.matches("[a-fA-F0-9]{64}", oid);
	}

	@Override
	public FilestoreModel.Status addObject(String oid, long size, UserModel user, RepositoryModel repo) {

		//Handle access control
		if (!user.canPush(repo)) {
			if (user == UserModel.ANONYMOUS) {
				return Status.AuthenticationRequired;
			} else {
				return Status.Error_Unauthorized;
			}
		}

		//Handle object details
		if (!isValidOid(oid)) { return Status.Error_Invalid_Oid; }

		if (fileCache.containsKey(oid)) {
			FilestoreModel item = fileCache.get(oid);

			if (!item.isInErrorState() && (size != UNDEFINED_SIZE) && (item.getSize() != size)) {
				return Status.Error_Size_Mismatch;
			}

			item.addRepository(repo.name);

			if (item.isInErrorState()) {
				item.reset(user, size);
			}
		} else {

			if (size  < 0) {return Status.Error_Invalid_Size; }
			if ((getMaxUploadSize() != UNDEFINED_SIZE) && (size > getMaxUploadSize())) { return Status.Error_Exceeds_Size_Limit; }

			FilestoreModel model = new FilestoreModel(oid, size, user, repo.name);
			fileCache.put(oid, model);
			saveFilestoreModel(model);
		}

		return fileCache.get(oid).getStatus();
	}

	@Override
	public FilestoreModel.Status uploadBlob(String oid, long size, UserModel user, RepositoryModel repo, InputStream streamIn) {

		//Access control and object logic
		Status state = addObject(oid, size, user, repo);

		if (state != Status.Upload_Pending) {
			return state;
		}

		FilestoreModel model = fileCache.get(oid);

		if (!model.actionUpload(user)) {
			return Status.Upload_In_Progress;
		} else {
			long actualSize = 0;
			File file = getStoragePath(oid);

			try {
				file.getParentFile().mkdirs();
				file.createNewFile();

				try (FileOutputStream streamOut = new FileOutputStream(file)) {

					actualSize = IOUtils.copyLarge(streamIn, streamOut);

					streamOut.flush();
					streamOut.close();

					if (model.getSize() != actualSize) {
						model.setStatus(Status.Error_Size_Mismatch, user);

						logger.warn(MessageFormat.format("Failed to upload blob {0} due to size mismatch, expected {1} got {2}",
								oid, model.getSize(), actualSize));
					} else {
						String actualOid = "";

						try (FileInputStream fileForHash = new FileInputStream(file)) {
							actualOid = DigestUtils.sha256Hex(fileForHash);
							fileForHash.close();
						}

						if (oid.equalsIgnoreCase(actualOid)) {
							model.setStatus(Status.Available, user);
						} else {
							model.setStatus(Status.Error_Hash_Mismatch, user);

							logger.warn(MessageFormat.format("Failed to upload blob {0} due to hash mismatch, got {1}", oid, actualOid));
						}
					}
				}
			} catch (Exception e) {

				model.setStatus(Status.Error_Unknown, user);
				logger.warn(MessageFormat.format("Failed to upload blob {0}", oid), e);
			} finally {
				saveFilestoreModel(model);
			}

			if (model.isInErrorState()) {
				file.delete();
				model.removeRepository(repo.name);
			}
		}

		return model.getStatus();
	}

	private FilestoreModel.Status canGetObject(String oid, UserModel user, RepositoryModel repo) {

		//Access Control
		if (!user.canView(repo)) {
			if (user == UserModel.ANONYMOUS) {
				return Status.AuthenticationRequired;
			} else {
				return Status.Error_Unauthorized;
			}
		}

		//Object Logic
		if (!isValidOid(oid)) {
			return Status.Error_Invalid_Oid;
		}

		if (!fileCache.containsKey(oid)) {
			return Status.Unavailable;
		}

		FilestoreModel item = fileCache.get(oid);

		if (item.getStatus() == Status.Available) {
			return Status.Available;
		}

		return Status.Unavailable;
	}

	@Override
	public FilestoreModel getObject(String oid, UserModel user, RepositoryModel repo) {

		if (canGetObject(oid, user, repo) == Status.Available) {
			return fileCache.get(oid);
		}

		return null;
	}

	@Override
	public FilestoreModel.Status downloadBlob(String oid, UserModel user, RepositoryModel repo, OutputStream streamOut) {

		//Access control and object logic
		Status status = canGetObject(oid, user, repo);

		if (status != Status.Available) {
			return status;
		}

		FilestoreModel item = fileCache.get(oid);

		if (streamOut != null) {
			try (FileInputStream streamIn = new FileInputStream(getStoragePath(oid))) {

				IOUtils.copyLarge(streamIn, streamOut);

				streamOut.flush();
				streamIn.close();
			} catch (EOFException e) {
				logger.error(MessageFormat.format("Client aborted connection for {0}", oid), e);
				return Status.Error_Unexpected_Stream_End;
			} catch (Exception e) {
				logger.error(MessageFormat.format("Failed to download blob {0}", oid), e);
				return Status.Error_Unknown;
			}
		}

		return item.getStatus();
	}

	@Override
	public List<FilestoreModel> getAllObjects(List<RepositoryModel> viewableRepositories) {
		
		List<String> viewableRepositoryNames = new ArrayList<String>(viewableRepositories.size());
		
		for (RepositoryModel repository : viewableRepositories) {
			viewableRepositoryNames.add(repository.name);
		}
		
		if (viewableRepositoryNames.size() == 0) {
			return null;
		}
		
		final Collection<FilestoreModel> allFiles = fileCache.values();
		List<FilestoreModel> userViewableFiles = new ArrayList<FilestoreModel>(allFiles.size());
		
		for (FilestoreModel file : allFiles) {
			if (file.isInRepositoryList(viewableRepositoryNames)) {
				userViewableFiles.add(file);
			}
		}
		
		return userViewableFiles;				
	}

	@Override
	public File getStorageFolder() {
		return runtimeManager.getFileOrFolder(Keys.filestore.storageFolder, "${baseFolder}/lfs");
	}

	@Override
	public File getStoragePath(String oid) {
		 return new File(getStorageFolder(), oid.substring(0, 2).concat("/").concat(oid.substring(2)));
	}

	@Override
	public long getMaxUploadSize() {
		return settings.getLong(Keys.filestore.maxUploadSize, -1);
	}

	@Override
	public long getFilestoreUsedByteCount() {
		Iterator<FilestoreModel> iterator = fileCache.values().iterator();
		long total = 0;

		while (iterator.hasNext()) {

			FilestoreModel item = iterator.next();
			if (item.getStatus() == Status.Available) {
				total += item.getSize();
			}
		}

		return total;
	}

	@Override
	public long getFilestoreAvailableByteCount() {

		try {
			return Files.getFileStore(getStorageFolder().toPath()).getUsableSpace();
		} catch (IOException e) {
			logger.error(MessageFormat.format("Failed to retrive available space in Filestore {0}", e));
		}

		return UNDEFINED_SIZE;
	};

	private synchronized void saveFilestoreModel(FilestoreModel model) {

		File metaFile = new File(getStorageFolder(), METAFILE);
		File metaFileTmp = new File(getStorageFolder(), METAFILE_TMP);
		boolean isNewFile = false;

		try {
			if (!metaFile.exists()) {
				metaFile.getParentFile().mkdirs();
				metaFile.createNewFile();
				isNewFile = true;
			}
			FileUtils.copyFile(metaFile, metaFileTmp);

		} catch (IOException e) {
			logger.error("Writing filestore model to file {0}, {1}", METAFILE, e);
		}

		try (RandomAccessFile fs = new RandomAccessFile(metaFileTmp, "rw")) {

			if (isNewFile) {
				fs.writeBytes("[");
			} else {
				fs.seek(fs.length() - 1);
				fs.writeBytes(",");
			}

			fs.writeBytes(gson().toJson(model));
			fs.writeBytes("]");

			fs.close();

		} catch (IOException e) {
			logger.error("Writing filestore model to file {0}, {1}", METAFILE_TMP, e);
		}

		try {
			if (metaFileTmp.exists()) {
				FileUtils.copyFile(metaFileTmp, metaFile);

				metaFileTmp.delete();
			} else {
				logger.error("Writing filestore model to file {0}", METAFILE);
			}
		}
		catch (IOException e) {
			logger.error("Writing filestore model to file {0}, {1}", METAFILE, e);
		}
	}

	/*
	 * Intended for testing purposes only
	 */
	@Override
	public void clearFilestoreCache() {
		fileCache.clear();
	}

	private static Gson gson(ExclusionStrategy... strategies) {
		GsonBuilder builder = new GsonBuilder();
		builder.registerTypeAdapter(Date.class, new GmtDateTypeAdapter());
		if (!ArrayUtils.isEmpty(strategies)) {
			builder.setExclusionStrategies(strategies);
		}
		return builder.create();
	}

}