diff options
32 files changed, 2651 insertions, 37 deletions
diff --git a/src/main/distrib/data/defaults.properties b/src/main/distrib/data/defaults.properties index 4606f5fc..ce6267a5 100644 --- a/src/main/distrib/data/defaults.properties +++ b/src/main/distrib/data/defaults.properties @@ -2030,3 +2030,17 @@ server.requireClientCertificates = false # SINCE 0.5.0 # RESTART REQUIRED server.shutdownPort = 8081 + +# +# Gitblit Filestore Settings +# +# The location to save the filestore blobs +# +# SINCE 1.7.0 +filestore.storageFolder = ${baseFolder}/lfs + +# Maximum allowable upload size +# The default value, -1, disables upload limits. +# Common unit suffixes of k, m, or g are supported. +# SINCE 1.7.0 +filestore.maxUploadSize = -1 diff --git a/src/main/java/com/gitblit/Constants.java b/src/main/java/com/gitblit/Constants.java index 787d7267..4aa8c0ca 100644 --- a/src/main/java/com/gitblit/Constants.java +++ b/src/main/java/com/gitblit/Constants.java @@ -60,6 +60,8 @@ public class Constants { public static final String R_PATH = "/r/";
public static final String GIT_PATH = "/git/";
+
+ public static final String REGEX_SHA256 = "[a-fA-F0-9]{64}";
public static final String ZIP_PATH = "/zip/";
@@ -140,6 +142,8 @@ public class Constants { public static final String ATTRIB_AUTHTYPE = NAME + ":authentication-type";
public static final String ATTRIB_AUTHUSER = NAME + ":authenticated-user";
+
+ public static final String R_LFS = "info/lfs/";
public static String getVersion() {
String v = Constants.class.getPackage().getImplementationVersion();
diff --git a/src/main/java/com/gitblit/FederationClient.java b/src/main/java/com/gitblit/FederationClient.java index 487080e5..64ff0172 100644 --- a/src/main/java/com/gitblit/FederationClient.java +++ b/src/main/java/com/gitblit/FederationClient.java @@ -100,7 +100,7 @@ public class FederationClient { UserManager users = new UserManager(runtime, null).start(); RepositoryManager repositories = new RepositoryManager(runtime, null, users).start(); FederationManager federation = new FederationManager(runtime, notifications, repositories).start(); - IGitblit gitblit = new GitblitManager(null, null, runtime, null, notifications, users, null, repositories, null, federation); + IGitblit gitblit = new GitblitManager(null, null, runtime, null, notifications, users, null, repositories, null, federation, null); FederationPullService puller = new FederationPullService(gitblit, federation.getFederationRegistrations()) { @Override diff --git a/src/main/java/com/gitblit/GitBlit.java b/src/main/java/com/gitblit/GitBlit.java index 68a91bb5..4e25d5c6 100644 --- a/src/main/java/com/gitblit/GitBlit.java +++ b/src/main/java/com/gitblit/GitBlit.java @@ -18,6 +18,7 @@ package com.gitblit; import com.gitblit.manager.GitblitManager; import com.gitblit.manager.IAuthenticationManager; import com.gitblit.manager.IFederationManager; +import com.gitblit.manager.IFilestoreManager; import com.gitblit.manager.INotificationManager; import com.gitblit.manager.IPluginManager; import com.gitblit.manager.IProjectManager; @@ -52,7 +53,8 @@ public class GitBlit extends GitblitManager { IAuthenticationManager authenticationManager, IRepositoryManager repositoryManager, IProjectManager projectManager, - IFederationManager federationManager) { + IFederationManager federationManager, + IFilestoreManager filestoreManager) { super( publicKeyManagerProvider, @@ -64,6 +66,7 @@ public class GitBlit extends GitblitManager { authenticationManager, repositoryManager, projectManager, - federationManager); + federationManager, + filestoreManager); } } diff --git a/src/main/java/com/gitblit/guice/CoreModule.java b/src/main/java/com/gitblit/guice/CoreModule.java index a942b2ec..e2d14399 100644 --- a/src/main/java/com/gitblit/guice/CoreModule.java +++ b/src/main/java/com/gitblit/guice/CoreModule.java @@ -20,8 +20,10 @@ import com.gitblit.GitBlit; import com.gitblit.IStoredSettings; import com.gitblit.manager.AuthenticationManager; import com.gitblit.manager.FederationManager; +import com.gitblit.manager.FilestoreManager; import com.gitblit.manager.IAuthenticationManager; import com.gitblit.manager.IFederationManager; +import com.gitblit.manager.IFilestoreManager; import com.gitblit.manager.IGitblit; import com.gitblit.manager.INotificationManager; import com.gitblit.manager.IPluginManager; @@ -72,6 +74,7 @@ public class CoreModule extends AbstractModule { bind(IRepositoryManager.class).to(RepositoryManager.class); bind(IProjectManager.class).to(ProjectManager.class); bind(IFederationManager.class).to(FederationManager.class); + bind(IFilestoreManager.class).to(FilestoreManager.class); // the monolithic manager bind(IGitblit.class).to(GitBlit.class); diff --git a/src/main/java/com/gitblit/guice/WebModule.java b/src/main/java/com/gitblit/guice/WebModule.java index a4062703..7c83e455 100644 --- a/src/main/java/com/gitblit/guice/WebModule.java +++ b/src/main/java/com/gitblit/guice/WebModule.java @@ -26,6 +26,7 @@ import com.gitblit.servlet.DownloadZipFilter; import com.gitblit.servlet.DownloadZipServlet; import com.gitblit.servlet.EnforceAuthenticationFilter; import com.gitblit.servlet.FederationServlet; +import com.gitblit.servlet.FilestoreServlet; import com.gitblit.servlet.GitFilter; import com.gitblit.servlet.GitServlet; import com.gitblit.servlet.LogoServlet; @@ -62,12 +63,14 @@ public class WebModule extends ServletModule { bind(AvatarGenerator.class).toProvider(AvatarGeneratorProvider.class); // servlets + serveRegex(FilestoreServlet.REGEX_PATH).with(FilestoreServlet.class); serve(fuzzy(Constants.R_PATH), fuzzy(Constants.GIT_PATH)).with(GitServlet.class); serve(fuzzy(Constants.RAW_PATH)).with(RawServlet.class); serve(fuzzy(Constants.PAGES)).with(PagesServlet.class); serve(fuzzy(Constants.RPC_PATH)).with(RpcServlet.class); serve(fuzzy(Constants.ZIP_PATH)).with(DownloadZipServlet.class); serve(fuzzy(Constants.SYNDICATION_PATH)).with(SyndicationServlet.class); + serve(fuzzy(Constants.FEDERATION_PATH)).with(FederationServlet.class); serve(fuzzy(Constants.SPARKLESHARE_INVITE_PATH)).with(SparkleShareInviteServlet.class); @@ -98,7 +101,8 @@ public class WebModule extends ServletModule { filter(fuzzy(Constants.RPC_PATH)).through(RpcFilter.class); filter(fuzzy(Constants.ZIP_PATH)).through(DownloadZipFilter.class); filter(fuzzy(Constants.SYNDICATION_PATH)).through(SyndicationFilter.class); - + + // Wicket String toIgnore = Joiner.on(",").join(Constants.R_PATH, Constants.GIT_PATH, Constants.RAW_PATH, Constants.PAGES, Constants.RPC_PATH, Constants.ZIP_PATH, Constants.SYNDICATION_PATH, diff --git a/src/main/java/com/gitblit/manager/FilestoreManager.java b/src/main/java/com/gitblit/manager/FilestoreManager.java new file mode 100644 index 00000000..33672e4a --- /dev/null +++ b/src/main/java/com/gitblit/manager/FilestoreManager.java @@ -0,0 +1,439 @@ +/* + * Copyright 2015 gitblit.com. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.gitblit.manager; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.FileReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.RandomAccessFile; +import java.lang.reflect.Type; +import java.nio.file.Files; +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.regex.Pattern; + +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.bouncycastle.util.io.StreamOverflowException; +import org.eclipse.jetty.io.EofException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.gitblit.IStoredSettings; +import com.gitblit.Keys; +import com.gitblit.models.FilestoreModel.Status; +import com.gitblit.models.FilestoreModel; +import com.gitblit.models.RepositoryModel; +import com.gitblit.models.UserModel; +import com.gitblit.utils.ArrayUtils; +import com.gitblit.utils.JsonUtils.GmtDateTypeAdapter; +import com.google.gson.ExclusionStrategy; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.reflect.TypeToken; +import com.google.inject.Inject; +import com.google.inject.Singleton; + +/** + * FilestoreManager handles files uploaded via: + * + git-lfs + * + ticket attachment (TBD) + * + * Files are stored using their SHA256 hash (as per git-lfs) + * If the same file is uploaded through different repositories no additional space is used + * Access is controlled through the current repository permissions. + * + * TODO: Identify what and how the actual BLOBs should work with federation + * + * @author Paul Martin + * + */ +@Singleton +public class FilestoreManager implements IFilestoreManager { + + private final Logger logger = LoggerFactory.getLogger(getClass()); + + private final IRuntimeManager runtimeManager; + + private final IStoredSettings settings; + + public static final int UNDEFINED_SIZE = -1; + + private static final String METAFILE = "filestore.json"; + + private static final String METAFILE_TMP = "filestore.json.tmp"; + + protected static final Type METAFILE_TYPE = new TypeToken<Collection<FilestoreModel>>() {}.getType(); + + private Map<String, FilestoreModel > fileCache = new ConcurrentHashMap<String, FilestoreModel>(); + + + @Inject + FilestoreManager( + IRuntimeManager runtimeManager) { + this.runtimeManager = runtimeManager; + this.settings = runtimeManager.getSettings(); + } + + @Override + public IManager start() { + + //Try to load any existing metadata + File metadata = new File(getStorageFolder(), METAFILE); + + if (metadata.exists()) { + Collection<FilestoreModel> items = null; + + Gson gson = gson(); + try (FileReader file = new FileReader(metadata)) { + items = gson.fromJson(file, METAFILE_TYPE); + file.close(); + + } catch (IOException e) { + e.printStackTrace(); + } + + for(Iterator<FilestoreModel> itr = items.iterator(); itr.hasNext(); ) { + FilestoreModel model = itr.next(); + fileCache.put(model.oid, model); + } + + logger.info("Loaded {} items from filestore metadata file", fileCache.size()); + } + else + { + logger.info("No filestore metadata file found"); + } + + return this; + } + + @Override + public IManager stop() { + return this; + } + + + @Override + public boolean isValidOid(String oid) { + //NOTE: Assuming SHA256 support only as per git-lfs + return Pattern.matches("[a-fA-F0-9]{64}", oid); + } + + @Override + public FilestoreModel.Status addObject(String oid, long size, UserModel user, RepositoryModel repo) { + + //Handle access control + if (!user.canPush(repo)) { + if (user == UserModel.ANONYMOUS) { + return Status.AuthenticationRequired; + } else { + return Status.Error_Unauthorized; + } + } + + //Handle object details + if (!isValidOid(oid)) { return Status.Error_Invalid_Oid; } + + if (fileCache.containsKey(oid)) { + FilestoreModel item = fileCache.get(oid); + + if (!item.isInErrorState() && (size != UNDEFINED_SIZE) && (item.getSize() != size)) { + return Status.Error_Size_Mismatch; + } + + item.addRepository(repo.name); + + if (item.isInErrorState()) { + item.reset(user, size); + } + } else { + + if (size < 0) {return Status.Error_Invalid_Size; } + if ((getMaxUploadSize() != UNDEFINED_SIZE) && (size > getMaxUploadSize())) { return Status.Error_Exceeds_Size_Limit; } + + FilestoreModel model = new FilestoreModel(oid, size, user, repo.name); + fileCache.put(oid, model); + saveFilestoreModel(model); + } + + return fileCache.get(oid).getStatus(); + } + + @Override + public FilestoreModel.Status uploadBlob(String oid, long size, UserModel user, RepositoryModel repo, InputStream streamIn) { + + //Access control and object logic + Status state = addObject(oid, size, user, repo); + + if (state != Status.Upload_Pending) { + return state; + } + + FilestoreModel model = fileCache.get(oid); + + if (!model.actionUpload(user)) { + return Status.Upload_In_Progress; + } else { + long actualSize = 0; + File file = getStoragePath(oid); + + try { + file.getParentFile().mkdirs(); + file.createNewFile(); + + try (FileOutputStream streamOut = new FileOutputStream(file)) { + + actualSize = IOUtils.copyLarge(streamIn, streamOut); + + streamOut.flush(); + streamOut.close(); + + if (model.getSize() != actualSize) { + model.setStatus(Status.Error_Size_Mismatch, user); + + logger.warn(MessageFormat.format("Failed to upload blob {0} due to size mismatch, expected {1} got {2}", + oid, model.getSize(), actualSize)); + } else { + String actualOid = ""; + + try (FileInputStream fileForHash = new FileInputStream(file)) { + actualOid = DigestUtils.sha256Hex(fileForHash); + fileForHash.close(); + } + + if (oid.equalsIgnoreCase(actualOid)) { + model.setStatus(Status.Available, user); + } else { + model.setStatus(Status.Error_Hash_Mismatch, user); + + logger.warn(MessageFormat.format("Failed to upload blob {0} due to hash mismatch, got {1}", oid, actualOid)); + } + } + } + } catch (Exception e) { + + model.setStatus(Status.Error_Unknown, user); + logger.warn(MessageFormat.format("Failed to upload blob {0}", oid), e); + } finally { + saveFilestoreModel(model); + } + + if (model.isInErrorState()) { + file.delete(); + model.removeRepository(repo.name); + } + } + + return model.getStatus(); + } + + private FilestoreModel.Status canGetObject(String oid, UserModel user, RepositoryModel repo) { + + //Access Control + if (!user.canView(repo)) { + if (user == UserModel.ANONYMOUS) { + return Status.AuthenticationRequired; + } else { + return Status.Error_Unauthorized; + } + } + + //Object Logic + if (!isValidOid(oid)) { + return Status.Error_Invalid_Oid; + } + + if (!fileCache.containsKey(oid)) { + return Status.Unavailable; + } + + FilestoreModel item = fileCache.get(oid); + + if (item.getStatus() == Status.Available) { + return Status.Available; + } + + return Status.Unavailable; + } + + @Override + public FilestoreModel getObject(String oid, UserModel user, RepositoryModel repo) { + + if (canGetObject(oid, user, repo) == Status.Available) { + return fileCache.get(oid); + } + + return null; + } + + @Override + public FilestoreModel.Status downloadBlob(String oid, UserModel user, RepositoryModel repo, OutputStream streamOut) { + + //Access control and object logic + Status status = canGetObject(oid, user, repo); + + if (status != Status.Available) { + return status; + } + + FilestoreModel item = fileCache.get(oid); + + if (streamOut != null) { + try (FileInputStream streamIn = new FileInputStream(getStoragePath(oid))) { + + IOUtils.copyLarge(streamIn, streamOut); + + streamOut.flush(); + streamIn.close(); + } catch (EofException e) { + logger.error(MessageFormat.format("Client aborted connection for {0}", oid), e); + return Status.Error_Unexpected_Stream_End; + } catch (Exception e) { + logger.error(MessageFormat.format("Failed to download blob {0}", oid), e); + return Status.Error_Unknown; + } + } + + return item.getStatus(); + } + + @Override + public List<FilestoreModel> getAllObjects() { + return new ArrayList<FilestoreModel>(fileCache.values()); + } + + @Override + public File getStorageFolder() { + return runtimeManager.getFileOrFolder(Keys.filestore.storageFolder, "${baseFolder}/lfs"); + } + + @Override + public File getStoragePath(String oid) { + return new File(getStorageFolder(), oid.substring(0, 2).concat("/").concat(oid.substring(2))); + } + + @Override + public long getMaxUploadSize() { + return settings.getLong(Keys.filestore.maxUploadSize, -1); + } + + @Override + public long getFilestoreUsedByteCount() { + Iterator<FilestoreModel> iterator = fileCache.values().iterator(); + long total = 0; + + while (iterator.hasNext()) { + + FilestoreModel item = iterator.next(); + if (item.getStatus() == Status.Available) { + total += item.getSize(); + } + } + + return total; + } + + @Override + public long getFilestoreAvailableByteCount() { + + try { + return Files.getFileStore(getStorageFolder().toPath()).getUsableSpace(); + } catch (IOException e) { + logger.error(MessageFormat.format("Failed to retrive available space in Filestore {0}", e)); + } + + return UNDEFINED_SIZE; + }; + + private synchronized void saveFilestoreModel(FilestoreModel model) { + + File metaFile = new File(getStorageFolder(), METAFILE); + File metaFileTmp = new File(getStorageFolder(), METAFILE_TMP); + boolean isNewFile = false; + + try { + if (!metaFile.exists()) { + metaFile.getParentFile().mkdirs(); + metaFile.createNewFile(); + isNewFile = true; + } + FileUtils.copyFile(metaFile, metaFileTmp); + + } catch (IOException e) { + logger.error("Writing filestore model to file {0}, {1}", METAFILE, e); + } + + try (RandomAccessFile fs = new RandomAccessFile(metaFileTmp, "rw")) { + + if (isNewFile) { + fs.writeBytes("["); + } else { + fs.seek(fs.length() - 1); + fs.writeBytes(","); + } + + fs.writeBytes(gson().toJson(model)); + fs.writeBytes("]"); + + fs.close(); + + } catch (IOException e) { + logger.error("Writing filestore model to file {0}, {1}", METAFILE_TMP, e); + } + + try { + if (metaFileTmp.exists()) { + FileUtils.copyFile(metaFileTmp, metaFile); + + metaFileTmp.delete(); + } else { + logger.error("Writing filestore model to file {0}", METAFILE); + } + } + catch (IOException e) { + logger.error("Writing filestore model to file {0}, {1}", METAFILE, e); + } + } + + /* + * Intended for testing purposes only + */ + public void clearFilestoreCache() { + fileCache.clear(); + } + + private static Gson gson(ExclusionStrategy... strategies) { + GsonBuilder builder = new GsonBuilder(); + builder.registerTypeAdapter(Date.class, new GmtDateTypeAdapter()); + if (!ArrayUtils.isEmpty(strategies)) { + builder.setExclusionStrategies(strategies); + } + return builder.create(); + } + +} diff --git a/src/main/java/com/gitblit/manager/GitblitManager.java b/src/main/java/com/gitblit/manager/GitblitManager.java index 8d25b3f5..a34e29d7 100644 --- a/src/main/java/com/gitblit/manager/GitblitManager.java +++ b/src/main/java/com/gitblit/manager/GitblitManager.java @@ -21,6 +21,7 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.io.OutputStream; import java.lang.reflect.Type; import java.text.MessageFormat; import java.util.ArrayList; @@ -58,6 +59,7 @@ import com.gitblit.extensions.RepositoryLifeCycleListener; import com.gitblit.models.FederationModel; import com.gitblit.models.FederationProposal; import com.gitblit.models.FederationSet; +import com.gitblit.models.FilestoreModel; import com.gitblit.models.ForkModel; import com.gitblit.models.GitClientApplication; import com.gitblit.models.Mailing; @@ -132,6 +134,8 @@ public class GitblitManager implements IGitblit { protected final IFederationManager federationManager; + protected final IFilestoreManager filestoreManager; + @Inject public GitblitManager( Provider<IPublicKeyManager> publicKeyManagerProvider, @@ -143,7 +147,8 @@ public class GitblitManager implements IGitblit { IAuthenticationManager authenticationManager, IRepositoryManager repositoryManager, IProjectManager projectManager, - IFederationManager federationManager) { + IFederationManager federationManager, + IFilestoreManager filestoreManager) { this.publicKeyManagerProvider = publicKeyManagerProvider; this.ticketServiceProvider = ticketServiceProvider; @@ -157,6 +162,7 @@ public class GitblitManager implements IGitblit { this.repositoryManager = repositoryManager; this.projectManager = projectManager; this.federationManager = federationManager; + this.filestoreManager = filestoreManager; } @Override @@ -1239,6 +1245,70 @@ public class GitblitManager implements IGitblit { } /* + * FILE STORAGE MANAGER + */ + + @Override + public boolean isValidOid(String oid) { + return filestoreManager.isValidOid(oid); + } + + @Override + public FilestoreModel.Status addObject(String oid, long size, UserModel user, RepositoryModel repo) { + return filestoreManager.addObject(oid, size, user, repo); + } + + @Override + public FilestoreModel getObject(String oid, UserModel user, RepositoryModel repo) { + return filestoreManager.getObject(oid, user, repo); + }; + + @Override + public FilestoreModel.Status uploadBlob(String oid, long size, UserModel user, RepositoryModel repo, InputStream streamIn ) { + return filestoreManager.uploadBlob(oid, size, user, repo, streamIn); + } + + @Override + public FilestoreModel.Status downloadBlob(String oid, UserModel user, RepositoryModel repo, OutputStream streamOut ) { + return filestoreManager.downloadBlob(oid, user, repo, streamOut); + } + + @Override + public List<FilestoreModel> getAllObjects() { + return filestoreManager.getAllObjects(); + } + + @Override + public File getStorageFolder() { + return filestoreManager.getStorageFolder(); + } + + @Override + public File getStoragePath(String oid) { + return filestoreManager.getStoragePath(oid); + } + + @Override + public long getMaxUploadSize() { + return filestoreManager.getMaxUploadSize(); + }; + + @Override + public void clearFilestoreCache() { + filestoreManager.clearFilestoreCache(); + }; + + @Override + public long getFilestoreUsedByteCount() { + return filestoreManager.getFilestoreUsedByteCount(); + }; + + @Override + public long getFilestoreAvailableByteCount() { + return filestoreManager.getFilestoreAvailableByteCount(); + }; + + /* * PLUGIN MANAGER */ @@ -1341,4 +1411,5 @@ public class GitblitManager implements IGitblit { public PluginRelease lookupRelease(String pluginId, String version) { return pluginManager.lookupRelease(pluginId, version); } + } diff --git a/src/main/java/com/gitblit/manager/IFilestoreManager.java b/src/main/java/com/gitblit/manager/IFilestoreManager.java new file mode 100644 index 00000000..0720650c --- /dev/null +++ b/src/main/java/com/gitblit/manager/IFilestoreManager.java @@ -0,0 +1,54 @@ +/* + * Copyright 2015 gitblit.com. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.gitblit.manager; + +import java.io.File; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.List; + +import com.gitblit.models.FilestoreModel; +import com.gitblit.models.RepositoryModel; +import com.gitblit.models.UserModel; + + +public interface IFilestoreManager extends IManager { + + boolean isValidOid(String oid); + + FilestoreModel.Status addObject(String oid, long size, UserModel user, RepositoryModel repo); + + FilestoreModel getObject(String oid, UserModel user, RepositoryModel repo); + + FilestoreModel.Status uploadBlob(String oid, long size, UserModel user, RepositoryModel repo, InputStream streamIn ); + + FilestoreModel.Status downloadBlob(String oid, UserModel user, RepositoryModel repo, OutputStream streamOut ); + + List<FilestoreModel> getAllObjects(); + + File getStorageFolder(); + + File getStoragePath(String oid); + + long getMaxUploadSize(); + + void clearFilestoreCache(); + + long getFilestoreUsedByteCount(); + + long getFilestoreAvailableByteCount(); + +} diff --git a/src/main/java/com/gitblit/manager/IGitblit.java b/src/main/java/com/gitblit/manager/IGitblit.java index 6c5b374c..489de62d 100644 --- a/src/main/java/com/gitblit/manager/IGitblit.java +++ b/src/main/java/com/gitblit/manager/IGitblit.java @@ -33,7 +33,8 @@ public interface IGitblit extends IManager, IAuthenticationManager, IRepositoryManager, IProjectManager, - IFederationManager { + IFederationManager, + IFilestoreManager { /** * Creates a complete user object. diff --git a/src/main/java/com/gitblit/models/FilestoreModel.java b/src/main/java/com/gitblit/models/FilestoreModel.java new file mode 100644 index 00000000..ff7b210e --- /dev/null +++ b/src/main/java/com/gitblit/models/FilestoreModel.java @@ -0,0 +1,159 @@ +/* + * Copyright 2015 gitblit.com. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.gitblit.models; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.NoSuchElementException; + +/** + * A FilestoreModel represents a file stored outside a repository but referenced by the repository using a unique objectID + * + * @author Paul Martin + * + */ +public class FilestoreModel implements Serializable { + + private static final long serialVersionUID = 1L; + + public final String oid; + + private Long size; + private Status status; + + //Audit + private String stateChangedBy; + private Date stateChangedOn; + + //Access Control + private List<String> repositories; + + public FilestoreModel(String id, long expectedSize, UserModel user, String repo) { + oid = id; + size = expectedSize; + status = Status.Upload_Pending; + stateChangedBy = user.getName(); + stateChangedOn = new Date(); + repositories = new ArrayList<String>(); + repositories.add(repo); + } + + public synchronized long getSize() { + return size; + } + + public synchronized Status getStatus() { + return status; + } + + public synchronized String getChangedBy() { + return stateChangedBy; + } + + public synchronized Date getChangedOn() { + return stateChangedOn; + } + + public synchronized void setStatus(Status status, UserModel user) { + this.status = status; + stateChangedBy = user.getName(); + stateChangedOn = new Date(); + } + + public synchronized void reset(UserModel user, long size) { + status = Status.Upload_Pending; + stateChangedBy = user.getName(); + stateChangedOn = new Date(); + this.size = size; + } + + /* + * Handles possible race condition with concurrent connections + * @return true if action can proceed, false otherwise + */ + public synchronized boolean actionUpload(UserModel user) { + if (status == Status.Upload_Pending) { + status = Status.Upload_In_Progress; + stateChangedBy = user.getName(); + stateChangedOn = new Date(); + return true; + } + + return false; + } + + public synchronized boolean isInErrorState() { + return (this.status.value < 0); + } + + public synchronized void addRepository(String repo) { + if (!repositories.contains(repo)) { + repositories.add(repo); + } + } + + public synchronized void removeRepository(String repo) { + repositories.remove(repo); + } + + public static enum Status { + + Deleted(-30), + AuthenticationRequired(-20), + + Error_Unknown(-8), + Error_Unexpected_Stream_End(-7), + Error_Invalid_Oid(-6), + Error_Invalid_Size(-5), + Error_Hash_Mismatch(-4), + Error_Size_Mismatch(-3), + Error_Exceeds_Size_Limit(-2), + Error_Unauthorized(-1), + //Negative values provide additional information and may be treated as 0 when not required + Unavailable(0), + Upload_Pending(1), + Upload_In_Progress(2), + Available(3); + + final int value; + + Status(int value) { + this.value = value; + } + + public int getValue() { + return value; + } + + @Override + public String toString() { + return name().toLowerCase().replace('_', ' '); + } + + public static Status fromState(int state) { + for (Status s : values()) { + if (s.getValue() == state) { + return s; + } + } + throw new NoSuchElementException(String.valueOf(state)); + } + } + +} + diff --git a/src/main/java/com/gitblit/servlet/AccessRestrictionFilter.java b/src/main/java/com/gitblit/servlet/AccessRestrictionFilter.java index ee4a91aa..bfbc0899 100644 --- a/src/main/java/com/gitblit/servlet/AccessRestrictionFilter.java +++ b/src/main/java/com/gitblit/servlet/AccessRestrictionFilter.java @@ -17,6 +17,8 @@ package com.gitblit.servlet; import java.io.IOException;
import java.text.MessageFormat;
+import java.util.Collections;
+import java.util.Iterator;
import javax.servlet.FilterChain;
import javax.servlet.ServletException;
@@ -84,16 +86,17 @@ public abstract class AccessRestrictionFilter extends AuthenticationFilter { *
* @return true if the filter allows repository creation
*/
- protected abstract boolean isCreationAllowed();
+ protected abstract boolean isCreationAllowed(String action);
/**
* Determine if the action may be executed on the repository.
*
* @param repository
* @param action
+ * @param method
* @return true if the action may be performed
*/
- protected abstract boolean isActionAllowed(RepositoryModel repository, String action);
+ protected abstract boolean isActionAllowed(RepositoryModel repository, String action, String method);
/**
* Determine if the repository requires authentication.
@@ -102,7 +105,7 @@ public abstract class AccessRestrictionFilter extends AuthenticationFilter { * @param action
* @return true if authentication required
*/
- protected abstract boolean requiresAuthentication(RepositoryModel repository, String action);
+ protected abstract boolean requiresAuthentication(RepositoryModel repository, String action, String method);
/**
* Determine if the user can access the repository and perform the specified
@@ -126,7 +129,26 @@ public abstract class AccessRestrictionFilter extends AuthenticationFilter { protected RepositoryModel createRepository(UserModel user, String repository, String action) {
return null;
}
-
+
+ /**
+ * Allows authentication header to be altered based on the action requested
+ * Default is WWW-Authenticate
+ * @param action
+ * @return authentication type header
+ */
+ protected String getAuthenticationHeader(String action) {
+ return "WWW-Authenticate";
+ }
+
+ /**
+ * Allows request headers to be used as part of filtering
+ * @param request
+ * @return true (default) if headers are valid, false otherwise
+ */
+ protected boolean hasValidRequestHeader(String action, HttpServletRequest request) {
+ return true;
+ }
+
/**
* doFilter does the actual work of preprocessing the request to ensure that
* the user may proceed.
@@ -163,13 +185,14 @@ public abstract class AccessRestrictionFilter extends AuthenticationFilter { // Load the repository model
RepositoryModel model = repositoryManager.getRepositoryModel(repository);
if (model == null) {
- if (isCreationAllowed()) {
+ if (isCreationAllowed(urlRequestType)) {
if (user == null) {
// challenge client to provide credentials for creation. send 401.
if (runtimeManager.isDebugMode()) {
logger.info(MessageFormat.format("ARF: CREATE CHALLENGE {0}", fullUrl));
}
- httpResponse.setHeader("WWW-Authenticate", CHALLENGE);
+
+ httpResponse.setHeader(getAuthenticationHeader(urlRequestType), CHALLENGE);
httpResponse.sendError(HttpServletResponse.SC_UNAUTHORIZED);
return;
} else {
@@ -188,7 +211,7 @@ public abstract class AccessRestrictionFilter extends AuthenticationFilter { }
// Confirm that the action may be executed on the repository
- if (!isActionAllowed(model, urlRequestType)) {
+ if (!isActionAllowed(model, urlRequestType, httpRequest.getMethod())) {
logger.info(MessageFormat.format("ARF: action {0} on {1} forbidden ({2})",
urlRequestType, model, HttpServletResponse.SC_FORBIDDEN));
httpResponse.sendError(HttpServletResponse.SC_FORBIDDEN);
@@ -210,13 +233,13 @@ public abstract class AccessRestrictionFilter extends AuthenticationFilter { }
// BASIC authentication challenge and response processing
- if (!StringUtils.isEmpty(urlRequestType) && requiresAuthentication(model, urlRequestType)) {
+ if (!StringUtils.isEmpty(urlRequestType) && requiresAuthentication(model, urlRequestType, httpRequest.getMethod())) {
if (user == null) {
// challenge client to provide credentials. send 401.
if (runtimeManager.isDebugMode()) {
logger.info(MessageFormat.format("ARF: CHALLENGE {0}", fullUrl));
}
- httpResponse.setHeader("WWW-Authenticate", CHALLENGE);
+ httpResponse.setHeader(getAuthenticationHeader(urlRequestType), CHALLENGE);
httpResponse.sendError(HttpServletResponse.SC_UNAUTHORIZED);
return;
} else {
@@ -248,4 +271,17 @@ public abstract class AccessRestrictionFilter extends AuthenticationFilter { // pass processing to the restricted servlet.
chain.doFilter(authenticatedRequest, httpResponse);
}
+
+ public static boolean hasContentInRequestHeader(HttpServletRequest request, String headerName, String content)
+ {
+ Iterator<String> headerItr = Collections.list(request.getHeaders(headerName)).iterator();
+
+ while (headerItr.hasNext()) {
+ if (headerItr.next().contains(content)) {
+ return true;
+ }
+ }
+
+ return false;
+ }
}
\ No newline at end of file diff --git a/src/main/java/com/gitblit/servlet/DownloadZipFilter.java b/src/main/java/com/gitblit/servlet/DownloadZipFilter.java index de471482..146f6d4f 100644 --- a/src/main/java/com/gitblit/servlet/DownloadZipFilter.java +++ b/src/main/java/com/gitblit/servlet/DownloadZipFilter.java @@ -81,7 +81,7 @@ public class DownloadZipFilter extends AccessRestrictionFilter { * @return true if the filter allows repository creation
*/
@Override
- protected boolean isCreationAllowed() {
+ protected boolean isCreationAllowed(String action) {
return false;
}
@@ -90,10 +90,11 @@ public class DownloadZipFilter extends AccessRestrictionFilter { *
* @param repository
* @param action
+ * @param method
* @return true if the action may be performed
*/
@Override
- protected boolean isActionAllowed(RepositoryModel repository, String action) {
+ protected boolean isActionAllowed(RepositoryModel repository, String action, String method) {
return true;
}
@@ -102,10 +103,11 @@ public class DownloadZipFilter extends AccessRestrictionFilter { *
* @param repository
* @param action
+ * @param method
* @return true if authentication required
*/
@Override
- protected boolean requiresAuthentication(RepositoryModel repository, String action) {
+ protected boolean requiresAuthentication(RepositoryModel repository, String action, String method) {
return repository.accessRestriction.atLeast(AccessRestrictionType.VIEW);
}
diff --git a/src/main/java/com/gitblit/servlet/FilestoreServlet.java b/src/main/java/com/gitblit/servlet/FilestoreServlet.java new file mode 100644 index 00000000..19751483 --- /dev/null +++ b/src/main/java/com/gitblit/servlet/FilestoreServlet.java @@ -0,0 +1,493 @@ +/* + * Copyright 2015 gitblit.com. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.gitblit.servlet; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.Serializable; +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.google.inject.Inject; +import com.google.inject.Singleton; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.gitblit.Constants; +import com.gitblit.IStoredSettings; +import com.gitblit.models.FilestoreModel; +import com.gitblit.models.RepositoryModel; +import com.gitblit.models.FilestoreModel.Status; +import com.gitblit.manager.FilestoreManager; +import com.gitblit.manager.IGitblit; +import com.gitblit.models.UserModel; +import com.gitblit.utils.JsonUtils; + + +/** + * Handles large file storage as per the Git LFS v1 Batch API + * + * Further details can be found at https://github.com/github/git-lfs + * + * @author Paul Martin + */ +@Singleton +public class FilestoreServlet extends HttpServlet { + + private static final long serialVersionUID = 1L; + public static final int PROTOCOL_VERSION = 1; + + public static final String GIT_LFS_META_MIME = "application/vnd.git-lfs+json"; + + public static final String REGEX_PATH = "^(.*?)/(r|git)/(.*?)/info/lfs/objects/(batch|" + Constants.REGEX_SHA256 + ")"; + public static final int REGEX_GROUP_BASE_URI = 1; + public static final int REGEX_GROUP_PREFIX = 2; + public static final int REGEX_GROUP_REPOSITORY = 3; + public static final int REGEX_GROUP_ENDPOINT = 4; + + protected final Logger logger; + + private static IGitblit gitblit; + + @Inject + public FilestoreServlet(IStoredSettings settings, IGitblit gitblit) { + + super(); + logger = LoggerFactory.getLogger(getClass()); + + FilestoreServlet.gitblit = gitblit; + } + + + /** + * Handles batch upload request (metadata) + * + * @param request + * @param response + * @throws javax.servlet.ServletException + * @throws java.io.IOException + */ + @Override + protected void doPost(HttpServletRequest request, + HttpServletResponse response) throws ServletException ,IOException { + + UrlInfo info = getInfoFromRequest(request); + if (info == null) { + sendError(response, HttpServletResponse.SC_NOT_FOUND); + return; + } + + //Post is for batch operations so no oid should be defined + if (info.oid != null) { + sendError(response, HttpServletResponse.SC_BAD_REQUEST); + return; + } + + IGitLFS.Batch batch = deserialize(request, response, IGitLFS.Batch.class); + + if (batch == null) { + sendError(response, HttpServletResponse.SC_BAD_REQUEST); + return; + } + + UserModel user = getUserOrAnonymous(request); + + IGitLFS.BatchResponse batchResponse = new IGitLFS.BatchResponse(); + + if (batch.operation.equalsIgnoreCase("upload")) { + for (IGitLFS.Request item : batch.objects) { + + Status state = gitblit.addObject(item.oid, item.size, user, info.repository); + + batchResponse.objects.add(getResponseForUpload(info.baseUrl, item.oid, item.size, user.getName(), info.repository.name, state)); + } + } else if (batch.operation.equalsIgnoreCase("download")) { + for (IGitLFS.Request item : batch.objects) { + + Status state = gitblit.downloadBlob(item.oid, user, info.repository, null); + batchResponse.objects.add(getResponseForDownload(info.baseUrl, item.oid, item.size, user.getName(), info.repository.name, state)); + } + } else { + sendError(response, HttpServletResponse.SC_NOT_IMPLEMENTED); + return; + } + + response.setStatus(HttpServletResponse.SC_OK); + serialize(response, batchResponse); + } + + /** + * Handles the actual upload (BLOB) + * + * @param request + * @param response + * @throws javax.servlet.ServletException + * @throws java.io.IOException + */ + @Override + protected void doPut(HttpServletRequest request, + HttpServletResponse response) throws ServletException ,IOException { + + UrlInfo info = getInfoFromRequest(request); + + if (info == null) { + sendError(response, HttpServletResponse.SC_NOT_FOUND); + return; + } + + //Put is a singular operation so must have oid + if (info.oid == null) { + sendError(response, HttpServletResponse.SC_BAD_REQUEST); + return; + } + + UserModel user = getUserOrAnonymous(request); + long size = FilestoreManager.UNDEFINED_SIZE; + + + + FilestoreModel.Status status = gitblit.uploadBlob(info.oid, size, user, info.repository, request.getInputStream()); + IGitLFS.Response responseObject = getResponseForUpload(info.baseUrl, info.oid, size, user.getName(), info.repository.name, status); + + logger.info(MessageFormat.format("FILESTORE-AUDIT {0}:{4} {1} {2}@{3}", + "PUT", info.oid, user.getName(), info.repository.name, status.toString() )); + + if (responseObject.error == null) { + response.setStatus(responseObject.successCode); + } else { + serialize(response, responseObject.error); + } + }; + + /** + * Handles a download + * Treated as hypermedia request if accept header contains Git-LFS MIME + * otherwise treated as a download of the blob + * @param request + * @param response + * @throws javax.servlet.ServletException + * @throws java.io.IOException + */ + @Override + protected void doGet(HttpServletRequest request, + HttpServletResponse response) throws ServletException ,IOException { + + UrlInfo info = getInfoFromRequest(request); + + if (info == null || info.oid == null) { + sendError(response, HttpServletResponse.SC_NOT_FOUND); + return; + } + + UserModel user = getUserOrAnonymous(request); + + FilestoreModel model = gitblit.getObject(info.oid, user, info.repository); + long size = FilestoreManager.UNDEFINED_SIZE; + + boolean isMetaRequest = AccessRestrictionFilter.hasContentInRequestHeader(request, "Accept", GIT_LFS_META_MIME); + FilestoreModel.Status status = Status.Unavailable; + + if (model != null) { + size = model.getSize(); + status = model.getStatus(); + } + + if (!isMetaRequest) { + status = gitblit.downloadBlob(info.oid, user, info.repository, response.getOutputStream()); + + logger.info(MessageFormat.format("FILESTORE-AUDIT {0}:{4} {1} {2}@{3}", + "GET", info.oid, user.getName(), info.repository.name, status.toString() )); + } + + if (status == Status.Error_Unexpected_Stream_End) { + return; + } + + IGitLFS.Response responseObject = getResponseForDownload(info.baseUrl, + info.oid, size, user.getName(), info.repository.name, status); + + if (responseObject.error == null) { + response.setStatus(responseObject.successCode); + + if (isMetaRequest) { + serialize(response, responseObject); + } + } else { + response.setStatus(responseObject.error.code); + serialize(response, responseObject.error); + } + }; + + private void sendError(HttpServletResponse response, int code) throws IOException { + + String msg = ""; + + switch (code) + { + case HttpServletResponse.SC_NOT_FOUND: msg = "Not Found"; break; + case HttpServletResponse.SC_NOT_IMPLEMENTED: msg = "Not Implemented"; break; + case HttpServletResponse.SC_BAD_REQUEST: msg = "Malformed Git-LFS request"; break; + + default: msg = "Unknown Error"; + } + + response.setStatus(code); + serialize(response, new IGitLFS.ObjectError(code, msg)); + } + + @SuppressWarnings("incomplete-switch") + private IGitLFS.Response getResponseForUpload(String baseUrl, String oid, long size, String user, String repo, FilestoreModel.Status state) { + + switch (state) { + case AuthenticationRequired: + return new IGitLFS.Response(oid, size, 401, MessageFormat.format("Authentication required to write to repository {0}", repo)); + case Error_Unauthorized: + return new IGitLFS.Response(oid, size, 403, MessageFormat.format("User {0}, does not have write permissions to repository {1}", user, repo)); + case Error_Exceeds_Size_Limit: + return new IGitLFS.Response(oid, size, 509, MessageFormat.format("Object is larger than allowed limit of {1}", gitblit.getMaxUploadSize())); + case Error_Hash_Mismatch: + return new IGitLFS.Response(oid, size, 422, "Hash mismatch"); + case Error_Invalid_Oid: + return new IGitLFS.Response(oid, size, 422, MessageFormat.format("{0} is not a valid oid", oid)); + case Error_Invalid_Size: + return new IGitLFS.Response(oid, size, 422, MessageFormat.format("{0} is not a valid size", size)); + case Error_Size_Mismatch: + return new IGitLFS.Response(oid, size, 422, "Object size mismatch"); + case Deleted: + return new IGitLFS.Response(oid, size, 410, "Object was deleted : ".concat("TBD Reason") ); + case Upload_In_Progress: + return new IGitLFS.Response(oid, size, 503, "File currently being uploaded by another user"); + case Unavailable: + return new IGitLFS.Response(oid, size, 404, MessageFormat.format("Repository {0}, does not exist for user {1}", repo, user)); + case Upload_Pending: + return new IGitLFS.Response(oid, size, 202, "upload", getObjectUri(baseUrl, repo, oid) ); + case Available: + return new IGitLFS.Response(oid, size, 200, "upload", getObjectUri(baseUrl, repo, oid) ); + } + + return new IGitLFS.Response(oid, size, 500, "Unknown Error"); + } + + @SuppressWarnings("incomplete-switch") + private IGitLFS.Response getResponseForDownload(String baseUrl, String oid, long size, String user, String repo, FilestoreModel.Status state) { + + switch (state) { + case Error_Unauthorized: + return new IGitLFS.Response(oid, size, 403, MessageFormat.format("User {0}, does not have read permissions to repository {1}", user, repo)); + case Error_Invalid_Oid: + return new IGitLFS.Response(oid, size, 422, MessageFormat.format("{0} is not a valid oid", oid)); + case Error_Unknown: + return new IGitLFS.Response(oid, size, 500, "Unknown Error"); + case Deleted: + return new IGitLFS.Response(oid, size, 410, "Object was deleted : ".concat("TBD Reason") ); + case Available: + return new IGitLFS.Response(oid, size, 200, "download", getObjectUri(baseUrl, repo, oid) ); + } + + return new IGitLFS.Response(oid, size, 404, "Object not available"); + } + + + private String getObjectUri(String baseUrl, String repo, String oid) { + return baseUrl + "/" + repo + "/" + Constants.R_LFS + "objects/" + oid; + } + + + protected void serialize(HttpServletResponse response, Object o) throws IOException { + if (o != null) { + // Send JSON response + String json = JsonUtils.toJsonString(o); + response.setCharacterEncoding(Constants.ENCODING); + response.setContentType(GIT_LFS_META_MIME); + response.getWriter().append(json); + } + } + + protected <X> X deserialize(HttpServletRequest request, HttpServletResponse response, + Class<X> clazz) { + + String json = ""; + try { + + json = readJson(request, response); + + return JsonUtils.fromJsonString(json.toString(), clazz); + + } catch (Exception e) { + //Intentional silent fail + } + + return null; + } + + private String readJson(HttpServletRequest request, HttpServletResponse response) + throws IOException { + BufferedReader reader = request.getReader(); + StringBuilder json = new StringBuilder(); + String line = null; + while ((line = reader.readLine()) != null) { + json.append(line); + } + reader.close(); + + if (json.length() == 0) { + logger.error(MessageFormat.format("Failed to receive json data from {0}", + request.getRemoteAddr())); + response.setStatus(HttpServletResponse.SC_BAD_REQUEST); + return null; + } + return json.toString(); + } + + private UserModel getUserOrAnonymous(HttpServletRequest r) { + UserModel user = (UserModel) r.getUserPrincipal(); + if (user != null) { return user; } + return UserModel.ANONYMOUS; + } + + private static class UrlInfo { + public RepositoryModel repository; + public String oid; + public String baseUrl; + + public UrlInfo(RepositoryModel repo, String oid, String baseUrl) { + this.repository = repo; + this.oid = oid; + this.baseUrl = baseUrl; + } + } + + public static UrlInfo getInfoFromRequest(HttpServletRequest httpRequest) { + + String url = httpRequest.getRequestURL().toString(); + Pattern p = Pattern.compile(REGEX_PATH); + Matcher m = p.matcher(url); + + + if (m.find()) { + RepositoryModel repo = gitblit.getRepositoryModel(m.group(REGEX_GROUP_REPOSITORY)); + String baseUrl = m.group(REGEX_GROUP_BASE_URI) + "/" + m.group(REGEX_GROUP_PREFIX); + + if (m.group(REGEX_GROUP_ENDPOINT).equals("batch")) { + return new UrlInfo(repo, null, baseUrl); + } else { + return new UrlInfo(repo, m.group(REGEX_GROUP_ENDPOINT), baseUrl); + } + } + + return null; + } + + + public interface IGitLFS { + + @SuppressWarnings("serial") + public class Request implements Serializable + { + public String oid; + public long size; + } + + + @SuppressWarnings("serial") + public class Batch implements Serializable + { + public String operation; + public List<Request> objects; + } + + + @SuppressWarnings("serial") + public class Response implements Serializable + { + public String oid; + public long size; + public Map<String, HyperMediaLink> actions; + public ObjectError error; + public transient int successCode; + + public Response(String id, long itemSize, int errorCode, String errorText) { + oid = id; + size = itemSize; + actions = null; + successCode = 0; + error = new ObjectError(errorCode, errorText); + } + + public Response(String id, long itemSize, int actionCode, String action, String uri) { + oid = id; + size = itemSize; + error = null; + successCode = actionCode; + actions = new HashMap<String, HyperMediaLink>(); + actions.put(action, new HyperMediaLink(action, uri)); + } + + } + + @SuppressWarnings("serial") + public class BatchResponse implements Serializable { + public List<Response> objects; + + public BatchResponse() { + objects = new ArrayList<Response>(); + } + } + + + @SuppressWarnings("serial") + public class ObjectError implements Serializable + { + public String message; + public int code; + public String documentation_url; + public Integer request_id; + + public ObjectError(int errorCode, String errorText) { + code = errorCode; + message = errorText; + request_id = null; + } + } + + @SuppressWarnings("serial") + public class HyperMediaLink implements Serializable + { + public String href; + public transient String header; + //public Date expires_at; + + public HyperMediaLink(String action, String uri) { + header = action; + href = uri; + } + } + } + + + +} diff --git a/src/main/java/com/gitblit/servlet/GitFilter.java b/src/main/java/com/gitblit/servlet/GitFilter.java index b29fdb6a..27408f02 100644 --- a/src/main/java/com/gitblit/servlet/GitFilter.java +++ b/src/main/java/com/gitblit/servlet/GitFilter.java @@ -19,6 +19,7 @@ import java.text.MessageFormat; import com.google.inject.Inject;
import com.google.inject.Singleton;
+
import javax.servlet.http.HttpServletRequest;
import com.gitblit.Constants.AccessRestrictionType;
@@ -48,9 +49,11 @@ public class GitFilter extends AccessRestrictionFilter { protected static final String gitReceivePack = "/git-receive-pack";
protected static final String gitUploadPack = "/git-upload-pack";
-
+
+ protected static final String gitLfs = "/info/lfs";
+
protected static final String[] suffixes = { gitReceivePack, gitUploadPack, "/info/refs", "/HEAD",
- "/objects" };
+ "/objects", gitLfs };
private IStoredSettings settings;
@@ -116,6 +119,8 @@ public class GitFilter extends AccessRestrictionFilter { return gitReceivePack;
} else if (suffix.contains("?service=git-upload-pack")) {
return gitUploadPack;
+ } else if (suffix.startsWith(gitLfs)) {
+ return gitLfs;
} else {
return gitUploadPack;
}
@@ -144,7 +149,13 @@ public class GitFilter extends AccessRestrictionFilter { * @return true if the server allows repository creation on-push
*/
@Override
- protected boolean isCreationAllowed() {
+ protected boolean isCreationAllowed(String action) {
+
+ //Repository must already exist before large files can be deposited
+ if (action.equals(gitLfs)) {
+ return false;
+ }
+
return settings.getBoolean(Keys.git.allowCreateOnPush, true);
}
@@ -156,9 +167,15 @@ public class GitFilter extends AccessRestrictionFilter { * @return true if the action may be performed
*/
@Override
- protected boolean isActionAllowed(RepositoryModel repository, String action) {
+ protected boolean isActionAllowed(RepositoryModel repository, String action, String method) {
// the log here has been moved into ReceiveHook to provide clients with
// error messages
+ if (gitLfs.equals(action)) {
+ if (!method.matches("GET|POST|PUT|HEAD")) {
+ return false;
+ }
+ }
+
return true;
}
@@ -172,16 +189,25 @@ public class GitFilter extends AccessRestrictionFilter { *
* @param repository
* @param action
+ * @param method
* @return true if authentication required
*/
@Override
- protected boolean requiresAuthentication(RepositoryModel repository, String action) {
+ protected boolean requiresAuthentication(RepositoryModel repository, String action, String method) {
if (gitUploadPack.equals(action)) {
// send to client
return repository.accessRestriction.atLeast(AccessRestrictionType.CLONE);
} else if (gitReceivePack.equals(action)) {
// receive from client
return repository.accessRestriction.atLeast(AccessRestrictionType.PUSH);
+ } else if (gitLfs.equals(action)) {
+
+ if (method.matches("GET|HEAD")) {
+ return repository.accessRestriction.atLeast(AccessRestrictionType.CLONE);
+ } else {
+ //NOTE: Treat POST as PUT as as without reading message type cannot determine
+ return repository.accessRestriction.atLeast(AccessRestrictionType.PUSH);
+ }
}
return false;
}
@@ -230,6 +256,12 @@ public class GitFilter extends AccessRestrictionFilter { @Override
protected RepositoryModel createRepository(UserModel user, String repository, String action) {
boolean isPush = !StringUtils.isEmpty(action) && gitReceivePack.equals(action);
+
+ if (action.equals(gitLfs)) {
+ //Repository must already exist for any filestore actions
+ return null;
+ }
+
if (isPush) {
if (user.canCreate(repository)) {
// user is pushing to a new repository
@@ -281,4 +313,40 @@ public class GitFilter extends AccessRestrictionFilter { // repository could not be created or action was not a push
return null;
}
+
+ /**
+ * Git lfs action uses an alternative authentication header,
+ *
+ * @param action
+ * @return
+ */
+ @Override
+ protected String getAuthenticationHeader(String action) {
+
+ if (action.equals(gitLfs)) {
+ return "LFS-Authenticate";
+ }
+
+ return super.getAuthenticationHeader(action);
+ }
+
+ /**
+ * Interrogates the request headers based on the action
+ * @param action
+ * @param request
+ * @return
+ */
+ @Override
+ protected boolean hasValidRequestHeader(String action,
+ HttpServletRequest request) {
+
+ if (action.equals(gitLfs) && request.getMethod().equals("POST")) {
+ if ( !hasContentInRequestHeader(request, "Accept", FilestoreServlet.GIT_LFS_META_MIME)
+ || !hasContentInRequestHeader(request, "Content-Type", FilestoreServlet.GIT_LFS_META_MIME)) {
+ return false;
+ }
+ }
+
+ return super.hasValidRequestHeader(action, request);
+ }
}
diff --git a/src/main/java/com/gitblit/servlet/GitblitContext.java b/src/main/java/com/gitblit/servlet/GitblitContext.java index 077624c2..fb8f6b9d 100644 --- a/src/main/java/com/gitblit/servlet/GitblitContext.java +++ b/src/main/java/com/gitblit/servlet/GitblitContext.java @@ -44,6 +44,7 @@ import com.gitblit.guice.CoreModule; import com.gitblit.guice.WebModule; import com.gitblit.manager.IAuthenticationManager; import com.gitblit.manager.IFederationManager; +import com.gitblit.manager.IFilestoreManager; import com.gitblit.manager.IGitblit; import com.gitblit.manager.IManager; import com.gitblit.manager.INotificationManager; @@ -204,6 +205,7 @@ public class GitblitContext extends GuiceServletContextListener { startManager(injector, ITicketService.class); startManager(injector, IGitblit.class); startManager(injector, IServicesManager.class); + startManager(injector, IFilestoreManager.class); // start the plugin manager last so that plugins can depend on // deterministic access to all other managers in their start() methods diff --git a/src/main/java/com/gitblit/servlet/RawFilter.java b/src/main/java/com/gitblit/servlet/RawFilter.java index fe4af040..8913a197 100644 --- a/src/main/java/com/gitblit/servlet/RawFilter.java +++ b/src/main/java/com/gitblit/servlet/RawFilter.java @@ -98,7 +98,7 @@ public class RawFilter extends AccessRestrictionFilter { * @return true if the filter allows repository creation
*/
@Override
- protected boolean isCreationAllowed() {
+ protected boolean isCreationAllowed(String action) {
return false;
}
@@ -107,10 +107,11 @@ public class RawFilter extends AccessRestrictionFilter { *
* @param repository
* @param action
+ * @param method
* @return true if the action may be performed
*/
@Override
- protected boolean isActionAllowed(RepositoryModel repository, String action) {
+ protected boolean isActionAllowed(RepositoryModel repository, String action, String method) {
return true;
}
@@ -119,10 +120,11 @@ public class RawFilter extends AccessRestrictionFilter { *
* @param repository
* @param action
+ * @param method
* @return true if authentication required
*/
@Override
- protected boolean requiresAuthentication(RepositoryModel repository, String action) {
+ protected boolean requiresAuthentication(RepositoryModel repository, String action, String method) {
return repository.accessRestriction.atLeast(AccessRestrictionType.VIEW);
}
diff --git a/src/main/java/com/gitblit/utils/JsonUtils.java b/src/main/java/com/gitblit/utils/JsonUtils.java index be7148cb..f389776b 100644 --- a/src/main/java/com/gitblit/utils/JsonUtils.java +++ b/src/main/java/com/gitblit/utils/JsonUtils.java @@ -46,6 +46,7 @@ import com.google.gson.GsonBuilder; import com.google.gson.JsonDeserializationContext;
import com.google.gson.JsonDeserializer;
import com.google.gson.JsonElement;
+import com.google.gson.JsonParseException;
import com.google.gson.JsonPrimitive;
import com.google.gson.JsonSerializationContext;
import com.google.gson.JsonSerializer;
@@ -79,23 +80,29 @@ public class JsonUtils { /**
* Convert a json string to an object of the specified type.
- *
+ *
* @param json
* @param clazz
- * @return an object
+ * @return the deserialized object
+ * @throws JsonParseException
+ * @throws JsonSyntaxException
*/
- public static <X> X fromJsonString(String json, Class<X> clazz) {
+ public static <X> X fromJsonString(String json, Class<X> clazz) throws JsonParseException,
+ JsonSyntaxException {
return gson().fromJson(json, clazz);
}
/**
* Convert a json string to an object of the specified type.
- *
+ *
* @param json
- * @param clazz
- * @return an object
+ * @param type
+ * @return the deserialized object
+ * @throws JsonParseException
+ * @throws JsonSyntaxException
*/
- public static <X> X fromJsonString(String json, Type type) {
+ public static <X> X fromJsonString(String json, Type type) throws JsonParseException,
+ JsonSyntaxException {
return gson().fromJson(json, type);
}
diff --git a/src/main/java/com/gitblit/wicket/FilestoreUI.java b/src/main/java/com/gitblit/wicket/FilestoreUI.java new file mode 100644 index 00000000..8837ba18 --- /dev/null +++ b/src/main/java/com/gitblit/wicket/FilestoreUI.java @@ -0,0 +1,62 @@ +/* + * Copyright 2015 gitblit.com. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.gitblit.wicket; + +import org.apache.wicket.markup.html.basic.Label; + +import com.gitblit.models.FilestoreModel; +import com.gitblit.models.FilestoreModel.Status; + +/** + * Common filestore ui methods and classes. + * + * @author Paul Martin + * + */ +public class FilestoreUI { + + public static Label getStatusIcon(String wicketId, FilestoreModel item) { + return getStatusIcon(wicketId, item.getStatus()); + } + + public static Label getStatusIcon(String wicketId, Status status) { + Label label = new Label(wicketId); + + switch (status) { + case Upload_Pending: + WicketUtils.setCssClass(label, "fa fa-spinner fa-fw file-negative"); + break; + case Upload_In_Progress: + WicketUtils.setCssClass(label, "fa fa-spinner fa-spin fa-fw file-positive"); + break; + case Available: + WicketUtils.setCssClass(label, "fa fa-check fa-fw file-positive"); + break; + case Deleted: + WicketUtils.setCssClass(label, "fa fa-ban fa-fw file-negative"); + break; + case Unavailable: + WicketUtils.setCssClass(label, "fa fa-times fa-fw file-negative"); + break; + default: + WicketUtils.setCssClass(label, "fa fa-exclamation-triangle fa-fw file-negative"); + } + WicketUtils.setHtmlTooltip(label, status.toString()); + + return label; + } + +} diff --git a/src/main/java/com/gitblit/wicket/GitBlitWebApp.java b/src/main/java/com/gitblit/wicket/GitBlitWebApp.java index 359040b5..296c2544 100644 --- a/src/main/java/com/gitblit/wicket/GitBlitWebApp.java +++ b/src/main/java/com/gitblit/wicket/GitBlitWebApp.java @@ -37,6 +37,7 @@ import com.gitblit.Keys; import com.gitblit.extensions.GitblitWicketPlugin; import com.gitblit.manager.IAuthenticationManager; import com.gitblit.manager.IFederationManager; +import com.gitblit.manager.IFilestoreManager; import com.gitblit.manager.IGitblit; import com.gitblit.manager.INotificationManager; import com.gitblit.manager.IPluginManager; @@ -63,6 +64,7 @@ import com.gitblit.wicket.pages.EditRepositoryPage; import com.gitblit.wicket.pages.EditTicketPage; import com.gitblit.wicket.pages.ExportTicketPage; import com.gitblit.wicket.pages.FederationRegistrationPage; +import com.gitblit.wicket.pages.FilestorePage; import com.gitblit.wicket.pages.ForkPage; import com.gitblit.wicket.pages.ForksPage; import com.gitblit.wicket.pages.GitSearchPage; @@ -131,6 +133,8 @@ public class GitBlitWebApp extends WebApplication implements GitblitWicketApp { private final IGitblit gitblit; private final IServicesManager services; + + private final IFilestoreManager filestoreManager; @Inject public GitBlitWebApp( @@ -145,7 +149,8 @@ public class GitBlitWebApp extends WebApplication implements GitblitWicketApp { IProjectManager projectManager, IFederationManager federationManager, IGitblit gitblit, - IServicesManager services) { + IServicesManager services, + IFilestoreManager filestoreManager) { super(); this.publicKeyManagerProvider = publicKeyManagerProvider; @@ -162,6 +167,7 @@ public class GitBlitWebApp extends WebApplication implements GitblitWicketApp { this.federationManager = federationManager; this.gitblit = gitblit; this.services = services; + this.filestoreManager = filestoreManager; } @Override @@ -238,6 +244,9 @@ public class GitBlitWebApp extends WebApplication implements GitblitWicketApp { mount("/user", UserPage.class, "user"); mount("/forks", ForksPage.class, "r"); mount("/fork", ForkPage.class, "r"); + + // filestore URL + mount("/filestore", FilestorePage.class); // allow started Wicket plugins to initialize for (PluginWrapper pluginWrapper : pluginManager.getPlugins()) { @@ -476,4 +485,9 @@ public class GitBlitWebApp extends WebApplication implements GitblitWicketApp { public static GitBlitWebApp get() { return (GitBlitWebApp) WebApplication.get(); } + + @Override + public IFilestoreManager filestore() { + return filestoreManager; + } } diff --git a/src/main/java/com/gitblit/wicket/GitBlitWebApp.properties b/src/main/java/com/gitblit/wicket/GitBlitWebApp.properties index d8027548..36c416e7 100644 --- a/src/main/java/com/gitblit/wicket/GitBlitWebApp.properties +++ b/src/main/java/com/gitblit/wicket/GitBlitWebApp.properties @@ -764,4 +764,10 @@ gb.deleteRepositoryHeader = Delete Repository gb.deleteRepositoryDescription = Deleted repositories will be unrecoverable. gb.show_whitespace = show whitespace gb.ignore_whitespace = ignore whitespace -gb.allRepositories = All Repositories
\ No newline at end of file +gb.allRepositories = All Repositories +gb.oid = object id +gb.filestore = filestore +gb.filestoreStats = Filestore contains {0} files with a total size of {1}. ({2} remaining) +gb.statusChangedOn = status changed on +gb.statusChangedBy = status changed by +gb.filestoreHelp = How to use the Filestore?
\ No newline at end of file diff --git a/src/main/java/com/gitblit/wicket/GitblitWicketApp.java b/src/main/java/com/gitblit/wicket/GitblitWicketApp.java index 3041c5da..fefa0f4a 100644 --- a/src/main/java/com/gitblit/wicket/GitblitWicketApp.java +++ b/src/main/java/com/gitblit/wicket/GitblitWicketApp.java @@ -8,6 +8,7 @@ import org.apache.wicket.markup.html.WebPage; import com.gitblit.IStoredSettings; import com.gitblit.manager.IAuthenticationManager; import com.gitblit.manager.IFederationManager; +import com.gitblit.manager.IFilestoreManager; import com.gitblit.manager.IGitblit; import com.gitblit.manager.INotificationManager; import com.gitblit.manager.IPluginManager; @@ -74,5 +75,7 @@ public interface GitblitWicketApp { public abstract ITicketService tickets(); public abstract TimeZone getTimezone(); + + public abstract IFilestoreManager filestore(); }
\ No newline at end of file diff --git a/src/main/java/com/gitblit/wicket/pages/FilestorePage.html b/src/main/java/com/gitblit/wicket/pages/FilestorePage.html new file mode 100644 index 00000000..e373e704 --- /dev/null +++ b/src/main/java/com/gitblit/wicket/pages/FilestorePage.html @@ -0,0 +1,37 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> +<html xmlns="http://www.w3.org/1999/xhtml" + xmlns:wicket="http://wicket.apache.org/dtds.data/wicket-xhtml1.3-strict.dtd" + xml:lang="en" + lang="en"> + +<body> +<wicket:extend> +<div class="container"> + + <div class="markdown" style="padding: 10px 0px 5px 0px;"> + <span wicket:id="repositoriesMessage">[repositories message]</span> + <span style="float:right"><a href="#" wicket:id="filestoreHelp"><span wicket:id="helpMessage">[help message]</span></a></span> + </div> + + <table class="repositories"> + <tr> + <th><wicket:message key="gb.status">[Object status]</wicket:message></th> + <th><wicket:message key="gb.statusChangedOn">[changedOn]</wicket:message></th> + <th><wicket:message key="gb.statusChangedBy">[changedBy]</wicket:message></th> + <th><wicket:message key="gb.oid">[Object ID]</wicket:message></th> + <th><wicket:message key="gb.size">[file size]</wicket:message></th> + </tr> + <tbody> + <tr wicket:id="fileRow"> + <td><center><span class="list" wicket:id="status">[Object state]</span></center></td> + <td><span class="list" wicket:id="on">[changedOn]</span></td> + <td><span class="list" wicket:id="by">[changedBy]</span></td> + <td class="sha256"><span class="list" wicket:id="oid">[Object ID]</span></td> + <td><span class="list" wicket:id="size">[file size]</span></td> + </tr> + </tbody> + </table> +</div> +</wicket:extend> +</body> +</html>
\ No newline at end of file diff --git a/src/main/java/com/gitblit/wicket/pages/FilestorePage.java b/src/main/java/com/gitblit/wicket/pages/FilestorePage.java new file mode 100644 index 00000000..5f103edd --- /dev/null +++ b/src/main/java/com/gitblit/wicket/pages/FilestorePage.java @@ -0,0 +1,114 @@ +/* + * Copyright 2015 gitblit.com. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.gitblit.wicket.pages; + +import java.text.DateFormat; +import java.text.MessageFormat; +import java.text.SimpleDateFormat; +import java.util.List; + +import org.apache.commons.io.FileUtils; +import org.apache.wicket.Component; +import org.apache.wicket.markup.html.basic.Label; +import org.apache.wicket.markup.html.link.BookmarkablePageLink; +import org.apache.wicket.markup.repeater.Item; +import org.apache.wicket.markup.repeater.data.DataView; +import org.apache.wicket.markup.repeater.data.ListDataProvider; + +import com.gitblit.Constants; +import com.gitblit.Keys; +import com.gitblit.models.FilestoreModel; +import com.gitblit.models.UserModel; +import com.gitblit.wicket.FilestoreUI; +import com.gitblit.wicket.GitBlitWebSession; +import com.gitblit.wicket.WicketUtils; + +/** + * Page to display the current status of the filestore. + * Certain errors also displayed to aid in fault finding + * + * @author Paul Martin + * + * + */ +public class FilestorePage extends RootPage { + + public FilestorePage() { + super(); + setupPage("", ""); + // check to see if we should display a login message + boolean authenticateView = app().settings().getBoolean(Keys.web.authenticateViewPages, true); + if (authenticateView && !GitBlitWebSession.get().isLoggedIn()) { + String messageSource = app().settings().getString(Keys.web.loginMessage, "gitblit"); + return; + } + + final List<FilestoreModel> files = app().filestore().getAllObjects(); + final long nBytesUsed = app().filestore().getFilestoreUsedByteCount(); + final long nBytesAvailable = app().filestore().getFilestoreAvailableByteCount(); + + // Load the markdown welcome message + String messageSource = app().settings().getString(Keys.web.repositoriesMessage, "gitblit"); + String message = MessageFormat.format(getString("gb.filestoreStats"), files.size(), + FileUtils.byteCountToDisplaySize(nBytesUsed), FileUtils.byteCountToDisplaySize(nBytesAvailable) ); + + Component repositoriesMessage = new Label("repositoriesMessage", message) + .setEscapeModelStrings(false).setVisible(message.length() > 0); + + add(repositoriesMessage); + + BookmarkablePageLink<Void> helpLink = new BookmarkablePageLink<Void>("filestoreHelp", FilestoreUsage.class); + helpLink.add(new Label("helpMessage", getString("gb.filestoreHelp"))); + add(helpLink); + + + DataView<FilestoreModel> filesView = new DataView<FilestoreModel>("fileRow", + new ListDataProvider<FilestoreModel>(files)) { + private static final long serialVersionUID = 1L; + private int counter; + + @Override + protected void onBeforeRender() { + super.onBeforeRender(); + counter = 0; + } + + @Override + public void populateItem(final Item<FilestoreModel> item) { + final FilestoreModel entry = item.getModelObject(); + + DateFormat dateFormater = new SimpleDateFormat(Constants.ISO8601); + + UserModel user = app().users().getUserModel(entry.getChangedBy()); + user = user == null ? UserModel.ANONYMOUS : user; + + Label icon = FilestoreUI.getStatusIcon("status", entry); + item.add(icon); + item.add(new Label("on", dateFormater.format(entry.getChangedOn()))); + item.add(new Label("by", user.getDisplayName())); + + item.add(new Label("oid", entry.oid)); + item.add(new Label("size", FileUtils.byteCountToDisplaySize(entry.getSize()))); + + WicketUtils.setAlternatingBackground(item, counter); + counter++; + } + + }; + + add(filesView); + } +} diff --git a/src/main/java/com/gitblit/wicket/pages/FilestoreUsage.html b/src/main/java/com/gitblit/wicket/pages/FilestoreUsage.html new file mode 100644 index 00000000..e9bff47c --- /dev/null +++ b/src/main/java/com/gitblit/wicket/pages/FilestoreUsage.html @@ -0,0 +1,69 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> +<html xmlns="http://www.w3.org/1999/xhtml" + xmlns:wicket="http://wicket.apache.org/dtds.data/wicket-xhtml1.3-strict.dtd" + xml:lang="en" + lang="en"> + +<body> +<wicket:extend> +<div class="container"> +<div class="markdown"> +<div class="row"> +<div class="span10 offset1"> + + <div class="alert alert-danger"> + <h3><center>Using the Filestore</center></h3> + <p> + <strong>All clients intending to use the filestore must first install the <a href="https://git-lfs.github.com/">Git-LFS Client</a> and then run <code>git lfs init</code> to register the hooks globally.</strong><br/> + <i>This version of GitBlit has been verified with Git-LFS client version 0.6.0 which requires Git v1.8.2 or higher.</i> + </p> + </div> + + <h3>Clone</h3> + <p> + Just <code>git clone</code> as usual, no further action is required as GitBlit is configured to use the default Git-LFS end point <code>{repository}/info/lfs/objects/</code>.<br/> + <i>If the repository uses a 3rd party Git-LFS server you will need to <a href="https://github.com/github/git-lfs/blob/master/docs/spec.md#the-server">manually configure the correct endpoints</a></i>. + </p> + + <h3>Add</h3> + <p>After configuring the file types or paths to be tracked using <code>git lfs track "*.bin"</code> just add files as usual with <code>git add</code> command.<br/> + <i>Tracked files can also be configured manually using the <code>.gitattributes</code> file</i>.</p> + + <h3>Remove</h3> + <p>When you remove a Git-LFS tracked file only the pointer file will be removed from your repository.<br/> + <i>All files remain on the server to allow previous versions to be checked out.</i> + </p> + + <h3>Learn more...</h3> + <p><a href="https://github.com/github/git-lfs/blob/master/docs/spec.md">See the current Git-LFS specification for further details</a>.</p> + <br /> + + <div class="alert alert-warn"> + <h3><center>Limitations & Warnings</center></h3> + <p>GitBlit currently provides a server-only implementation of the opensource Git-LFS API, <a href="https://github.com/github/git-lfs/wiki/Implementations">other implementations</a> are available.<br/> + However, until <a href="https://bugs.eclipse.org/bugs/show_bug.cgi?id=470333">JGit provides Git-LFS client capabilities</a> some GitBlit features may not be fully supported when using the filestore. + Notably: + <ul> + <li>Mirroring a repository that uses Git-LFS - Only the pointer files, not the large files, are mirrored.</li> + <li>Federation - Only the pointer files, not the large files, are transfered.</li> + </ul> + </p> + </div> + + <div class="alert alert-info"> + <h3><center>GitBlit Configuration</center></h3> + <p>GitBlit provides the following configuration items when using the filestore: + <h4>filestore.storageFolder</h4> + <p>Defines the path on the server where filestore objects are to be saved. This defaults to <code>${baseFolder}/lfs</code></p> + <h4>filestore.maxUploadSize</h4> + <p>Defines the maximum allowable size that can be uploaded to the filestore. Once a file is uploaded it will be unaffected by later changes in this property. This defaults to <code>-1</code> indicating no limits.</p> + </p> + </div> + +</div> +</div> +</div> +</div> +</wicket:extend> +</body> +</html> diff --git a/src/main/java/com/gitblit/wicket/pages/FilestoreUsage.java b/src/main/java/com/gitblit/wicket/pages/FilestoreUsage.java new file mode 100644 index 00000000..9bd8e55d --- /dev/null +++ b/src/main/java/com/gitblit/wicket/pages/FilestoreUsage.java @@ -0,0 +1,25 @@ +/* + * Copyright 2015 gitblit.com. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.gitblit.wicket.pages; + +public class FilestoreUsage extends RootSubPage { + + public FilestoreUsage() { + super(); + setupPage("", ""); + } + +} diff --git a/src/main/java/com/gitblit/wicket/pages/RootPage.java b/src/main/java/com/gitblit/wicket/pages/RootPage.java index 79a4fc67..93d44fc7 100644 --- a/src/main/java/com/gitblit/wicket/pages/RootPage.java +++ b/src/main/java/com/gitblit/wicket/pages/RootPage.java @@ -191,6 +191,7 @@ public abstract class RootPage extends BasePage { } navLinks.add(new PageNavLink("gb.repositories", RepositoriesPage.class, getRootPageParameters())); + navLinks.add(new PageNavLink("gb.filestore", FilestorePage.class, getRootPageParameters())); navLinks.add(new PageNavLink("gb.activity", ActivityPage.class, getRootPageParameters())); if (allowLucene) { navLinks.add(new PageNavLink("gb.search", LuceneSearchPage.class)); diff --git a/src/main/resources/gitblit.css b/src/main/resources/gitblit.css index c0329059..0cc8fd02 100644 --- a/src/main/resources/gitblit.css +++ b/src/main/resources/gitblit.css @@ -1729,6 +1729,12 @@ table.pretty tr.commit { }
}
+td.sha256 {
+ max-width: 20em;
+ overflow: hidden;
+ text-overflow: ellipsis;
+}
+
table.comments td {
padding: 4px;
line-height: 17px;
@@ -1922,7 +1928,7 @@ td.date { white-space: nowrap;
}
-span.sha1, span.sha1 a, span.sha1 a span, .commit_message, span.shortsha1, td.sha1 {
+span.sha1, span.sha1 a, span.sha1 a span, .commit_message, span.shortsha1, td.sha1, td.sha256 {
font-family: consolas, monospace;
font-size: 13px;
}
@@ -2340,3 +2346,11 @@ div.markdown table.text th, div.markdown table.text td { .priority-low {
color:#0072B2;
}
+
+.file-positive {
+ color:#009E73;
+}
+
+.file-negative {
+ color:#D51900;
+}
diff --git a/src/test/java/com/gitblit/tests/FilestoreManagerTest.java b/src/test/java/com/gitblit/tests/FilestoreManagerTest.java new file mode 100644 index 00000000..c76e9dd6 --- /dev/null +++ b/src/test/java/com/gitblit/tests/FilestoreManagerTest.java @@ -0,0 +1,547 @@ +package com.gitblit.tests; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.util.Date; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.commons.codec.digest.DigestUtils; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import com.gitblit.Constants.AccessPermission; +import com.gitblit.Constants.AccessRestrictionType; +import com.gitblit.Constants.AuthorizationControl; +import com.gitblit.Keys; +import com.gitblit.models.FilestoreModel.Status; +import com.gitblit.models.RepositoryModel; +import com.gitblit.models.UserModel; +import com.gitblit.utils.FileUtils; + + +/** + * Test of the filestore manager and confirming filesystem updated + * + * @author Paul Martin + * + */ +public class FilestoreManagerTest extends GitblitUnitTest { + + private static final AtomicBoolean started = new AtomicBoolean(false); + + private static final BlobInfo blob_zero = new BlobInfo(0); + private static final BlobInfo blob_512KB = new BlobInfo(512*FileUtils.KB); + private static final BlobInfo blob_6MB = new BlobInfo(6*FileUtils.MB); + + private static int download_limit_default = -1; + private static int download_limit_test = 5*FileUtils.MB; + + private static final String invalid_hash_empty = ""; + private static final String invalid_hash_major = "INVALID_HASH"; + private static final String invalid_hash_regex_attack = blob_512KB.hash.replace('a', '*'); + private static final String invalid_hash_one_long = blob_512KB.hash.concat("a"); + private static final String invalid_hash_one_short = blob_512KB.hash.substring(1); + + + + @BeforeClass + public static void startGitblit() throws Exception { + started.set(GitBlitSuite.startGitblit()); + } + + @AfterClass + public static void stopGitblit() throws Exception { + if (started.get()) { + GitBlitSuite.stopGitblit(); + } + } + + + + @Test + public void testAdminAccess() throws Exception { + + FileUtils.delete(filestore().getStorageFolder()); + filestore().clearFilestoreCache(); + + RepositoryModel r = new RepositoryModel("myrepo.git", null, null, new Date()); + ByteArrayOutputStream streamOut = new ByteArrayOutputStream(); + + UserModel u = new UserModel("admin"); + u.canAdmin = true; + + //No upload limit + settings().overrideSetting(Keys.filestore.maxUploadSize, download_limit_default); + + //Invalid hash tests + assertEquals(Status.Error_Invalid_Oid, filestore().downloadBlob(invalid_hash_major, u, r, streamOut)); + assertEquals(Status.Error_Invalid_Oid, filestore().downloadBlob(invalid_hash_regex_attack, u, r, streamOut)); + assertEquals(Status.Error_Invalid_Oid, filestore().downloadBlob(invalid_hash_one_long, u, r, streamOut)); + assertEquals(Status.Error_Invalid_Oid, filestore().downloadBlob(invalid_hash_one_short, u, r, streamOut)); + + // Download prior to upload + streamOut.reset(); + assertEquals(Status.Unavailable, filestore().downloadBlob(blob_512KB.hash, u, r, streamOut)); + streamOut.reset(); + assertEquals(Status.Unavailable, filestore().downloadBlob(blob_512KB.hash, u, r, streamOut)); + + //Bad input is rejected with no upload taking place + assertEquals(Status.Error_Invalid_Oid, filestore().uploadBlob(invalid_hash_empty, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob))); + assertEquals(Status.Error_Invalid_Oid, filestore().uploadBlob(invalid_hash_major, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob))); + assertEquals(Status.Error_Invalid_Oid, filestore().uploadBlob(invalid_hash_regex_attack, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob))); + assertEquals(Status.Error_Invalid_Oid, filestore().uploadBlob(invalid_hash_one_long, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob))); + assertEquals(Status.Error_Invalid_Oid, filestore().uploadBlob(invalid_hash_one_short, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob))); + + assertEquals(Status.Error_Invalid_Size, filestore().uploadBlob(blob_512KB.hash, -1, u, r, new ByteArrayInputStream(blob_512KB.blob))); + + assertEquals(Status.Error_Size_Mismatch, filestore().uploadBlob(blob_512KB.hash, 0, u, r, new ByteArrayInputStream(blob_512KB.blob))); + assertEquals(Status.Error_Size_Mismatch, filestore().uploadBlob(blob_512KB.hash, blob_512KB.length-1, u, r, new ByteArrayInputStream(blob_512KB.blob))); + assertEquals(Status.Error_Size_Mismatch, filestore().uploadBlob(blob_512KB.hash, blob_512KB.length+1, u, r, new ByteArrayInputStream(blob_512KB.blob))); + + assertEquals(Status.Error_Hash_Mismatch, filestore().uploadBlob(blob_zero.hash, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob))); + + //Confirm no upload with bad input + streamOut.reset(); + assertEquals(Status.Unavailable, filestore().downloadBlob(blob_512KB.hash, u, r, streamOut)); + + //Good input will accept the upload + assertEquals(Status.Available, filestore().uploadBlob(blob_512KB.hash, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob))); + streamOut.reset(); + assertEquals(Status.Available, filestore().downloadBlob(blob_512KB.hash, u, r, streamOut)); + assertArrayEquals(blob_512KB.blob, streamOut.toByteArray()); + + //Subsequent failed uploads do not affect file + assertEquals(Status.Error_Size_Mismatch, filestore().uploadBlob(blob_512KB.hash, blob_512KB.length-1, u, r, new ByteArrayInputStream(blob_512KB.blob))); + assertEquals(Status.Error_Size_Mismatch, filestore().uploadBlob(blob_512KB.hash, blob_512KB.length+1, u, r, new ByteArrayInputStream(blob_512KB.blob))); + + streamOut.reset(); + assertEquals(Status.Available, filestore().downloadBlob(blob_512KB.hash, u, r, streamOut)); + assertArrayEquals(blob_512KB.blob, streamOut.toByteArray()); + + //Zero length upload is valid + assertEquals(Status.Available, filestore().uploadBlob(blob_zero.hash, blob_zero.length, u, r, new ByteArrayInputStream(blob_zero.blob))); + + streamOut.reset(); + assertEquals(Status.Available, filestore().downloadBlob(blob_zero.hash, u, r, streamOut)); + assertArrayEquals(blob_zero.blob, streamOut.toByteArray()); + + + //Pre-informed upload identifies identical errors as immediate upload + assertEquals(Status.Upload_Pending, filestore().addObject(blob_6MB.hash, blob_6MB.length, u, r)); + streamOut.reset(); + assertEquals(Status.Unavailable, filestore().downloadBlob(blob_6MB.hash, u, r, streamOut)); + + assertEquals(Status.Error_Invalid_Oid, filestore().uploadBlob(invalid_hash_empty, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob))); + assertEquals(Status.Error_Invalid_Oid, filestore().uploadBlob(invalid_hash_major, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob))); + assertEquals(Status.Error_Invalid_Oid, filestore().uploadBlob(invalid_hash_regex_attack, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob))); + assertEquals(Status.Error_Invalid_Oid, filestore().uploadBlob(invalid_hash_one_long, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob))); + assertEquals(Status.Error_Invalid_Oid, filestore().uploadBlob(invalid_hash_one_short, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob))); + + assertEquals(Status.Error_Size_Mismatch, filestore().uploadBlob(blob_6MB.hash, 0, u, r, new ByteArrayInputStream(blob_6MB.blob))); + assertEquals(Status.Error_Size_Mismatch, filestore().uploadBlob(blob_6MB.hash, blob_6MB.length-1, u, r, new ByteArrayInputStream(blob_6MB.blob))); + assertEquals(Status.Error_Size_Mismatch, filestore().uploadBlob(blob_6MB.hash, blob_6MB.length+1, u, r, new ByteArrayInputStream(blob_6MB.blob))); + + streamOut.reset(); + assertEquals(Status.Unavailable, filestore().downloadBlob(blob_6MB.hash, u, r, streamOut)); + + //Good input will accept the upload + assertEquals(Status.Available, filestore().uploadBlob(blob_6MB.hash, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob))); + streamOut.reset(); + assertEquals(Status.Available, filestore().downloadBlob(blob_6MB.hash, u, r, streamOut)); + assertArrayEquals(blob_6MB.blob, streamOut.toByteArray()); + + //Confirm the relevant files exist + assertTrue("Admin did not save zero length file!", filestore().getStoragePath(blob_zero.hash).exists()); + assertTrue("Admin did not save 512KB file!", filestore().getStoragePath(blob_512KB.hash).exists()); + assertTrue("Admin did not save 6MB file!", filestore().getStoragePath(blob_6MB.hash).exists()); + + //Clear the files and cache to test upload limit property + FileUtils.delete(filestore().getStorageFolder()); + filestore().clearFilestoreCache(); + + settings().overrideSetting(Keys.filestore.maxUploadSize, download_limit_test); + + assertEquals(Status.Available, filestore().uploadBlob(blob_512KB.hash, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob))); + streamOut.reset(); + assertEquals(Status.Available, filestore().downloadBlob(blob_512KB.hash, u, r, streamOut)); + assertArrayEquals(blob_512KB.blob, streamOut.toByteArray()); + + assertEquals(Status.Error_Exceeds_Size_Limit, filestore().uploadBlob(blob_6MB.hash, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob))); + streamOut.reset(); + assertEquals(Status.Unavailable, filestore().downloadBlob(blob_6MB.hash, u, r, streamOut)); + + assertTrue("Admin did not save 512KB file!", filestore().getStoragePath(blob_512KB.hash).exists()); + assertFalse("Admin saved 6MB file despite (over filesize limit)!", filestore().getStoragePath(blob_6MB.hash).exists()); + + } + + @Test + public void testAuthenticatedAccess() throws Exception { + + FileUtils.delete(filestore().getStorageFolder()); + filestore().clearFilestoreCache(); + + RepositoryModel r = new RepositoryModel("myrepo.git", null, null, new Date()); + r.authorizationControl = AuthorizationControl.AUTHENTICATED; + r.accessRestriction = AccessRestrictionType.VIEW; + + ByteArrayOutputStream streamOut = new ByteArrayOutputStream(); + + UserModel u = new UserModel("test"); + + //No upload limit + settings().overrideSetting(Keys.filestore.maxUploadSize, download_limit_default); + + //Invalid hash tests + assertEquals(Status.Error_Invalid_Oid, filestore().downloadBlob(invalid_hash_major, u, r, streamOut)); + assertEquals(Status.Error_Invalid_Oid, filestore().downloadBlob(invalid_hash_regex_attack, u, r, streamOut)); + assertEquals(Status.Error_Invalid_Oid, filestore().downloadBlob(invalid_hash_one_long, u, r, streamOut)); + assertEquals(Status.Error_Invalid_Oid, filestore().downloadBlob(invalid_hash_one_short, u, r, streamOut)); + + // Download prior to upload + streamOut.reset(); + assertEquals(Status.Unavailable, filestore().downloadBlob(blob_512KB.hash, u, r, streamOut)); + streamOut.reset(); + assertEquals(Status.Unavailable, filestore().downloadBlob(blob_512KB.hash, u, r, streamOut)); + + //Bad input is rejected with no upload taking place + assertEquals(Status.Error_Invalid_Oid, filestore().uploadBlob(invalid_hash_empty, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob))); + assertEquals(Status.Error_Invalid_Oid, filestore().uploadBlob(invalid_hash_major, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob))); + assertEquals(Status.Error_Invalid_Oid, filestore().uploadBlob(invalid_hash_regex_attack, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob))); + assertEquals(Status.Error_Invalid_Oid, filestore().uploadBlob(invalid_hash_one_long, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob))); + assertEquals(Status.Error_Invalid_Oid, filestore().uploadBlob(invalid_hash_one_short, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob))); + + assertEquals(Status.Error_Invalid_Size, filestore().uploadBlob(blob_512KB.hash, -1, u, r, new ByteArrayInputStream(blob_512KB.blob))); + + assertEquals(Status.Error_Size_Mismatch, filestore().uploadBlob(blob_512KB.hash, 0, u, r, new ByteArrayInputStream(blob_512KB.blob))); + assertEquals(Status.Error_Size_Mismatch, filestore().uploadBlob(blob_512KB.hash, blob_512KB.length-1, u, r, new ByteArrayInputStream(blob_512KB.blob))); + assertEquals(Status.Error_Size_Mismatch, filestore().uploadBlob(blob_512KB.hash, blob_512KB.length+1, u, r, new ByteArrayInputStream(blob_512KB.blob))); + + assertEquals(Status.Error_Hash_Mismatch, filestore().uploadBlob(blob_zero.hash, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob))); + + //Confirm no upload with bad input + streamOut.reset(); + assertEquals(Status.Unavailable, filestore().downloadBlob(blob_512KB.hash, u, r, streamOut)); + + //Good input will accept the upload + assertEquals(Status.Available, filestore().uploadBlob(blob_512KB.hash, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob))); + streamOut.reset(); + assertEquals(Status.Available, filestore().downloadBlob(blob_512KB.hash, u, r, streamOut)); + assertArrayEquals(blob_512KB.blob, streamOut.toByteArray()); + + //Subsequent failed uploads do not affect file + assertEquals(Status.Error_Size_Mismatch, filestore().uploadBlob(blob_512KB.hash, blob_512KB.length-1, u, r, new ByteArrayInputStream(blob_512KB.blob))); + assertEquals(Status.Error_Size_Mismatch, filestore().uploadBlob(blob_512KB.hash, blob_512KB.length+1, u, r, new ByteArrayInputStream(blob_512KB.blob))); + + streamOut.reset(); + assertEquals(Status.Available, filestore().downloadBlob(blob_512KB.hash, u, r, streamOut)); + assertArrayEquals(blob_512KB.blob, streamOut.toByteArray()); + + //Zero length upload is valid + assertEquals(Status.Available, filestore().uploadBlob(blob_zero.hash, blob_zero.length, u, r, new ByteArrayInputStream(blob_zero.blob))); + + streamOut.reset(); + assertEquals(Status.Available, filestore().downloadBlob(blob_zero.hash, u, r, streamOut)); + assertArrayEquals(blob_zero.blob, streamOut.toByteArray()); + + + //Pre-informed upload identifies identical errors as immediate upload + assertEquals(Status.Upload_Pending, filestore().addObject(blob_6MB.hash, blob_6MB.length, u, r)); + streamOut.reset(); + assertEquals(Status.Unavailable, filestore().downloadBlob(blob_6MB.hash, u, r, streamOut)); + + assertEquals(Status.Error_Invalid_Oid, filestore().uploadBlob(invalid_hash_empty, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob))); + assertEquals(Status.Error_Invalid_Oid, filestore().uploadBlob(invalid_hash_major, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob))); + assertEquals(Status.Error_Invalid_Oid, filestore().uploadBlob(invalid_hash_regex_attack, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob))); + assertEquals(Status.Error_Invalid_Oid, filestore().uploadBlob(invalid_hash_one_long, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob))); + assertEquals(Status.Error_Invalid_Oid, filestore().uploadBlob(invalid_hash_one_short, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob))); + + assertEquals(Status.Error_Size_Mismatch, filestore().uploadBlob(blob_6MB.hash, 0, u, r, new ByteArrayInputStream(blob_6MB.blob))); + assertEquals(Status.Error_Size_Mismatch, filestore().uploadBlob(blob_6MB.hash, blob_6MB.length-1, u, r, new ByteArrayInputStream(blob_6MB.blob))); + assertEquals(Status.Error_Size_Mismatch, filestore().uploadBlob(blob_6MB.hash, blob_6MB.length+1, u, r, new ByteArrayInputStream(blob_6MB.blob))); + + streamOut.reset(); + assertEquals(Status.Unavailable, filestore().downloadBlob(blob_6MB.hash, u, r, streamOut)); + + //Good input will accept the upload + assertEquals(Status.Available, filestore().uploadBlob(blob_6MB.hash, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob))); + streamOut.reset(); + assertEquals(Status.Available, filestore().downloadBlob(blob_6MB.hash, u, r, streamOut)); + assertArrayEquals(blob_6MB.blob, streamOut.toByteArray()); + + //Confirm the relevant files exist + assertTrue("Authenticated user did not save zero length file!", filestore().getStoragePath(blob_zero.hash).exists()); + assertTrue("Authenticated user did not save 512KB file!", filestore().getStoragePath(blob_512KB.hash).exists()); + assertTrue("Authenticated user did not save 6MB file!", filestore().getStoragePath(blob_6MB.hash).exists()); + + //Clear the files and cache to test upload limit property + FileUtils.delete(filestore().getStorageFolder()); + filestore().clearFilestoreCache(); + + settings().overrideSetting(Keys.filestore.maxUploadSize, download_limit_test); + + assertEquals(Status.Available, filestore().uploadBlob(blob_512KB.hash, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob))); + streamOut.reset(); + assertEquals(Status.Available, filestore().downloadBlob(blob_512KB.hash, u, r, streamOut)); + assertArrayEquals(blob_512KB.blob, streamOut.toByteArray()); + + assertEquals(Status.Error_Exceeds_Size_Limit, filestore().uploadBlob(blob_6MB.hash, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob))); + streamOut.reset(); + assertEquals(Status.Unavailable, filestore().downloadBlob(blob_6MB.hash, u, r, streamOut)); + + assertTrue("Authenticated user did not save 512KB file!", filestore().getStoragePath(blob_512KB.hash).exists()); + assertFalse("Authenticated user saved 6MB file (over filesize limit)!", filestore().getStoragePath(blob_6MB.hash).exists()); + + } + + @Test + public void testAnonymousAccess() throws Exception { + + FileUtils.delete(filestore().getStorageFolder()); + filestore().clearFilestoreCache(); + + RepositoryModel r = new RepositoryModel("myrepo.git", null, null, new Date()); + r.authorizationControl = AuthorizationControl.NAMED; + r.accessRestriction = AccessRestrictionType.CLONE; + + ByteArrayOutputStream streamOut = new ByteArrayOutputStream(); + + UserModel u = UserModel.ANONYMOUS; + + //No upload limit + settings().overrideSetting(Keys.filestore.maxUploadSize, download_limit_default); + + //Invalid hash tests + assertEquals(Status.Error_Invalid_Oid, filestore().downloadBlob(invalid_hash_major, u, r, streamOut)); + assertEquals(Status.Error_Invalid_Oid, filestore().downloadBlob(invalid_hash_regex_attack, u, r, streamOut)); + assertEquals(Status.Error_Invalid_Oid, filestore().downloadBlob(invalid_hash_one_long, u, r, streamOut)); + assertEquals(Status.Error_Invalid_Oid, filestore().downloadBlob(invalid_hash_one_short, u, r, streamOut)); + + // Download prior to upload + streamOut.reset(); + assertEquals(Status.Unavailable, filestore().downloadBlob(blob_512KB.hash, u, r, streamOut)); + streamOut.reset(); + assertEquals(Status.Unavailable, filestore().downloadBlob(blob_512KB.hash, u, r, streamOut)); + + //Bad input is rejected with no upload taking place + assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(invalid_hash_empty, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob))); + assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(invalid_hash_major, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob))); + assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(invalid_hash_regex_attack, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob))); + assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(invalid_hash_one_long, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob))); + assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(invalid_hash_one_short, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob))); + + assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(blob_512KB.hash, -1, u, r, new ByteArrayInputStream(blob_512KB.blob))); + + assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(blob_512KB.hash, 0, u, r, new ByteArrayInputStream(blob_512KB.blob))); + assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(blob_512KB.hash, blob_512KB.length-1, u, r, new ByteArrayInputStream(blob_512KB.blob))); + assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(blob_512KB.hash, blob_512KB.length+1, u, r, new ByteArrayInputStream(blob_512KB.blob))); + + assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(blob_zero.hash, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob))); + + //Confirm no upload with bad input + streamOut.reset(); + assertEquals(Status.Unavailable, filestore().downloadBlob(blob_512KB.hash, u, r, streamOut)); + + //Good input will accept the upload + assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(blob_512KB.hash, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob))); + streamOut.reset(); + assertEquals(Status.Unavailable, filestore().downloadBlob(blob_512KB.hash, u, r, streamOut)); + + //Subsequent failed uploads do not affect file + assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(blob_512KB.hash, blob_512KB.length-1, u, r, new ByteArrayInputStream(blob_512KB.blob))); + assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(blob_512KB.hash, blob_512KB.length+1, u, r, new ByteArrayInputStream(blob_512KB.blob))); + + streamOut.reset(); + assertEquals(Status.Unavailable, filestore().downloadBlob(blob_512KB.hash, u, r, streamOut)); + + //Zero length upload is valid + assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(blob_zero.hash, blob_zero.length, u, r, new ByteArrayInputStream(blob_zero.blob))); + + streamOut.reset(); + assertEquals(Status.Unavailable, filestore().downloadBlob(blob_zero.hash, u, r, streamOut)); + + + //Pre-informed upload identifies identical errors as immediate upload + assertEquals(Status.AuthenticationRequired, filestore().addObject(blob_6MB.hash, blob_6MB.length, u, r)); + streamOut.reset(); + assertEquals(Status.Unavailable, filestore().downloadBlob(blob_6MB.hash, u, r, streamOut)); + + assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(invalid_hash_empty, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob))); + assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(invalid_hash_major, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob))); + assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(invalid_hash_regex_attack, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob))); + assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(invalid_hash_one_long, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob))); + assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(invalid_hash_one_short, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob))); + + assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(blob_6MB.hash, -1, u, r, new ByteArrayInputStream(blob_6MB.blob))); + + assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(blob_6MB.hash, 0, u, r, new ByteArrayInputStream(blob_6MB.blob))); + assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(blob_6MB.hash, blob_6MB.length-1, u, r, new ByteArrayInputStream(blob_6MB.blob))); + assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(blob_6MB.hash, blob_6MB.length+1, u, r, new ByteArrayInputStream(blob_6MB.blob))); + + streamOut.reset(); + assertEquals(Status.Unavailable, filestore().downloadBlob(blob_6MB.hash, u, r, streamOut)); + + //Good input will accept the upload + assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(blob_6MB.hash, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob))); + streamOut.reset(); + assertEquals(Status.Unavailable, filestore().downloadBlob(blob_6MB.hash, u, r, streamOut)); + + //Confirm the relevant files do not exist + assertFalse("Anonymous user saved zero length file!", filestore().getStoragePath(blob_zero.hash).exists()); + assertFalse("Anonymous user 512KB file!", filestore().getStoragePath(blob_512KB.hash).exists()); + assertFalse("Anonymous user 6MB file!", filestore().getStoragePath(blob_6MB.hash).exists()); + + //Clear the files and cache to test upload limit property + FileUtils.delete(filestore().getStorageFolder()); + filestore().clearFilestoreCache(); + + settings().overrideSetting(Keys.filestore.maxUploadSize, download_limit_test); + + assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(blob_512KB.hash, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob))); + streamOut.reset(); + assertEquals(Status.Unavailable, filestore().downloadBlob(blob_512KB.hash, u, r, streamOut)); + + assertEquals(Status.AuthenticationRequired, filestore().uploadBlob(blob_6MB.hash, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob))); + streamOut.reset(); + assertEquals(Status.Unavailable, filestore().downloadBlob(blob_6MB.hash, u, r, streamOut)); + + assertFalse("Anonymous user saved 512KB file!", filestore().getStoragePath(blob_512KB.hash).exists()); + assertFalse("Anonymous user saved 6MB file (over filesize limit)!", filestore().getStoragePath(blob_6MB.hash).exists()); + + } + + @Test + public void testUnauthorizedAccess() throws Exception { + + FileUtils.delete(filestore().getStorageFolder()); + filestore().clearFilestoreCache(); + + RepositoryModel r = new RepositoryModel("myrepo.git", null, null, new Date()); + r.authorizationControl = AuthorizationControl.NAMED; + r.accessRestriction = AccessRestrictionType.VIEW; + + ByteArrayOutputStream streamOut = new ByteArrayOutputStream(); + + UserModel u = new UserModel("test"); + u.setRepositoryPermission(r.name, AccessPermission.CLONE); + + //No upload limit + settings().overrideSetting(Keys.filestore.maxUploadSize, download_limit_default); + + //Invalid hash tests + assertEquals(Status.Error_Invalid_Oid, filestore().downloadBlob(invalid_hash_major, u, r, streamOut)); + assertEquals(Status.Error_Invalid_Oid, filestore().downloadBlob(invalid_hash_regex_attack, u, r, streamOut)); + assertEquals(Status.Error_Invalid_Oid, filestore().downloadBlob(invalid_hash_one_long, u, r, streamOut)); + assertEquals(Status.Error_Invalid_Oid, filestore().downloadBlob(invalid_hash_one_short, u, r, streamOut)); + + // Download prior to upload + streamOut.reset(); + assertEquals(Status.Unavailable, filestore().downloadBlob(blob_512KB.hash, u, r, streamOut)); + streamOut.reset(); + assertEquals(Status.Unavailable, filestore().downloadBlob(blob_512KB.hash, u, r, streamOut)); + + //Bad input is rejected with no upload taking place + assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(invalid_hash_major, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob))); + assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(invalid_hash_regex_attack, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob))); + assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(invalid_hash_one_long, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob))); + assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(invalid_hash_empty, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob))); + assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(invalid_hash_one_short, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob))); + + assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(blob_512KB.hash, -1, u, r, new ByteArrayInputStream(blob_512KB.blob))); + + assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(blob_512KB.hash, 0, u, r, new ByteArrayInputStream(blob_512KB.blob))); + assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(blob_512KB.hash, blob_512KB.length-1, u, r, new ByteArrayInputStream(blob_512KB.blob))); + assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(blob_512KB.hash, blob_512KB.length+1, u, r, new ByteArrayInputStream(blob_512KB.blob))); + + assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(blob_zero.hash, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob))); + + //Confirm no upload with bad input + streamOut.reset(); + assertEquals(Status.Unavailable, filestore().downloadBlob(blob_512KB.hash, u, r, streamOut)); + + //Good input will accept the upload + assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(blob_512KB.hash, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob))); + streamOut.reset(); + assertEquals(Status.Unavailable, filestore().downloadBlob(blob_512KB.hash, u, r, streamOut)); + + //Subsequent failed uploads do not affect file + assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(blob_512KB.hash, blob_512KB.length-1, u, r, new ByteArrayInputStream(blob_512KB.blob))); + assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(blob_512KB.hash, blob_512KB.length+1, u, r, new ByteArrayInputStream(blob_512KB.blob))); + + streamOut.reset(); + assertEquals(Status.Unavailable, filestore().downloadBlob(blob_512KB.hash, u, r, streamOut)); + + //Zero length upload is valid + assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(blob_zero.hash, blob_zero.length, u, r, new ByteArrayInputStream(blob_zero.blob))); + + streamOut.reset(); + assertEquals(Status.Unavailable, filestore().downloadBlob(blob_zero.hash, u, r, streamOut)); + + + //Pre-informed upload identifies identical errors as immediate upload + assertEquals(Status.Error_Unauthorized, filestore().addObject(blob_6MB.hash, blob_6MB.length, u, r)); + streamOut.reset(); + assertEquals(Status.Unavailable, filestore().downloadBlob(blob_6MB.hash, u, r, streamOut)); + + assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(invalid_hash_empty, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob))); + assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(invalid_hash_major, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob))); + assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(invalid_hash_regex_attack, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob))); + assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(invalid_hash_one_long, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob))); + assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(invalid_hash_one_short, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob))); + + assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(blob_6MB.hash, -1, u, r, new ByteArrayInputStream(blob_6MB.blob))); + + assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(blob_6MB.hash, 0, u, r, new ByteArrayInputStream(blob_6MB.blob))); + assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(blob_6MB.hash, blob_6MB.length-1, u, r, new ByteArrayInputStream(blob_6MB.blob))); + assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(blob_6MB.hash, blob_6MB.length+1, u, r, new ByteArrayInputStream(blob_6MB.blob))); + + streamOut.reset(); + assertEquals(Status.Unavailable, filestore().downloadBlob(blob_6MB.hash, u, r, streamOut)); + + //Good input will accept the upload + assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(blob_6MB.hash, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob))); + streamOut.reset(); + assertEquals(Status.Unavailable, filestore().downloadBlob(blob_6MB.hash, u, r, streamOut)); + + //Confirm the relevant files exist + assertFalse("Unauthorized user saved zero length file!", filestore().getStoragePath(blob_zero.hash).exists()); + assertFalse("Unauthorized user saved 512KB file!", filestore().getStoragePath(blob_512KB.hash).exists()); + assertFalse("Unauthorized user saved 6MB file!", filestore().getStoragePath(blob_6MB.hash).exists()); + + //Clear the files and cache to test upload limit property + FileUtils.delete(filestore().getStorageFolder()); + filestore().clearFilestoreCache(); + + settings().overrideSetting(Keys.filestore.maxUploadSize, download_limit_test); + + assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(blob_512KB.hash, blob_512KB.length, u, r, new ByteArrayInputStream(blob_512KB.blob))); + streamOut.reset(); + assertEquals(Status.Unavailable, filestore().downloadBlob(blob_512KB.hash, u, r, streamOut)); + + assertEquals(Status.Error_Unauthorized, filestore().uploadBlob(blob_6MB.hash, blob_6MB.length, u, r, new ByteArrayInputStream(blob_6MB.blob))); + streamOut.reset(); + assertEquals(Status.Unavailable, filestore().downloadBlob(blob_6MB.hash, u, r, streamOut)); + + assertFalse("Unauthorized user saved 512KB file!", filestore().getStoragePath(blob_512KB.hash).exists()); + assertFalse("Unauthorized user saved 6MB file (over filesize limit)!", filestore().getStoragePath(blob_6MB.hash).exists()); + + } + +} + +/* + * Test helper structure to create blobs of a given size + */ +final class BlobInfo { + public byte[] blob; + public String hash; + public int length; + + public BlobInfo(int nBytes) { + blob = new byte[nBytes]; + new java.util.Random().nextBytes(blob); + hash = DigestUtils.sha256Hex(blob); + length = nBytes; + } +}
\ No newline at end of file diff --git a/src/test/java/com/gitblit/tests/FilestoreServletTest.java b/src/test/java/com/gitblit/tests/FilestoreServletTest.java new file mode 100644 index 00000000..4e4b056a --- /dev/null +++ b/src/test/java/com/gitblit/tests/FilestoreServletTest.java @@ -0,0 +1,355 @@ +package com.gitblit.tests; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.commons.io.IOUtils; +import org.apache.http.HttpEntity; +import org.apache.http.HttpHeaders; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpPut; +import org.apache.http.entity.ByteArrayEntity; +import org.apache.http.impl.client.HttpClientBuilder; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import com.gitblit.Keys; +import com.gitblit.manager.FilestoreManager; +import com.gitblit.models.RepositoryModel; +import com.gitblit.models.UserModel; +import com.gitblit.models.FilestoreModel.Status; +import com.gitblit.servlet.FilestoreServlet; +import com.gitblit.utils.FileUtils; + +public class FilestoreServletTest extends GitblitUnitTest { + + private static final AtomicBoolean started = new AtomicBoolean(false); + + private static final String SHA256_EG = "9a712c5d4037503a2d5ee1d07ad191eb99d051e84cbb020c171a5ae19bbe3cbd"; + + private static final String repoName = "helloworld.git"; + + private static final String repoLfs = "/r/" + repoName + "/info/lfs/objects/"; + + @BeforeClass + public static void startGitblit() throws Exception { + started.set(GitBlitSuite.startGitblit()); + } + + @AfterClass + public static void stopGitblit() throws Exception { + if (started.get()) { + GitBlitSuite.stopGitblit(); + } + } + + + @Test + public void testRegexGroups() throws Exception { + + Pattern p = Pattern.compile(FilestoreServlet.REGEX_PATH); + + String basicUrl = "https://localhost:8080/r/test.git/info/lfs/objects/"; + String batchUrl = basicUrl + "batch"; + String oidUrl = basicUrl + SHA256_EG; + + Matcher m = p.matcher(batchUrl); + assertTrue(m.find()); + assertEquals("https://localhost:8080", m.group(FilestoreServlet.REGEX_GROUP_BASE_URI)); + assertEquals("r", m.group(FilestoreServlet.REGEX_GROUP_PREFIX)); + assertEquals("test.git", m.group(FilestoreServlet.REGEX_GROUP_REPOSITORY)); + assertEquals("batch", m.group(FilestoreServlet.REGEX_GROUP_ENDPOINT)); + + m = p.matcher(oidUrl); + assertTrue(m.find()); + assertEquals("https://localhost:8080", m.group(FilestoreServlet.REGEX_GROUP_BASE_URI)); + assertEquals("r", m.group(FilestoreServlet.REGEX_GROUP_PREFIX)); + assertEquals("test.git", m.group(FilestoreServlet.REGEX_GROUP_REPOSITORY)); + assertEquals(SHA256_EG, m.group(FilestoreServlet.REGEX_GROUP_ENDPOINT)); + } + + @Test + public void testRegexGroupsNestedRepo() throws Exception { + + Pattern p = Pattern.compile(FilestoreServlet.REGEX_PATH); + + String basicUrl = "https://localhost:8080/r/nested/test.git/info/lfs/objects/"; + String batchUrl = basicUrl + "batch"; + String oidUrl = basicUrl + SHA256_EG; + + Matcher m = p.matcher(batchUrl); + assertTrue(m.find()); + assertEquals("https://localhost:8080", m.group(FilestoreServlet.REGEX_GROUP_BASE_URI)); + assertEquals("r", m.group(FilestoreServlet.REGEX_GROUP_PREFIX)); + assertEquals("nested/test.git", m.group(FilestoreServlet.REGEX_GROUP_REPOSITORY)); + assertEquals("batch", m.group(FilestoreServlet.REGEX_GROUP_ENDPOINT)); + + m = p.matcher(oidUrl); + assertTrue(m.find()); + assertEquals("https://localhost:8080", m.group(FilestoreServlet.REGEX_GROUP_BASE_URI)); + assertEquals("r", m.group(FilestoreServlet.REGEX_GROUP_PREFIX)); + assertEquals("nested/test.git", m.group(FilestoreServlet.REGEX_GROUP_REPOSITORY)); + assertEquals(SHA256_EG, m.group(FilestoreServlet.REGEX_GROUP_ENDPOINT)); + } + + @Test + public void testDownload() throws Exception { + + FileUtils.delete(filestore().getStorageFolder()); + filestore().clearFilestoreCache(); + + RepositoryModel r = gitblit().getRepositoryModel(repoName); + + UserModel u = new UserModel("admin"); + u.canAdmin = true; + + //No upload limit + settings().overrideSetting(Keys.filestore.maxUploadSize, FilestoreManager.UNDEFINED_SIZE); + + final BlobInfo blob = new BlobInfo(512*FileUtils.KB); + + //Emulate a pre-existing Git-LFS repository by using using internal pre-tested methods + assertEquals(Status.Available, filestore().uploadBlob(blob.hash, blob.length, u, r, new ByteArrayInputStream(blob.blob))); + + final String downloadURL = GitBlitSuite.url + repoLfs + blob.hash; + + HttpClient client = HttpClientBuilder.create().build(); + HttpGet request = new HttpGet(downloadURL); + + // add request header + request.addHeader(HttpHeaders.ACCEPT, FilestoreServlet.GIT_LFS_META_MIME); + HttpResponse response = client.execute(request); + + assertEquals(200, response.getStatusLine().getStatusCode()); + + String content = IOUtils.toString(response.getEntity().getContent(), "UTF-8"); + + String expectedContent = String.format("{%s:%s,%s:%d,%s:{%s:{%s:%s}}}", + "\"oid\"", "\"" + blob.hash + "\"", + "\"size\"", blob.length, + "\"actions\"", + "\"download\"", + "\"href\"", "\"" + downloadURL + "\""); + + assertEquals(expectedContent, content); + + + //Now try the binary download + request.removeHeaders(HttpHeaders.ACCEPT); + response = client.execute(request); + + assertEquals(200, response.getStatusLine().getStatusCode()); + + byte[] dlData = IOUtils.toByteArray(response.getEntity().getContent()); + + assertArrayEquals(blob.blob, dlData); + + } + + @Test + public void testDownloadMultiple() throws Exception { + + FileUtils.delete(filestore().getStorageFolder()); + filestore().clearFilestoreCache(); + + RepositoryModel r = gitblit().getRepositoryModel(repoName); + + UserModel u = new UserModel("admin"); + u.canAdmin = true; + + //No upload limit + settings().overrideSetting(Keys.filestore.maxUploadSize, FilestoreManager.UNDEFINED_SIZE); + + final BlobInfo blob = new BlobInfo(512*FileUtils.KB); + + //Emulate a pre-existing Git-LFS repository by using using internal pre-tested methods + assertEquals(Status.Available, filestore().uploadBlob(blob.hash, blob.length, u, r, new ByteArrayInputStream(blob.blob))); + + final String batchURL = GitBlitSuite.url + repoLfs + "batch"; + final String downloadURL = GitBlitSuite.url + repoLfs + blob.hash; + + HttpClient client = HttpClientBuilder.create().build(); + HttpPost request = new HttpPost(batchURL); + + // add request header + request.addHeader(HttpHeaders.ACCEPT, FilestoreServlet.GIT_LFS_META_MIME); + request.addHeader(HttpHeaders.CONTENT_ENCODING, FilestoreServlet.GIT_LFS_META_MIME); + + String content = String.format("{%s:%s,%s:[{%s:%s,%s:%d},{%s:%s,%s:%d}]}", + "\"operation\"", "\"download\"", + "\"objects\"", + "\"oid\"", "\"" + blob.hash + "\"", + "\"size\"", blob.length, + "\"oid\"", "\"" + SHA256_EG + "\"", + "\"size\"", 0); + + HttpEntity entity = new ByteArrayEntity(content.getBytes("UTF-8")); + request.setEntity(entity); + + HttpResponse response = client.execute(request); + + String responseMessage = IOUtils.toString(response.getEntity().getContent(), "UTF-8"); + assertEquals(200, response.getStatusLine().getStatusCode()); + + String expectedContent = String.format("{%s:[{%s:%s,%s:%d,%s:{%s:{%s:%s}}},{%s:%s,%s:%d,%s:{%s:%s,%s:%d}}]}", + "\"objects\"", + "\"oid\"", "\"" + blob.hash + "\"", + "\"size\"", blob.length, + "\"actions\"", + "\"download\"", + "\"href\"", "\"" + downloadURL + "\"", + "\"oid\"", "\"" + SHA256_EG + "\"", + "\"size\"", 0, + "\"error\"", + "\"message\"", "\"Object not available\"", + "\"code\"", 404 + ); + + assertEquals(expectedContent, responseMessage); + } + + @Test + public void testDownloadUnavailable() throws Exception { + + FileUtils.delete(filestore().getStorageFolder()); + filestore().clearFilestoreCache(); + + //No upload limit + settings().overrideSetting(Keys.filestore.maxUploadSize, FilestoreManager.UNDEFINED_SIZE); + + final BlobInfo blob = new BlobInfo(512*FileUtils.KB); + + final String downloadURL = GitBlitSuite.url + repoLfs + blob.hash; + + HttpClient client = HttpClientBuilder.create().build(); + HttpGet request = new HttpGet(downloadURL); + + // add request header + request.addHeader(HttpHeaders.ACCEPT, FilestoreServlet.GIT_LFS_META_MIME); + HttpResponse response = client.execute(request); + + assertEquals(404, response.getStatusLine().getStatusCode()); + + String content = IOUtils.toString(response.getEntity().getContent(), "UTF-8"); + + String expectedError = String.format("{%s:%s,%s:%d}", + "\"message\"", "\"Object not available\"", + "\"code\"", 404); + + assertEquals(expectedError, content); + } + + @Test + public void testUpload() throws Exception { + + FileUtils.delete(filestore().getStorageFolder()); + filestore().clearFilestoreCache(); + + RepositoryModel r = gitblit().getRepositoryModel(repoName); + + UserModel u = new UserModel("admin"); + u.canAdmin = true; + + //No upload limit + settings().overrideSetting(Keys.filestore.maxUploadSize, FilestoreManager.UNDEFINED_SIZE); + + final BlobInfo blob = new BlobInfo(512*FileUtils.KB); + + final String expectedUploadURL = GitBlitSuite.url + repoLfs + blob.hash; + final String initialUploadURL = GitBlitSuite.url + repoLfs + "batch"; + + HttpClient client = HttpClientBuilder.create().build(); + HttpPost request = new HttpPost(initialUploadURL); + + // add request header + request.addHeader(HttpHeaders.ACCEPT, FilestoreServlet.GIT_LFS_META_MIME); + request.addHeader(HttpHeaders.CONTENT_ENCODING, FilestoreServlet.GIT_LFS_META_MIME); + + String content = String.format("{%s:%s,%s:[{%s:%s,%s:%d}]}", + "\"operation\"", "\"upload\"", + "\"objects\"", + "\"oid\"", "\"" + blob.hash + "\"", + "\"size\"", blob.length); + + HttpEntity entity = new ByteArrayEntity(content.getBytes("UTF-8")); + request.setEntity(entity); + + HttpResponse response = client.execute(request); + String responseMessage = IOUtils.toString(response.getEntity().getContent(), "UTF-8"); + assertEquals(200, response.getStatusLine().getStatusCode()); + + String expectedContent = String.format("{%s:[{%s:%s,%s:%d,%s:{%s:{%s:%s}}}]}", + "\"objects\"", + "\"oid\"", "\"" + blob.hash + "\"", + "\"size\"", blob.length, + "\"actions\"", + "\"upload\"", + "\"href\"", "\"" + expectedUploadURL + "\""); + + assertEquals(expectedContent, responseMessage); + + + //Now try to upload the binary download + HttpPut putRequest = new HttpPut(expectedUploadURL); + putRequest.setEntity(new ByteArrayEntity(blob.blob)); + response = client.execute(putRequest); + + responseMessage = IOUtils.toString(response.getEntity().getContent(), "UTF-8"); + + assertEquals(200, response.getStatusLine().getStatusCode()); + + //Confirm behind the scenes that it is available + ByteArrayOutputStream savedBlob = new ByteArrayOutputStream(); + assertEquals(Status.Available, filestore().downloadBlob(blob.hash, u, r, savedBlob)); + assertArrayEquals(blob.blob, savedBlob.toByteArray()); + } + + @Test + public void testMalformedUpload() throws Exception { + + FileUtils.delete(filestore().getStorageFolder()); + filestore().clearFilestoreCache(); + + //No upload limit + settings().overrideSetting(Keys.filestore.maxUploadSize, FilestoreManager.UNDEFINED_SIZE); + + final BlobInfo blob = new BlobInfo(512*FileUtils.KB); + + final String initialUploadURL = GitBlitSuite.url + repoLfs + "batch"; + + HttpClient client = HttpClientBuilder.create().build(); + HttpPost request = new HttpPost(initialUploadURL); + + // add request header + request.addHeader(HttpHeaders.ACCEPT, FilestoreServlet.GIT_LFS_META_MIME); + request.addHeader(HttpHeaders.CONTENT_ENCODING, FilestoreServlet.GIT_LFS_META_MIME); + + //Malformed JSON, comma instead of colon and unquoted strings + String content = String.format("{%s:%s,%s:[{%s:%s,%s,%d}]}", + "operation", "upload", + "objects", + "oid", blob.hash, + "size", blob.length); + + HttpEntity entity = new ByteArrayEntity(content.getBytes("UTF-8")); + request.setEntity(entity); + + HttpResponse response = client.execute(request); + String responseMessage = IOUtils.toString(response.getEntity().getContent(), "UTF-8"); + assertEquals(400, response.getStatusLine().getStatusCode()); + + String expectedError = String.format("{%s:%s,%s:%d}", + "\"message\"", "\"Malformed Git-LFS request\"", + "\"code\"", 400); + + assertEquals(expectedError, responseMessage); + } + +} diff --git a/src/test/java/com/gitblit/tests/GitBlitSuite.java b/src/test/java/com/gitblit/tests/GitBlitSuite.java index af20a487..b01c82c4 100644 --- a/src/test/java/com/gitblit/tests/GitBlitSuite.java +++ b/src/test/java/com/gitblit/tests/GitBlitSuite.java @@ -66,7 +66,7 @@ import com.gitblit.utils.JGitUtils; ModelUtilsTest.class, JnaUtilsTest.class, LdapSyncServiceTest.class, FileTicketServiceTest.class,
BranchTicketServiceTest.class, RedisTicketServiceTest.class, AuthenticationManagerTest.class,
SshKeysDispatcherTest.class, UITicketTest.class, PathUtilsTest.class, SshKerberosAuthenticationTest.class,
- GravatarTest.class })
+ GravatarTest.class, FilestoreManagerTest.class, FilestoreServletTest.class })
public class GitBlitSuite {
public static final File BASEFOLDER = new File("data");
diff --git a/src/test/java/com/gitblit/tests/GitblitUnitTest.java b/src/test/java/com/gitblit/tests/GitblitUnitTest.java index 9dceaaf4..58bc60e4 100644 --- a/src/test/java/com/gitblit/tests/GitblitUnitTest.java +++ b/src/test/java/com/gitblit/tests/GitblitUnitTest.java @@ -18,6 +18,7 @@ package com.gitblit.tests; import com.gitblit.IStoredSettings; import com.gitblit.manager.IAuthenticationManager; import com.gitblit.manager.IFederationManager; +import com.gitblit.manager.IFilestoreManager; import com.gitblit.manager.IGitblit; import com.gitblit.manager.INotificationManager; import com.gitblit.manager.IProjectManager; @@ -64,4 +65,8 @@ public class GitblitUnitTest extends org.junit.Assert { public static IGitblit gitblit() { return GitblitContext.getManager(IGitblit.class); } + + public static IFilestoreManager filestore() { + return GitblitContext.getManager(IFilestoreManager.class); + } } |