From 697905cbf3b6fc1d8f22322a4f403b4abe2f0c0a Mon Sep 17 00:00:00 2001 From: Paul Martin Date: Mon, 7 Dec 2015 23:09:26 +0000 Subject: [PATCH] fix for #967 filestore menu for all users + Filestore listing filtered by user view permissions + Configuration help for filestore relocated to website files + Added migration example --- HOME.md | 2 + build.xml | 1 + .../com/gitblit/manager/FilestoreManager.java | 31 +++++++++- .../com/gitblit/manager/GitblitManager.java | 4 +- .../gitblit/manager/IFilestoreManager.java | 2 +- .../com/gitblit/models/FilestoreModel.java | 9 +++ .../gitblit/wicket/pages/FilestorePage.java | 14 ++++- .../gitblit/wicket/pages/FilestoreUsage.html | 34 +++-------- .../com/gitblit/wicket/pages/RootPage.java | 6 +- src/site/setup_filestore.mkd | 61 +++++++++++++++++++ 10 files changed, 126 insertions(+), 38 deletions(-) create mode 100644 src/site/setup_filestore.mkd diff --git a/HOME.md b/HOME.md index 7c5449d3..89527ee3 100644 --- a/HOME.md +++ b/HOME.md @@ -33,6 +33,8 @@ This documentation is the source content from which the [Gitblit website](http:/ [[src/site/setup_viewer.mkd]] [[src/site/administration.mkd]] [[src/site/setup_scaling.mkd]] +[[src/site/setup_filestore.mkd]] + ### Gitblit Tickets diff --git a/build.xml b/build.xml index c7dc499c..7205bf84 100644 --- a/build.xml +++ b/build.xml @@ -514,6 +514,7 @@ + diff --git a/src/main/java/com/gitblit/manager/FilestoreManager.java b/src/main/java/com/gitblit/manager/FilestoreManager.java index fe65e216..11108557 100644 --- a/src/main/java/com/gitblit/manager/FilestoreManager.java +++ b/src/main/java/com/gitblit/manager/FilestoreManager.java @@ -77,6 +77,8 @@ public class FilestoreManager implements IFilestoreManager { private final Logger logger = LoggerFactory.getLogger(getClass()); private final IRuntimeManager runtimeManager; + + private final IRepositoryManager repositoryManager; private final IStoredSettings settings; @@ -93,8 +95,10 @@ public class FilestoreManager implements IFilestoreManager { @Inject FilestoreManager( - IRuntimeManager runtimeManager) { + IRuntimeManager runtimeManager, + IRepositoryManager repositoryManager) { this.runtimeManager = runtimeManager; + this.repositoryManager = repositoryManager; this.settings = runtimeManager.getSettings(); } @@ -324,8 +328,29 @@ public class FilestoreManager implements IFilestoreManager { } @Override - public List getAllObjects() { - return new ArrayList(fileCache.values()); + public List getAllObjects(UserModel user) { + + final List viewableRepositories = repositoryManager.getRepositoryModels(user); + List viewableRepositoryNames = new ArrayList(viewableRepositories.size()); + + for (RepositoryModel repository : viewableRepositories) { + viewableRepositoryNames.add(repository.name); + } + + if (viewableRepositoryNames.size() == 0) { + return null; + } + + final Collection allFiles = fileCache.values(); + List userViewableFiles = new ArrayList(allFiles.size()); + + for (FilestoreModel file : allFiles) { + if (file.isInRepositoryList(viewableRepositoryNames)) { + userViewableFiles.add(file); + } + } + + return userViewableFiles; } @Override diff --git a/src/main/java/com/gitblit/manager/GitblitManager.java b/src/main/java/com/gitblit/manager/GitblitManager.java index 4a385fc1..85d5c19f 100644 --- a/src/main/java/com/gitblit/manager/GitblitManager.java +++ b/src/main/java/com/gitblit/manager/GitblitManager.java @@ -1274,8 +1274,8 @@ public class GitblitManager implements IGitblit { } @Override - public List getAllObjects() { - return filestoreManager.getAllObjects(); + public List getAllObjects(UserModel user) { + return filestoreManager.getAllObjects(user); } @Override diff --git a/src/main/java/com/gitblit/manager/IFilestoreManager.java b/src/main/java/com/gitblit/manager/IFilestoreManager.java index 0720650c..454331a3 100644 --- a/src/main/java/com/gitblit/manager/IFilestoreManager.java +++ b/src/main/java/com/gitblit/manager/IFilestoreManager.java @@ -37,7 +37,7 @@ public interface IFilestoreManager extends IManager { FilestoreModel.Status downloadBlob(String oid, UserModel user, RepositoryModel repo, OutputStream streamOut ); - List getAllObjects(); + List getAllObjects(UserModel user); File getStorageFolder(); diff --git a/src/main/java/com/gitblit/models/FilestoreModel.java b/src/main/java/com/gitblit/models/FilestoreModel.java index ff7b210e..4144df6b 100644 --- a/src/main/java/com/gitblit/models/FilestoreModel.java +++ b/src/main/java/com/gitblit/models/FilestoreModel.java @@ -111,6 +111,15 @@ public class FilestoreModel implements Serializable { repositories.remove(repo); } + public synchronized boolean isInRepositoryList(List repoList) { + for (String name : repositories) { + if (repoList.contains(name)) { + return true; + } + } + return false; + } + public static enum Status { Deleted(-30), diff --git a/src/main/java/com/gitblit/wicket/pages/FilestorePage.java b/src/main/java/com/gitblit/wicket/pages/FilestorePage.java index 97d5f25b..be0181bf 100644 --- a/src/main/java/com/gitblit/wicket/pages/FilestorePage.java +++ b/src/main/java/com/gitblit/wicket/pages/FilestorePage.java @@ -18,6 +18,7 @@ package com.gitblit.wicket.pages; import java.text.DateFormat; import java.text.MessageFormat; import java.text.SimpleDateFormat; +import java.util.ArrayList; import java.util.List; import org.apache.commons.io.FileUtils; @@ -31,9 +32,12 @@ import org.apache.wicket.markup.repeater.data.ListDataProvider; import com.gitblit.Constants; import com.gitblit.models.FilestoreModel; import com.gitblit.models.UserModel; +import com.gitblit.wicket.CacheControl; import com.gitblit.wicket.FilestoreUI; +import com.gitblit.wicket.GitBlitWebSession; import com.gitblit.wicket.RequiresAdminRole; import com.gitblit.wicket.WicketUtils; +import com.gitblit.wicket.CacheControl.LastModified; /** * Page to display the current status of the filestore. @@ -41,17 +45,22 @@ import com.gitblit.wicket.WicketUtils; * * @author Paul Martin */ -@RequiresAdminRole +@CacheControl(LastModified.ACTIVITY) public class FilestorePage extends RootPage { public FilestorePage() { super(); setupPage("", ""); - final List files = app().filestore().getAllObjects(); + final UserModel user = (GitBlitWebSession.get().getUser() == null) ? UserModel.ANONYMOUS : GitBlitWebSession.get().getUser(); final long nBytesUsed = app().filestore().getFilestoreUsedByteCount(); final long nBytesAvailable = app().filestore().getFilestoreAvailableByteCount(); + List files = app().filestore().getAllObjects(user); + if (files == null) { + files = new ArrayList(); + } + String message = MessageFormat.format(getString("gb.filestoreStats"), files.size(), FileUtils.byteCountToDisplaySize(nBytesUsed), FileUtils.byteCountToDisplaySize(nBytesAvailable) ); @@ -64,7 +73,6 @@ public class FilestorePage extends RootPage { helpLink.add(new Label("helpMessage", getString("gb.filestoreHelp"))); add(helpLink); - DataView filesView = new DataView("fileRow", new ListDataProvider(files)) { private static final long serialVersionUID = 1L; diff --git a/src/main/java/com/gitblit/wicket/pages/FilestoreUsage.html b/src/main/java/com/gitblit/wicket/pages/FilestoreUsage.html index e9bff47c..89e98cad 100644 --- a/src/main/java/com/gitblit/wicket/pages/FilestoreUsage.html +++ b/src/main/java/com/gitblit/wicket/pages/FilestoreUsage.html @@ -12,16 +12,19 @@
-

Using the Filestore

+

Using the filestore

- All clients intending to use the filestore must first install the Git-LFS Client and then run git lfs init to register the hooks globally.
- This version of GitBlit has been verified with Git-LFS client version 0.6.0 which requires Git v1.8.2 or higher. + All clients intending to use the filestore must first install the Git-LFS Client and then run git lfs install
+

+ If using password authentication it is recommended that you configure the git credential storage to avoid Git-LFS asking for your password on each file
+ On Windows for example: git config --global credential.helper wincred +

Clone

- Just git clone as usual, no further action is required as GitBlit is configured to use the default Git-LFS end point {repository}/info/lfs/objects/.
+ Just git clone as usual, no further action is required as Gitblit is configured to use the default Git-LFS end point {repository}/info/lfs/objects/.
If the repository uses a 3rd party Git-LFS server you will need to manually configure the correct endpoints.

@@ -38,28 +41,7 @@

See the current Git-LFS specification for further details.


-
-

Limitations & Warnings

-

GitBlit currently provides a server-only implementation of the opensource Git-LFS API, other implementations are available.
- However, until JGit provides Git-LFS client capabilities some GitBlit features may not be fully supported when using the filestore. - Notably: -

    -
  • Mirroring a repository that uses Git-LFS - Only the pointer files, not the large files, are mirrored.
    • -
  • Federation - Only the pointer files, not the large files, are transfered.
    • -
-

-
- -
-

GitBlit Configuration

-

GitBlit provides the following configuration items when using the filestore: -

filestore.storageFolder

-

Defines the path on the server where filestore objects are to be saved. This defaults to ${baseFolder}/lfs

-

filestore.maxUploadSize

-

Defines the maximum allowable size that can be uploaded to the filestore. Once a file is uploaded it will be unaffected by later changes in this property. This defaults to -1 indicating no limits.

-

-
- +
diff --git a/src/main/java/com/gitblit/wicket/pages/RootPage.java b/src/main/java/com/gitblit/wicket/pages/RootPage.java index 6ed5a357..12779ca2 100644 --- a/src/main/java/com/gitblit/wicket/pages/RootPage.java +++ b/src/main/java/com/gitblit/wicket/pages/RootPage.java @@ -197,9 +197,9 @@ public abstract class RootPage extends BasePage { } navLinks.add(new PageNavLink("gb.repositories", RepositoriesPage.class, getRootPageParameters())); - if (user.canAdmin()) { - navLinks.add(new PageNavLink("gb.filestore", FilestorePage.class, getRootPageParameters())); - } + + navLinks.add(new PageNavLink("gb.filestore", FilestorePage.class, getRootPageParameters())); + navLinks.add(new PageNavLink("gb.activity", ActivityPage.class, getRootPageParameters())); if (allowLucene) { navLinks.add(new PageNavLink("gb.search", LuceneSearchPage.class)); diff --git a/src/site/setup_filestore.mkd b/src/site/setup_filestore.mkd new file mode 100644 index 00000000..9ffc9c67 --- /dev/null +++ b/src/site/setup_filestore.mkd @@ -0,0 +1,61 @@ +## Configure Git Large File Storage + +Gitblit provides a filestore that supports the [Git Large File Storage (LFS) API](https://git-lfs.github.com/). + +### Server Configuration + +Gitblit is configured to work straight away. However you may want to update the following in `gitblit.properties`: + + + + + + + + + + + + + + + +
parametervaluedescription
filestore.storageFolder${baseFolder}/lfsThe path on the server where filestore objects are to be saved.
filestore.maxUploadSize-1The maximum allowable size that can be uploaded to the filestore. Once a file is uploaded it will be unaffected by later changes in this property. The default of -1 indicates no limits.
+ + +### Limitations + +Gitblit currently provides a server-only implementation of the opensource Git LFS API. + +1. Files in the filestore are not currently searchable by Lucene. +2. Mirroring a repository that uses Git LFS will only mirror the pointer files, not the large files. +3. Federation - Only the pointer files, not the large files, are transfered. + +Items 2 & 3 are pending [JGit Git LFS client capabilities](https://bugs.eclipse.org/bugs/show_bug.cgi?id=470333). + + +### How does it work? + +1. Files that should be handled by Git LFS are defined in the `.gitattributes` file. +2. Git LFS installs a pre-commit hook when installed `git lfs install`. +3. When a commit is made the pre-commit hook replaces the defined Git LFS files with a pointer file containing metadata about the file so that it can be found later. +4. When a commit is pushed, the changeset is sent to the git repository and the large files are sent to the filestore. + +For further details check out the [Git LFS specification](https://github.com/github/git-lfs/blob/master/docs/spec.md). + +### Convert/Migrate existing repository + +It is possible to migrate an existing repository containing large files to one that leverages the filestore. However, commit hash history will be altered. + +The following command may be run on a local repository: + + git filter-branch --prune-empty --tree-filter ' + git lfs track "*.docx" "*.pdf" > /dev/null + git add .gitattributes + git ls-files | xargs -d "\n" git check-attr filter | grep "filter: lfs" | sed -r "s/(.*): filter: lfs/\1/" | xargs -d "\n" -r bash -c "git rm -f --cached \"\$@\"; git add \"\$@\"" bash \ + ' --tag-name-filter cat -- --all + + +### Further Considerations + +While [other Git LFS implementations are available](https://github.com/github/git-lfs/wiki/Implementations) as there is no current [JGit LFS client capability](https://bugs.eclipse.org/bugs/show_bug.cgi?id=470333), Gitblit will be unable to access them. \ No newline at end of file -- 2.39.5