+ Filestore listing filtered by user view permissions + Configuration help for filestore relocated to website files + Added migration exampletags/v1.8.0
@@ -33,6 +33,8 @@ This documentation is the source content from which the [Gitblit website](http:/ | |||
[[src/site/setup_viewer.mkd]] | |||
[[src/site/administration.mkd]] | |||
[[src/site/setup_scaling.mkd]] | |||
[[src/site/setup_filestore.mkd]] | |||
### Gitblit Tickets | |||
@@ -514,6 +514,7 @@ | |||
<page name="mirrors" src="setup_mirrors.mkd" /> | |||
<page name="scaling" src="setup_scaling.mkd" /> | |||
<page name="fail2ban" src="setup_fail2ban.mkd" /> | |||
<page name="filestore (Git LFS)" src="setup_filestore.mkd" /> | |||
<divider /> | |||
<page name="Gitblit as a viewer" src="setup_viewer.mkd" /> | |||
</menu> |
@@ -77,6 +77,8 @@ public class FilestoreManager implements IFilestoreManager { | |||
private final Logger logger = LoggerFactory.getLogger(getClass()); | |||
private final IRuntimeManager runtimeManager; | |||
private final IRepositoryManager repositoryManager; | |||
private final IStoredSettings settings; | |||
@@ -93,8 +95,10 @@ public class FilestoreManager implements IFilestoreManager { | |||
@Inject | |||
FilestoreManager( | |||
IRuntimeManager runtimeManager) { | |||
IRuntimeManager runtimeManager, | |||
IRepositoryManager repositoryManager) { | |||
this.runtimeManager = runtimeManager; | |||
this.repositoryManager = repositoryManager; | |||
this.settings = runtimeManager.getSettings(); | |||
} | |||
@@ -324,8 +328,29 @@ public class FilestoreManager implements IFilestoreManager { | |||
} | |||
@Override | |||
public List<FilestoreModel> getAllObjects() { | |||
return new ArrayList<FilestoreModel>(fileCache.values()); | |||
public List<FilestoreModel> getAllObjects(UserModel user) { | |||
final List<RepositoryModel> viewableRepositories = repositoryManager.getRepositoryModels(user); | |||
List<String> viewableRepositoryNames = new ArrayList<String>(viewableRepositories.size()); | |||
for (RepositoryModel repository : viewableRepositories) { | |||
viewableRepositoryNames.add(repository.name); | |||
} | |||
if (viewableRepositoryNames.size() == 0) { | |||
return null; | |||
} | |||
final Collection<FilestoreModel> allFiles = fileCache.values(); | |||
List<FilestoreModel> userViewableFiles = new ArrayList<FilestoreModel>(allFiles.size()); | |||
for (FilestoreModel file : allFiles) { | |||
if (file.isInRepositoryList(viewableRepositoryNames)) { | |||
userViewableFiles.add(file); | |||
} | |||
} | |||
return userViewableFiles; | |||
} | |||
@Override |
@@ -1274,8 +1274,8 @@ public class GitblitManager implements IGitblit { | |||
} | |||
@Override | |||
public List<FilestoreModel> getAllObjects() { | |||
return filestoreManager.getAllObjects(); | |||
public List<FilestoreModel> getAllObjects(UserModel user) { | |||
return filestoreManager.getAllObjects(user); | |||
} | |||
@Override |
@@ -37,7 +37,7 @@ public interface IFilestoreManager extends IManager { | |||
FilestoreModel.Status downloadBlob(String oid, UserModel user, RepositoryModel repo, OutputStream streamOut ); | |||
List<FilestoreModel> getAllObjects(); | |||
List<FilestoreModel> getAllObjects(UserModel user); | |||
File getStorageFolder(); | |||
@@ -111,6 +111,15 @@ public class FilestoreModel implements Serializable { | |||
repositories.remove(repo); | |||
} | |||
public synchronized boolean isInRepositoryList(List<String> repoList) { | |||
for (String name : repositories) { | |||
if (repoList.contains(name)) { | |||
return true; | |||
} | |||
} | |||
return false; | |||
} | |||
public static enum Status { | |||
Deleted(-30), |
@@ -18,6 +18,7 @@ package com.gitblit.wicket.pages; | |||
import java.text.DateFormat; | |||
import java.text.MessageFormat; | |||
import java.text.SimpleDateFormat; | |||
import java.util.ArrayList; | |||
import java.util.List; | |||
import org.apache.commons.io.FileUtils; | |||
@@ -31,9 +32,12 @@ import org.apache.wicket.markup.repeater.data.ListDataProvider; | |||
import com.gitblit.Constants; | |||
import com.gitblit.models.FilestoreModel; | |||
import com.gitblit.models.UserModel; | |||
import com.gitblit.wicket.CacheControl; | |||
import com.gitblit.wicket.FilestoreUI; | |||
import com.gitblit.wicket.GitBlitWebSession; | |||
import com.gitblit.wicket.RequiresAdminRole; | |||
import com.gitblit.wicket.WicketUtils; | |||
import com.gitblit.wicket.CacheControl.LastModified; | |||
/** | |||
* Page to display the current status of the filestore. | |||
@@ -41,17 +45,22 @@ import com.gitblit.wicket.WicketUtils; | |||
* | |||
* @author Paul Martin | |||
*/ | |||
@RequiresAdminRole | |||
@CacheControl(LastModified.ACTIVITY) | |||
public class FilestorePage extends RootPage { | |||
public FilestorePage() { | |||
super(); | |||
setupPage("", ""); | |||
final List<FilestoreModel> files = app().filestore().getAllObjects(); | |||
final UserModel user = (GitBlitWebSession.get().getUser() == null) ? UserModel.ANONYMOUS : GitBlitWebSession.get().getUser(); | |||
final long nBytesUsed = app().filestore().getFilestoreUsedByteCount(); | |||
final long nBytesAvailable = app().filestore().getFilestoreAvailableByteCount(); | |||
List<FilestoreModel> files = app().filestore().getAllObjects(user); | |||
if (files == null) { | |||
files = new ArrayList<FilestoreModel>(); | |||
} | |||
String message = MessageFormat.format(getString("gb.filestoreStats"), files.size(), | |||
FileUtils.byteCountToDisplaySize(nBytesUsed), FileUtils.byteCountToDisplaySize(nBytesAvailable) ); | |||
@@ -64,7 +73,6 @@ public class FilestorePage extends RootPage { | |||
helpLink.add(new Label("helpMessage", getString("gb.filestoreHelp"))); | |||
add(helpLink); | |||
DataView<FilestoreModel> filesView = new DataView<FilestoreModel>("fileRow", | |||
new ListDataProvider<FilestoreModel>(files)) { | |||
private static final long serialVersionUID = 1L; |
@@ -12,16 +12,19 @@ | |||
<div class="span10 offset1"> | |||
<div class="alert alert-danger"> | |||
<h3><center>Using the Filestore</center></h3> | |||
<h3><center>Using the filestore</center></h3> | |||
<p> | |||
<strong>All clients intending to use the filestore must first install the <a href="https://git-lfs.github.com/">Git-LFS Client</a> and then run <code>git lfs init</code> to register the hooks globally.</strong><br/> | |||
<i>This version of GitBlit has been verified with Git-LFS client version 0.6.0 which requires Git v1.8.2 or higher.</i> | |||
<strong>All clients intending to use the filestore must first install the <a href="https://git-lfs.github.com/">Git-LFS Client</a> and then run <code>git lfs install</code></strong><br/> | |||
<p> | |||
If using password authentication it is recommended that you configure the <a href="https://git-scm.com/book/en/v2/Git-Tools-Credential-Storage">git credential storage</a> to avoid Git-LFS asking for your password on each file<br/> | |||
On Windows for example: <code>git config --global credential.helper wincred</code> | |||
</p> | |||
</p> | |||
</div> | |||
<h3>Clone</h3> | |||
<p> | |||
Just <code>git clone</code> as usual, no further action is required as GitBlit is configured to use the default Git-LFS end point <code>{repository}/info/lfs/objects/</code>.<br/> | |||
Just <code>git clone</code> as usual, no further action is required as Gitblit is configured to use the default Git-LFS end point <code>{repository}/info/lfs/objects/</code>.<br/> | |||
<i>If the repository uses a 3rd party Git-LFS server you will need to <a href="https://github.com/github/git-lfs/blob/master/docs/spec.md#the-server">manually configure the correct endpoints</a></i>. | |||
</p> | |||
@@ -38,28 +41,7 @@ | |||
<p><a href="https://github.com/github/git-lfs/blob/master/docs/spec.md">See the current Git-LFS specification for further details</a>.</p> | |||
<br /> | |||
<div class="alert alert-warn"> | |||
<h3><center>Limitations & Warnings</center></h3> | |||
<p>GitBlit currently provides a server-only implementation of the opensource Git-LFS API, <a href="https://github.com/github/git-lfs/wiki/Implementations">other implementations</a> are available.<br/> | |||
However, until <a href="https://bugs.eclipse.org/bugs/show_bug.cgi?id=470333">JGit provides Git-LFS client capabilities</a> some GitBlit features may not be fully supported when using the filestore. | |||
Notably: | |||
<ul> | |||
<li>Mirroring a repository that uses Git-LFS - Only the pointer files, not the large files, are mirrored.</li> | |||
<li>Federation - Only the pointer files, not the large files, are transfered.</li> | |||
</ul> | |||
</p> | |||
</div> | |||
<div class="alert alert-info"> | |||
<h3><center>GitBlit Configuration</center></h3> | |||
<p>GitBlit provides the following configuration items when using the filestore: | |||
<h4>filestore.storageFolder</h4> | |||
<p>Defines the path on the server where filestore objects are to be saved. This defaults to <code>${baseFolder}/lfs</code></p> | |||
<h4>filestore.maxUploadSize</h4> | |||
<p>Defines the maximum allowable size that can be uploaded to the filestore. Once a file is uploaded it will be unaffected by later changes in this property. This defaults to <code>-1</code> indicating no limits.</p> | |||
</p> | |||
</div> | |||
</div> | |||
</div> | |||
</div> |
@@ -197,9 +197,9 @@ public abstract class RootPage extends BasePage { | |||
} | |||
navLinks.add(new PageNavLink("gb.repositories", RepositoriesPage.class, | |||
getRootPageParameters())); | |||
if (user.canAdmin()) { | |||
navLinks.add(new PageNavLink("gb.filestore", FilestorePage.class, getRootPageParameters())); | |||
} | |||
navLinks.add(new PageNavLink("gb.filestore", FilestorePage.class, getRootPageParameters())); | |||
navLinks.add(new PageNavLink("gb.activity", ActivityPage.class, getRootPageParameters())); | |||
if (allowLucene) { | |||
navLinks.add(new PageNavLink("gb.search", LuceneSearchPage.class)); |
@@ -0,0 +1,61 @@ | |||
## Configure Git Large File Storage | |||
Gitblit provides a filestore that supports the [Git Large File Storage (LFS) API](https://git-lfs.github.com/). | |||
### Server Configuration | |||
Gitblit is configured to work straight away. However you may want to update the following in `gitblit.properties`: | |||
<table class="table"> | |||
<thead> | |||
<tr><th>parameter</th><th>value</th><th>description</th></tr> | |||
</thead> | |||
<tbody> | |||
<tr> | |||
<th>filestore.storageFolder</th><td>${baseFolder}/lfs</td> | |||
<td>The path on the server where filestore objects are to be saved.</td> | |||
</tr> | |||
<tr> | |||
<th>filestore.maxUploadSize</th><td>-1</td> | |||
<td>The maximum allowable size that can be uploaded to the filestore. Once a file is uploaded it will be unaffected by later changes in this property. The default of -1 indicates no limits.</td> | |||
</tr> | |||
</tbody> | |||
</table> | |||
### Limitations | |||
Gitblit currently provides a server-only implementation of the opensource Git LFS API. | |||
1. Files in the filestore are not currently searchable by Lucene. | |||
2. Mirroring a repository that uses Git LFS will only mirror the pointer files, not the large files. | |||
3. Federation - Only the pointer files, not the large files, are transfered. | |||
Items 2 & 3 are pending [JGit Git LFS client capabilities](https://bugs.eclipse.org/bugs/show_bug.cgi?id=470333). | |||
### How does it work? | |||
1. Files that should be handled by Git LFS are defined in the `.gitattributes` file. | |||
2. Git LFS installs a pre-commit hook when installed `git lfs install`. | |||
3. When a commit is made the pre-commit hook replaces the defined Git LFS files with a pointer file containing metadata about the file so that it can be found later. | |||
4. When a commit is pushed, the changeset is sent to the git repository and the large files are sent to the filestore. | |||
For further details check out the [Git LFS specification](https://github.com/github/git-lfs/blob/master/docs/spec.md). | |||
### Convert/Migrate existing repository | |||
It is possible to migrate an existing repository containing large files to one that leverages the filestore. However, commit hash history will be altered. | |||
The following command may be run on a local repository: | |||
git filter-branch --prune-empty --tree-filter ' | |||
git lfs track "*.docx" "*.pdf" > /dev/null | |||
git add .gitattributes | |||
git ls-files | xargs -d "\n" git check-attr filter | grep "filter: lfs" | sed -r "s/(.*): filter: lfs/\1/" | xargs -d "\n" -r bash -c "git rm -f --cached \"\$@\"; git add \"\$@\"" bash \ | |||
' --tag-name-filter cat -- --all | |||
### Further Considerations | |||
While [other Git LFS implementations are available](https://github.com/github/git-lfs/wiki/Implementations) as there is no current [JGit LFS client capability](https://bugs.eclipse.org/bugs/show_bug.cgi?id=470333), Gitblit will be unable to access them. |