+ Filestore listing filtered by user view permissions + Configuration help for filestore relocated to website files + Added migration exampletags/v1.8.0
[[src/site/setup_viewer.mkd]] | [[src/site/setup_viewer.mkd]] | ||||
[[src/site/administration.mkd]] | [[src/site/administration.mkd]] | ||||
[[src/site/setup_scaling.mkd]] | [[src/site/setup_scaling.mkd]] | ||||
[[src/site/setup_filestore.mkd]] | |||||
### Gitblit Tickets | ### Gitblit Tickets | ||||
<page name="mirrors" src="setup_mirrors.mkd" /> | <page name="mirrors" src="setup_mirrors.mkd" /> | ||||
<page name="scaling" src="setup_scaling.mkd" /> | <page name="scaling" src="setup_scaling.mkd" /> | ||||
<page name="fail2ban" src="setup_fail2ban.mkd" /> | <page name="fail2ban" src="setup_fail2ban.mkd" /> | ||||
<page name="filestore (Git LFS)" src="setup_filestore.mkd" /> | |||||
<divider /> | <divider /> | ||||
<page name="Gitblit as a viewer" src="setup_viewer.mkd" /> | <page name="Gitblit as a viewer" src="setup_viewer.mkd" /> | ||||
</menu> | </menu> |
private final Logger logger = LoggerFactory.getLogger(getClass()); | private final Logger logger = LoggerFactory.getLogger(getClass()); | ||||
private final IRuntimeManager runtimeManager; | private final IRuntimeManager runtimeManager; | ||||
private final IRepositoryManager repositoryManager; | |||||
private final IStoredSettings settings; | private final IStoredSettings settings; | ||||
@Inject | @Inject | ||||
FilestoreManager( | FilestoreManager( | ||||
IRuntimeManager runtimeManager) { | |||||
IRuntimeManager runtimeManager, | |||||
IRepositoryManager repositoryManager) { | |||||
this.runtimeManager = runtimeManager; | this.runtimeManager = runtimeManager; | ||||
this.repositoryManager = repositoryManager; | |||||
this.settings = runtimeManager.getSettings(); | this.settings = runtimeManager.getSettings(); | ||||
} | } | ||||
} | } | ||||
@Override | @Override | ||||
public List<FilestoreModel> getAllObjects() { | |||||
return new ArrayList<FilestoreModel>(fileCache.values()); | |||||
public List<FilestoreModel> getAllObjects(UserModel user) { | |||||
final List<RepositoryModel> viewableRepositories = repositoryManager.getRepositoryModels(user); | |||||
List<String> viewableRepositoryNames = new ArrayList<String>(viewableRepositories.size()); | |||||
for (RepositoryModel repository : viewableRepositories) { | |||||
viewableRepositoryNames.add(repository.name); | |||||
} | |||||
if (viewableRepositoryNames.size() == 0) { | |||||
return null; | |||||
} | |||||
final Collection<FilestoreModel> allFiles = fileCache.values(); | |||||
List<FilestoreModel> userViewableFiles = new ArrayList<FilestoreModel>(allFiles.size()); | |||||
for (FilestoreModel file : allFiles) { | |||||
if (file.isInRepositoryList(viewableRepositoryNames)) { | |||||
userViewableFiles.add(file); | |||||
} | |||||
} | |||||
return userViewableFiles; | |||||
} | } | ||||
@Override | @Override |
} | } | ||||
@Override | @Override | ||||
public List<FilestoreModel> getAllObjects() { | |||||
return filestoreManager.getAllObjects(); | |||||
public List<FilestoreModel> getAllObjects(UserModel user) { | |||||
return filestoreManager.getAllObjects(user); | |||||
} | } | ||||
@Override | @Override |
FilestoreModel.Status downloadBlob(String oid, UserModel user, RepositoryModel repo, OutputStream streamOut ); | FilestoreModel.Status downloadBlob(String oid, UserModel user, RepositoryModel repo, OutputStream streamOut ); | ||||
List<FilestoreModel> getAllObjects(); | |||||
List<FilestoreModel> getAllObjects(UserModel user); | |||||
File getStorageFolder(); | File getStorageFolder(); | ||||
repositories.remove(repo); | repositories.remove(repo); | ||||
} | } | ||||
public synchronized boolean isInRepositoryList(List<String> repoList) { | |||||
for (String name : repositories) { | |||||
if (repoList.contains(name)) { | |||||
return true; | |||||
} | |||||
} | |||||
return false; | |||||
} | |||||
public static enum Status { | public static enum Status { | ||||
Deleted(-30), | Deleted(-30), |
import java.text.DateFormat; | import java.text.DateFormat; | ||||
import java.text.MessageFormat; | import java.text.MessageFormat; | ||||
import java.text.SimpleDateFormat; | import java.text.SimpleDateFormat; | ||||
import java.util.ArrayList; | |||||
import java.util.List; | import java.util.List; | ||||
import org.apache.commons.io.FileUtils; | import org.apache.commons.io.FileUtils; | ||||
import com.gitblit.Constants; | import com.gitblit.Constants; | ||||
import com.gitblit.models.FilestoreModel; | import com.gitblit.models.FilestoreModel; | ||||
import com.gitblit.models.UserModel; | import com.gitblit.models.UserModel; | ||||
import com.gitblit.wicket.CacheControl; | |||||
import com.gitblit.wicket.FilestoreUI; | import com.gitblit.wicket.FilestoreUI; | ||||
import com.gitblit.wicket.GitBlitWebSession; | |||||
import com.gitblit.wicket.RequiresAdminRole; | import com.gitblit.wicket.RequiresAdminRole; | ||||
import com.gitblit.wicket.WicketUtils; | import com.gitblit.wicket.WicketUtils; | ||||
import com.gitblit.wicket.CacheControl.LastModified; | |||||
/** | /** | ||||
* Page to display the current status of the filestore. | * Page to display the current status of the filestore. | ||||
* | * | ||||
* @author Paul Martin | * @author Paul Martin | ||||
*/ | */ | ||||
@RequiresAdminRole | |||||
@CacheControl(LastModified.ACTIVITY) | |||||
public class FilestorePage extends RootPage { | public class FilestorePage extends RootPage { | ||||
public FilestorePage() { | public FilestorePage() { | ||||
super(); | super(); | ||||
setupPage("", ""); | setupPage("", ""); | ||||
final List<FilestoreModel> files = app().filestore().getAllObjects(); | |||||
final UserModel user = (GitBlitWebSession.get().getUser() == null) ? UserModel.ANONYMOUS : GitBlitWebSession.get().getUser(); | |||||
final long nBytesUsed = app().filestore().getFilestoreUsedByteCount(); | final long nBytesUsed = app().filestore().getFilestoreUsedByteCount(); | ||||
final long nBytesAvailable = app().filestore().getFilestoreAvailableByteCount(); | final long nBytesAvailable = app().filestore().getFilestoreAvailableByteCount(); | ||||
List<FilestoreModel> files = app().filestore().getAllObjects(user); | |||||
if (files == null) { | |||||
files = new ArrayList<FilestoreModel>(); | |||||
} | |||||
String message = MessageFormat.format(getString("gb.filestoreStats"), files.size(), | String message = MessageFormat.format(getString("gb.filestoreStats"), files.size(), | ||||
FileUtils.byteCountToDisplaySize(nBytesUsed), FileUtils.byteCountToDisplaySize(nBytesAvailable) ); | FileUtils.byteCountToDisplaySize(nBytesUsed), FileUtils.byteCountToDisplaySize(nBytesAvailable) ); | ||||
helpLink.add(new Label("helpMessage", getString("gb.filestoreHelp"))); | helpLink.add(new Label("helpMessage", getString("gb.filestoreHelp"))); | ||||
add(helpLink); | add(helpLink); | ||||
DataView<FilestoreModel> filesView = new DataView<FilestoreModel>("fileRow", | DataView<FilestoreModel> filesView = new DataView<FilestoreModel>("fileRow", | ||||
new ListDataProvider<FilestoreModel>(files)) { | new ListDataProvider<FilestoreModel>(files)) { | ||||
private static final long serialVersionUID = 1L; | private static final long serialVersionUID = 1L; |
<div class="span10 offset1"> | <div class="span10 offset1"> | ||||
<div class="alert alert-danger"> | <div class="alert alert-danger"> | ||||
<h3><center>Using the Filestore</center></h3> | |||||
<h3><center>Using the filestore</center></h3> | |||||
<p> | <p> | ||||
<strong>All clients intending to use the filestore must first install the <a href="https://git-lfs.github.com/">Git-LFS Client</a> and then run <code>git lfs init</code> to register the hooks globally.</strong><br/> | |||||
<i>This version of GitBlit has been verified with Git-LFS client version 0.6.0 which requires Git v1.8.2 or higher.</i> | |||||
<strong>All clients intending to use the filestore must first install the <a href="https://git-lfs.github.com/">Git-LFS Client</a> and then run <code>git lfs install</code></strong><br/> | |||||
<p> | |||||
If using password authentication it is recommended that you configure the <a href="https://git-scm.com/book/en/v2/Git-Tools-Credential-Storage">git credential storage</a> to avoid Git-LFS asking for your password on each file<br/> | |||||
On Windows for example: <code>git config --global credential.helper wincred</code> | |||||
</p> | |||||
</p> | </p> | ||||
</div> | </div> | ||||
<h3>Clone</h3> | <h3>Clone</h3> | ||||
<p> | <p> | ||||
Just <code>git clone</code> as usual, no further action is required as GitBlit is configured to use the default Git-LFS end point <code>{repository}/info/lfs/objects/</code>.<br/> | |||||
Just <code>git clone</code> as usual, no further action is required as Gitblit is configured to use the default Git-LFS end point <code>{repository}/info/lfs/objects/</code>.<br/> | |||||
<i>If the repository uses a 3rd party Git-LFS server you will need to <a href="https://github.com/github/git-lfs/blob/master/docs/spec.md#the-server">manually configure the correct endpoints</a></i>. | <i>If the repository uses a 3rd party Git-LFS server you will need to <a href="https://github.com/github/git-lfs/blob/master/docs/spec.md#the-server">manually configure the correct endpoints</a></i>. | ||||
</p> | </p> | ||||
<p><a href="https://github.com/github/git-lfs/blob/master/docs/spec.md">See the current Git-LFS specification for further details</a>.</p> | <p><a href="https://github.com/github/git-lfs/blob/master/docs/spec.md">See the current Git-LFS specification for further details</a>.</p> | ||||
<br /> | <br /> | ||||
<div class="alert alert-warn"> | |||||
<h3><center>Limitations & Warnings</center></h3> | |||||
<p>GitBlit currently provides a server-only implementation of the opensource Git-LFS API, <a href="https://github.com/github/git-lfs/wiki/Implementations">other implementations</a> are available.<br/> | |||||
However, until <a href="https://bugs.eclipse.org/bugs/show_bug.cgi?id=470333">JGit provides Git-LFS client capabilities</a> some GitBlit features may not be fully supported when using the filestore. | |||||
Notably: | |||||
<ul> | |||||
<li>Mirroring a repository that uses Git-LFS - Only the pointer files, not the large files, are mirrored.</li> | |||||
<li>Federation - Only the pointer files, not the large files, are transfered.</li> | |||||
</ul> | |||||
</p> | |||||
</div> | |||||
<div class="alert alert-info"> | |||||
<h3><center>GitBlit Configuration</center></h3> | |||||
<p>GitBlit provides the following configuration items when using the filestore: | |||||
<h4>filestore.storageFolder</h4> | |||||
<p>Defines the path on the server where filestore objects are to be saved. This defaults to <code>${baseFolder}/lfs</code></p> | |||||
<h4>filestore.maxUploadSize</h4> | |||||
<p>Defines the maximum allowable size that can be uploaded to the filestore. Once a file is uploaded it will be unaffected by later changes in this property. This defaults to <code>-1</code> indicating no limits.</p> | |||||
</p> | |||||
</div> | |||||
</div> | </div> | ||||
</div> | </div> | ||||
</div> | </div> |
} | } | ||||
navLinks.add(new PageNavLink("gb.repositories", RepositoriesPage.class, | navLinks.add(new PageNavLink("gb.repositories", RepositoriesPage.class, | ||||
getRootPageParameters())); | getRootPageParameters())); | ||||
if (user.canAdmin()) { | |||||
navLinks.add(new PageNavLink("gb.filestore", FilestorePage.class, getRootPageParameters())); | |||||
} | |||||
navLinks.add(new PageNavLink("gb.filestore", FilestorePage.class, getRootPageParameters())); | |||||
navLinks.add(new PageNavLink("gb.activity", ActivityPage.class, getRootPageParameters())); | navLinks.add(new PageNavLink("gb.activity", ActivityPage.class, getRootPageParameters())); | ||||
if (allowLucene) { | if (allowLucene) { | ||||
navLinks.add(new PageNavLink("gb.search", LuceneSearchPage.class)); | navLinks.add(new PageNavLink("gb.search", LuceneSearchPage.class)); |
## Configure Git Large File Storage | |||||
Gitblit provides a filestore that supports the [Git Large File Storage (LFS) API](https://git-lfs.github.com/). | |||||
### Server Configuration | |||||
Gitblit is configured to work straight away. However you may want to update the following in `gitblit.properties`: | |||||
<table class="table"> | |||||
<thead> | |||||
<tr><th>parameter</th><th>value</th><th>description</th></tr> | |||||
</thead> | |||||
<tbody> | |||||
<tr> | |||||
<th>filestore.storageFolder</th><td>${baseFolder}/lfs</td> | |||||
<td>The path on the server where filestore objects are to be saved.</td> | |||||
</tr> | |||||
<tr> | |||||
<th>filestore.maxUploadSize</th><td>-1</td> | |||||
<td>The maximum allowable size that can be uploaded to the filestore. Once a file is uploaded it will be unaffected by later changes in this property. The default of -1 indicates no limits.</td> | |||||
</tr> | |||||
</tbody> | |||||
</table> | |||||
### Limitations | |||||
Gitblit currently provides a server-only implementation of the opensource Git LFS API. | |||||
1. Files in the filestore are not currently searchable by Lucene. | |||||
2. Mirroring a repository that uses Git LFS will only mirror the pointer files, not the large files. | |||||
3. Federation - Only the pointer files, not the large files, are transfered. | |||||
Items 2 & 3 are pending [JGit Git LFS client capabilities](https://bugs.eclipse.org/bugs/show_bug.cgi?id=470333). | |||||
### How does it work? | |||||
1. Files that should be handled by Git LFS are defined in the `.gitattributes` file. | |||||
2. Git LFS installs a pre-commit hook when installed `git lfs install`. | |||||
3. When a commit is made the pre-commit hook replaces the defined Git LFS files with a pointer file containing metadata about the file so that it can be found later. | |||||
4. When a commit is pushed, the changeset is sent to the git repository and the large files are sent to the filestore. | |||||
For further details check out the [Git LFS specification](https://github.com/github/git-lfs/blob/master/docs/spec.md). | |||||
### Convert/Migrate existing repository | |||||
It is possible to migrate an existing repository containing large files to one that leverages the filestore. However, commit hash history will be altered. | |||||
The following command may be run on a local repository: | |||||
git filter-branch --prune-empty --tree-filter ' | |||||
git lfs track "*.docx" "*.pdf" > /dev/null | |||||
git add .gitattributes | |||||
git ls-files | xargs -d "\n" git check-attr filter | grep "filter: lfs" | sed -r "s/(.*): filter: lfs/\1/" | xargs -d "\n" -r bash -c "git rm -f --cached \"\$@\"; git add \"\$@\"" bash \ | |||||
' --tag-name-filter cat -- --all | |||||
### Further Considerations | |||||
While [other Git LFS implementations are available](https://github.com/github/git-lfs/wiki/Implementations) as there is no current [JGit LFS client capability](https://bugs.eclipse.org/bugs/show_bug.cgi?id=470333), Gitblit will be unable to access them. |