From bd0e83e350fc703bcae72a28c41b09d9a9cec594 Mon Sep 17 00:00:00 2001 From: Paul Martin Date: Sat, 10 Oct 2015 12:46:51 +0100 Subject: Git-LFS support + Metadata maintained in append-only JSON file providing complete audit history. + Filestore menu item + Lists filestore items + Current size and availability + Link to GitBlit Filestore help page (top right) + Hooks into existing repository permissions + Uses default repository path for out-of-box operation with Git-LFS client + accessRestrictionFilter now has access to http method and auth header + Testing for servlet and manager --- src/main/java/com/gitblit/wicket/FilestoreUI.java | 62 +++++++++++ .../java/com/gitblit/wicket/GitBlitWebApp.java | 16 ++- .../com/gitblit/wicket/GitBlitWebApp.properties | 8 +- .../java/com/gitblit/wicket/GitblitWicketApp.java | 3 + .../com/gitblit/wicket/pages/FilestorePage.html | 37 +++++++ .../com/gitblit/wicket/pages/FilestorePage.java | 114 +++++++++++++++++++++ .../com/gitblit/wicket/pages/FilestoreUsage.html | 69 +++++++++++++ .../com/gitblit/wicket/pages/FilestoreUsage.java | 25 +++++ .../java/com/gitblit/wicket/pages/RootPage.java | 1 + 9 files changed, 333 insertions(+), 2 deletions(-) create mode 100644 src/main/java/com/gitblit/wicket/FilestoreUI.java create mode 100644 src/main/java/com/gitblit/wicket/pages/FilestorePage.html create mode 100644 src/main/java/com/gitblit/wicket/pages/FilestorePage.java create mode 100644 src/main/java/com/gitblit/wicket/pages/FilestoreUsage.html create mode 100644 src/main/java/com/gitblit/wicket/pages/FilestoreUsage.java (limited to 'src/main/java/com/gitblit/wicket') diff --git a/src/main/java/com/gitblit/wicket/FilestoreUI.java b/src/main/java/com/gitblit/wicket/FilestoreUI.java new file mode 100644 index 00000000..8837ba18 --- /dev/null +++ b/src/main/java/com/gitblit/wicket/FilestoreUI.java @@ -0,0 +1,62 @@ +/* + * Copyright 2015 gitblit.com. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.gitblit.wicket; + +import org.apache.wicket.markup.html.basic.Label; + +import com.gitblit.models.FilestoreModel; +import com.gitblit.models.FilestoreModel.Status; + +/** + * Common filestore ui methods and classes. + * + * @author Paul Martin + * + */ +public class FilestoreUI { + + public static Label getStatusIcon(String wicketId, FilestoreModel item) { + return getStatusIcon(wicketId, item.getStatus()); + } + + public static Label getStatusIcon(String wicketId, Status status) { + Label label = new Label(wicketId); + + switch (status) { + case Upload_Pending: + WicketUtils.setCssClass(label, "fa fa-spinner fa-fw file-negative"); + break; + case Upload_In_Progress: + WicketUtils.setCssClass(label, "fa fa-spinner fa-spin fa-fw file-positive"); + break; + case Available: + WicketUtils.setCssClass(label, "fa fa-check fa-fw file-positive"); + break; + case Deleted: + WicketUtils.setCssClass(label, "fa fa-ban fa-fw file-negative"); + break; + case Unavailable: + WicketUtils.setCssClass(label, "fa fa-times fa-fw file-negative"); + break; + default: + WicketUtils.setCssClass(label, "fa fa-exclamation-triangle fa-fw file-negative"); + } + WicketUtils.setHtmlTooltip(label, status.toString()); + + return label; + } + +} diff --git a/src/main/java/com/gitblit/wicket/GitBlitWebApp.java b/src/main/java/com/gitblit/wicket/GitBlitWebApp.java index 359040b5..296c2544 100644 --- a/src/main/java/com/gitblit/wicket/GitBlitWebApp.java +++ b/src/main/java/com/gitblit/wicket/GitBlitWebApp.java @@ -37,6 +37,7 @@ import com.gitblit.Keys; import com.gitblit.extensions.GitblitWicketPlugin; import com.gitblit.manager.IAuthenticationManager; import com.gitblit.manager.IFederationManager; +import com.gitblit.manager.IFilestoreManager; import com.gitblit.manager.IGitblit; import com.gitblit.manager.INotificationManager; import com.gitblit.manager.IPluginManager; @@ -63,6 +64,7 @@ import com.gitblit.wicket.pages.EditRepositoryPage; import com.gitblit.wicket.pages.EditTicketPage; import com.gitblit.wicket.pages.ExportTicketPage; import com.gitblit.wicket.pages.FederationRegistrationPage; +import com.gitblit.wicket.pages.FilestorePage; import com.gitblit.wicket.pages.ForkPage; import com.gitblit.wicket.pages.ForksPage; import com.gitblit.wicket.pages.GitSearchPage; @@ -131,6 +133,8 @@ public class GitBlitWebApp extends WebApplication implements GitblitWicketApp { private final IGitblit gitblit; private final IServicesManager services; + + private final IFilestoreManager filestoreManager; @Inject public GitBlitWebApp( @@ -145,7 +149,8 @@ public class GitBlitWebApp extends WebApplication implements GitblitWicketApp { IProjectManager projectManager, IFederationManager federationManager, IGitblit gitblit, - IServicesManager services) { + IServicesManager services, + IFilestoreManager filestoreManager) { super(); this.publicKeyManagerProvider = publicKeyManagerProvider; @@ -162,6 +167,7 @@ public class GitBlitWebApp extends WebApplication implements GitblitWicketApp { this.federationManager = federationManager; this.gitblit = gitblit; this.services = services; + this.filestoreManager = filestoreManager; } @Override @@ -238,6 +244,9 @@ public class GitBlitWebApp extends WebApplication implements GitblitWicketApp { mount("/user", UserPage.class, "user"); mount("/forks", ForksPage.class, "r"); mount("/fork", ForkPage.class, "r"); + + // filestore URL + mount("/filestore", FilestorePage.class); // allow started Wicket plugins to initialize for (PluginWrapper pluginWrapper : pluginManager.getPlugins()) { @@ -476,4 +485,9 @@ public class GitBlitWebApp extends WebApplication implements GitblitWicketApp { public static GitBlitWebApp get() { return (GitBlitWebApp) WebApplication.get(); } + + @Override + public IFilestoreManager filestore() { + return filestoreManager; + } } diff --git a/src/main/java/com/gitblit/wicket/GitBlitWebApp.properties b/src/main/java/com/gitblit/wicket/GitBlitWebApp.properties index d8027548..36c416e7 100644 --- a/src/main/java/com/gitblit/wicket/GitBlitWebApp.properties +++ b/src/main/java/com/gitblit/wicket/GitBlitWebApp.properties @@ -764,4 +764,10 @@ gb.deleteRepositoryHeader = Delete Repository gb.deleteRepositoryDescription = Deleted repositories will be unrecoverable. gb.show_whitespace = show whitespace gb.ignore_whitespace = ignore whitespace -gb.allRepositories = All Repositories \ No newline at end of file +gb.allRepositories = All Repositories +gb.oid = object id +gb.filestore = filestore +gb.filestoreStats = Filestore contains {0} files with a total size of {1}. ({2} remaining) +gb.statusChangedOn = status changed on +gb.statusChangedBy = status changed by +gb.filestoreHelp = How to use the Filestore? \ No newline at end of file diff --git a/src/main/java/com/gitblit/wicket/GitblitWicketApp.java b/src/main/java/com/gitblit/wicket/GitblitWicketApp.java index 3041c5da..fefa0f4a 100644 --- a/src/main/java/com/gitblit/wicket/GitblitWicketApp.java +++ b/src/main/java/com/gitblit/wicket/GitblitWicketApp.java @@ -8,6 +8,7 @@ import org.apache.wicket.markup.html.WebPage; import com.gitblit.IStoredSettings; import com.gitblit.manager.IAuthenticationManager; import com.gitblit.manager.IFederationManager; +import com.gitblit.manager.IFilestoreManager; import com.gitblit.manager.IGitblit; import com.gitblit.manager.INotificationManager; import com.gitblit.manager.IPluginManager; @@ -74,5 +75,7 @@ public interface GitblitWicketApp { public abstract ITicketService tickets(); public abstract TimeZone getTimezone(); + + public abstract IFilestoreManager filestore(); } \ No newline at end of file diff --git a/src/main/java/com/gitblit/wicket/pages/FilestorePage.html b/src/main/java/com/gitblit/wicket/pages/FilestorePage.html new file mode 100644 index 00000000..e373e704 --- /dev/null +++ b/src/main/java/com/gitblit/wicket/pages/FilestorePage.html @@ -0,0 +1,37 @@ + + + + + +
+ +
+ [repositories message] + [help message] +
+ + + + + + + + + + + + + + + + + + +
[Object status][changedOn][changedBy][Object ID][file size]
[Object state]
[changedOn][changedBy][Object ID][file size]
+
+ + + \ No newline at end of file diff --git a/src/main/java/com/gitblit/wicket/pages/FilestorePage.java b/src/main/java/com/gitblit/wicket/pages/FilestorePage.java new file mode 100644 index 00000000..5f103edd --- /dev/null +++ b/src/main/java/com/gitblit/wicket/pages/FilestorePage.java @@ -0,0 +1,114 @@ +/* + * Copyright 2015 gitblit.com. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.gitblit.wicket.pages; + +import java.text.DateFormat; +import java.text.MessageFormat; +import java.text.SimpleDateFormat; +import java.util.List; + +import org.apache.commons.io.FileUtils; +import org.apache.wicket.Component; +import org.apache.wicket.markup.html.basic.Label; +import org.apache.wicket.markup.html.link.BookmarkablePageLink; +import org.apache.wicket.markup.repeater.Item; +import org.apache.wicket.markup.repeater.data.DataView; +import org.apache.wicket.markup.repeater.data.ListDataProvider; + +import com.gitblit.Constants; +import com.gitblit.Keys; +import com.gitblit.models.FilestoreModel; +import com.gitblit.models.UserModel; +import com.gitblit.wicket.FilestoreUI; +import com.gitblit.wicket.GitBlitWebSession; +import com.gitblit.wicket.WicketUtils; + +/** + * Page to display the current status of the filestore. + * Certain errors also displayed to aid in fault finding + * + * @author Paul Martin + * + * + */ +public class FilestorePage extends RootPage { + + public FilestorePage() { + super(); + setupPage("", ""); + // check to see if we should display a login message + boolean authenticateView = app().settings().getBoolean(Keys.web.authenticateViewPages, true); + if (authenticateView && !GitBlitWebSession.get().isLoggedIn()) { + String messageSource = app().settings().getString(Keys.web.loginMessage, "gitblit"); + return; + } + + final List files = app().filestore().getAllObjects(); + final long nBytesUsed = app().filestore().getFilestoreUsedByteCount(); + final long nBytesAvailable = app().filestore().getFilestoreAvailableByteCount(); + + // Load the markdown welcome message + String messageSource = app().settings().getString(Keys.web.repositoriesMessage, "gitblit"); + String message = MessageFormat.format(getString("gb.filestoreStats"), files.size(), + FileUtils.byteCountToDisplaySize(nBytesUsed), FileUtils.byteCountToDisplaySize(nBytesAvailable) ); + + Component repositoriesMessage = new Label("repositoriesMessage", message) + .setEscapeModelStrings(false).setVisible(message.length() > 0); + + add(repositoriesMessage); + + BookmarkablePageLink helpLink = new BookmarkablePageLink("filestoreHelp", FilestoreUsage.class); + helpLink.add(new Label("helpMessage", getString("gb.filestoreHelp"))); + add(helpLink); + + + DataView filesView = new DataView("fileRow", + new ListDataProvider(files)) { + private static final long serialVersionUID = 1L; + private int counter; + + @Override + protected void onBeforeRender() { + super.onBeforeRender(); + counter = 0; + } + + @Override + public void populateItem(final Item item) { + final FilestoreModel entry = item.getModelObject(); + + DateFormat dateFormater = new SimpleDateFormat(Constants.ISO8601); + + UserModel user = app().users().getUserModel(entry.getChangedBy()); + user = user == null ? UserModel.ANONYMOUS : user; + + Label icon = FilestoreUI.getStatusIcon("status", entry); + item.add(icon); + item.add(new Label("on", dateFormater.format(entry.getChangedOn()))); + item.add(new Label("by", user.getDisplayName())); + + item.add(new Label("oid", entry.oid)); + item.add(new Label("size", FileUtils.byteCountToDisplaySize(entry.getSize()))); + + WicketUtils.setAlternatingBackground(item, counter); + counter++; + } + + }; + + add(filesView); + } +} diff --git a/src/main/java/com/gitblit/wicket/pages/FilestoreUsage.html b/src/main/java/com/gitblit/wicket/pages/FilestoreUsage.html new file mode 100644 index 00000000..e9bff47c --- /dev/null +++ b/src/main/java/com/gitblit/wicket/pages/FilestoreUsage.html @@ -0,0 +1,69 @@ + + + + + +
+
+
+
+ +
+

Using the Filestore

+

+ All clients intending to use the filestore must first install the Git-LFS Client and then run git lfs init to register the hooks globally.
+ This version of GitBlit has been verified with Git-LFS client version 0.6.0 which requires Git v1.8.2 or higher. +

+
+ +

Clone

+

+ Just git clone as usual, no further action is required as GitBlit is configured to use the default Git-LFS end point {repository}/info/lfs/objects/.
+ If the repository uses a 3rd party Git-LFS server you will need to manually configure the correct endpoints. +

+ +

Add

+

After configuring the file types or paths to be tracked using git lfs track "*.bin" just add files as usual with git add command.
+ Tracked files can also be configured manually using the .gitattributes file.

+ +

Remove

+

When you remove a Git-LFS tracked file only the pointer file will be removed from your repository.
+ All files remain on the server to allow previous versions to be checked out. +

+ +

Learn more...

+

See the current Git-LFS specification for further details.

+
+ +
+

Limitations & Warnings

+

GitBlit currently provides a server-only implementation of the opensource Git-LFS API, other implementations are available.
+ However, until JGit provides Git-LFS client capabilities some GitBlit features may not be fully supported when using the filestore. + Notably: +

    +
  • Mirroring a repository that uses Git-LFS - Only the pointer files, not the large files, are mirrored.
    • +
  • Federation - Only the pointer files, not the large files, are transfered.
    • +
+

+
+ +
+

GitBlit Configuration

+

GitBlit provides the following configuration items when using the filestore: +

filestore.storageFolder

+

Defines the path on the server where filestore objects are to be saved. This defaults to ${baseFolder}/lfs

+

filestore.maxUploadSize

+

Defines the maximum allowable size that can be uploaded to the filestore. Once a file is uploaded it will be unaffected by later changes in this property. This defaults to -1 indicating no limits.

+

+
+ +
+
+
+
+ + + diff --git a/src/main/java/com/gitblit/wicket/pages/FilestoreUsage.java b/src/main/java/com/gitblit/wicket/pages/FilestoreUsage.java new file mode 100644 index 00000000..9bd8e55d --- /dev/null +++ b/src/main/java/com/gitblit/wicket/pages/FilestoreUsage.java @@ -0,0 +1,25 @@ +/* + * Copyright 2015 gitblit.com. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.gitblit.wicket.pages; + +public class FilestoreUsage extends RootSubPage { + + public FilestoreUsage() { + super(); + setupPage("", ""); + } + +} diff --git a/src/main/java/com/gitblit/wicket/pages/RootPage.java b/src/main/java/com/gitblit/wicket/pages/RootPage.java index 79a4fc67..93d44fc7 100644 --- a/src/main/java/com/gitblit/wicket/pages/RootPage.java +++ b/src/main/java/com/gitblit/wicket/pages/RootPage.java @@ -191,6 +191,7 @@ public abstract class RootPage extends BasePage { } navLinks.add(new PageNavLink("gb.repositories", RepositoriesPage.class, getRootPageParameters())); + navLinks.add(new PageNavLink("gb.filestore", FilestorePage.class, getRootPageParameters())); navLinks.add(new PageNavLink("gb.activity", ActivityPage.class, getRootPageParameters())); if (allowLucene) { navLinks.add(new PageNavLink("gb.search", LuceneSearchPage.class)); -- cgit v1.2.3