Expose JGit's runtime configuration settings (issue 93)

6 files changed, 391 insertions, 116 deletions

diff --git a/distrib/gitblit.properties b/distrib/gitblit.properties
index 58833c0f..5292a91e 100644
--- a/distrib/gitblit.properties
+++ b/distrib/gitblit.properties
@@ -47,6 +47,98 @@ git.onlyAccessBareRepositories = false
 # SINCE 1.0.0

 git.defaultAccessRestriction = NONE

 

+# Number of bytes of a pack file to load into memory in a single read operation.

+# This is the "page size" of the JGit buffer cache, used for all pack access

+# operations. All disk IO occurs as single window reads. Setting this too large

+# may cause the process to load more data than is required; setting this too small

+# may increase the frequency of read() system calls.

+#

+# Default on JGit is 8 KiB on all platforms.

+#

+# Common unit suffixes of k, m, or g are supported.

+#

+# SINCE 1.0.0

+# RESTART REQUIRED

+git.packedGitWindowSize = 8k

+

+# Maximum number of bytes to load and cache in memory from pack files. If JGit

+# needs to access more than this many bytes it will unload less frequently used

+# windows to reclaim memory space within the process. As this buffer must be shared

+# with the rest of the JVM heap, it should be a fraction of the total memory available.

+#

+# The JGit team recommends setting this value larger than the size of your biggest

+# repository. This ensures you can serve most requests from memory.

+#

+# Default on JGit is 10 MiB on all platforms.

+#

+# Common unit suffixes of k, m, or g are supported.

+#

+# SINCE 1.0.0

+# RESTART REQUIRED

+git.packedGitLimit = 10m

+

+# Maximum number of bytes to reserve for caching base objects that multiple deltafied

+# objects reference. By storing the entire decompressed base object in a cache Git

+# is able to avoid unpacking and decompressing frequently used base objects multiple times.

+#

+# Default on JGit is 10 MiB on all platforms. You probably do not need to adjust

+# this value.

+#

+# Common unit suffixes of k, m, or g are supported.

+#

+# SINCE 1.0.0

+# RESTART REQUIRED

+git.deltaBaseCacheLimit = 10m

+

+# Maximum number of pack files to have open at once. A pack file must be opened

+# in order for any of its data to be available in a cached window.

+#

+# If you increase this to a larger setting you may need to also adjust the ulimit

+# on file descriptors for the host JVM, as Gitblit needs additional file descriptors

+# available for network sockets and other repository data manipulation.

+#

+# Default on JGit is 128 file descriptors on all platforms.

+#

+# SINCE 1.0.0

+# RESTART REQUIRED

+git.packedGitOpenFiles = 128

+

+# Largest object size, in bytes, that JGit will allocate as a contiguous byte

+# array. Any file revision larger than this threshold will have to be streamed,

+# typically requiring the use of temporary files under $GIT_DIR/objects to implement

+# psuedo-random access during delta decompression.

+#

+# Servers with very high traffic should set this to be larger than the size of

+# their common big files. For example a server managing the Android platform

+# typically has to deal with ~10-12 MiB XML files, so 15 m would be a reasonable

+# setting in that environment. Setting this too high may cause the JVM to run out

+# of heap space when handling very big binary files, such as device firmware or

+# CD-ROM ISO images. Make sure to adjust your JVM heap accordingly. 

+#

+# Default is 50 MiB on all platforms.

+#

+# Common unit suffixes of k, m, or g are supported.

+#

+# SINCE 1.0.0

+# RESTART REQUIRED

+git.streamFileThreshold = 50m

+

+# When true, JGit will use mmap() rather than malloc()+read() to load data from

+# pack files.  The use of mmap can be problematic on some JVMs as the garbage

+# collector must deduce that a memory mapped segment is no longer in use before

+# a call to munmap() can be made by the JVM native code.

+#

+# In server applications (such as Gitblit) that need to access many pack files,

+# setting this to true risks artificially running out of virtual address space, 

+# as the garbage collector cannot reclaim unused mapped spaces fast enough.

+#

+# Default on JGit is false. Although potentially slower, it yields much more

+# predictable behavior.

+#

+# SINCE 1.0.0

+# RESTART REQUIRED

+git.packedGitMmap = false

+

 #

 # Groovy Integration

 #

@@ -165,121 +257,6 @@ realm.passwordStorage = md5
 # SINCE 0.5.0 

 realm.minPasswordLength = 5

 

-# URL of the LDAP server.

-#

-# SINCE 1.0.0

-realm.ldap.server = ldap://localhost

-

-# Login username for LDAP searches.

-# If this value is unspecified, anonymous LDAP login will be used.

-# 

-# e.g. mydomain\\username

-#

-# SINCE 1.0.0

-realm.ldap.username = cn=Directory Manager

-

-# Login password for LDAP searches.

-#

-# SINCE 1.0.0

-realm.ldap.password = password

-

-# The LdapUserService must be backed by another user service for standard user

-# and team management.

-# default: users.conf

-#

-# SINCE 1.0.0

-# RESTART REQUIRED

-realm.ldap.backingUserService = users.conf

-

-# Delegate team membership control to LDAP.

-#

-# If true, team user memberships will be specified by LDAP groups.  This will

-# disable team selection in Edit User and user selection in Edit Team.

-#

-# If false, LDAP will only be used for authentication and Gitblit will maintain

-# team memberships with the *realm.ldap.backingUserService*.

-#

-# SINCE 1.0.0

-realm.ldap.maintainTeams = false

-

-# Root node for all LDAP users

-#

-# This is the root node from which subtree user searches will begin.

-# If blank, Gitblit will search ALL nodes.

-#

-# SINCE 1.0.0

-realm.ldap.accountBase = OU=Users,OU=UserControl,OU=MyOrganization,DC=MyDomain

-

-# Filter criteria for LDAP users

-#

-# Query pattern to use when searching for a user account. This may be any valid 

-# LDAP query expression, including the standard (&) and (|) operators.

-#

-# Variables may be injected via the ${variableName} syntax.

-# Recognized variables are:

-#    ${username} - The text entered as the user name

-#

-# SINCE 1.0.0

-realm.ldap.accountPattern = (&(objectClass=person)(sAMAccountName=${username}))

-

-# Root node for all LDAP groups to be used as Gitblit Teams

-#

-# This is the root node from which subtree team searches will begin.

-# If blank, Gitblit will search ALL nodes.  

-#

-# SINCE 1.0.0

-realm.ldap.groupBase = OU=Groups,OU=UserControl,OU=MyOrganization,DC=MyDomain

-

-# Filter criteria for LDAP groups

-#

-# Query pattern to use when searching for a team. This may be any valid 

-# LDAP query expression, including the standard (&) and (|) operators.

-#

-# Variables may be injected via the ${variableName} syntax.

-# Recognized variables are:

-#    ${username} - The text entered as the user name

-#    ${dn} - The Distinguished Name of the user logged in

-#

-# All attributes from the LDAP User record are available. For example, if a user

-# has an attribute "fullName" set to "John", "(fn=${fullName})" will be 

-# translated to "(fn=John)".

-#

-# SINCE 1.0.0

-realm.ldap.groupMemberPattern = (&(objectClass=group)(member=${dn}))

-

-# LDAP users or groups that should be given administrator privileges.

-#

-# Teams are specified with a leading '@' character.  Groups with spaces in the

-# name can be entered as "@team name".

-#

-# e.g. realm.ldap.admins = john @git_admins "@git admins"

-#

-# SPACE-DELIMITED

-# SINCE 1.0.0

-realm.ldap.admins = @Git_Admins

-

-# Attribute(s) on the USER record that indicate their display (or full) name.

-# Leave blank for no mapping available in LDAP.

-#

-# This may be a single attribute, or a string of multiple attributes.  Examples:

-#  displayName - Uses the attribute 'displayName' on the user record

-#  ${personalTitle}. ${givenName} ${surname} - Will concatenate the 3 

-#       attributes together, with a '.' after personalTitle

-#

-# SINCE 1.0.0

-realm.ldap.displayName = displayName

-

-# Attribute(s) on the USER record that indicate their email address.

-# Leave blank for no mapping available in LDAP.

-#

-# This may be a single attribute, or a string of multiple attributes.  Examples:

-#  email - Uses the attribute 'email' on the user record

-#  ${givenName}.${surname}@gitblit.com -Will concatenate the 2 attributes

-#       together with a '.' and '@' creating something like first.last@gitblit.com 

-#

-# SINCE 1.0.0

-realm.ldap.email = email

-

 #

 # Gitblit Web Settings

 #

@@ -754,6 +731,125 @@ federation.sets =
 #federation.example1.mergeAccounts = true

 

 #

+# Advanced Realm Settings

+#

+

+# URL of the LDAP server.

+#

+# SINCE 1.0.0

+realm.ldap.server = ldap://localhost

+

+# Login username for LDAP searches.

+# If this value is unspecified, anonymous LDAP login will be used.

+# 

+# e.g. mydomain\\username

+#

+# SINCE 1.0.0

+realm.ldap.username = cn=Directory Manager

+

+# Login password for LDAP searches.

+#

+# SINCE 1.0.0

+realm.ldap.password = password

+

+# The LdapUserService must be backed by another user service for standard user

+# and team management.

+# default: users.conf

+#

+# SINCE 1.0.0

+# RESTART REQUIRED

+realm.ldap.backingUserService = users.conf

+

+# Delegate team membership control to LDAP.

+#

+# If true, team user memberships will be specified by LDAP groups.  This will

+# disable team selection in Edit User and user selection in Edit Team.

+#

+# If false, LDAP will only be used for authentication and Gitblit will maintain

+# team memberships with the *realm.ldap.backingUserService*.

+#

+# SINCE 1.0.0

+realm.ldap.maintainTeams = false

+

+# Root node for all LDAP users

+#

+# This is the root node from which subtree user searches will begin.

+# If blank, Gitblit will search ALL nodes.

+#

+# SINCE 1.0.0

+realm.ldap.accountBase = OU=Users,OU=UserControl,OU=MyOrganization,DC=MyDomain

+

+# Filter criteria for LDAP users

+#

+# Query pattern to use when searching for a user account. This may be any valid 

+# LDAP query expression, including the standard (&) and (|) operators.

+#

+# Variables may be injected via the ${variableName} syntax.

+# Recognized variables are:

+#    ${username} - The text entered as the user name

+#

+# SINCE 1.0.0

+realm.ldap.accountPattern = (&(objectClass=person)(sAMAccountName=${username}))

+

+# Root node for all LDAP groups to be used as Gitblit Teams

+#

+# This is the root node from which subtree team searches will begin.

+# If blank, Gitblit will search ALL nodes.  

+#

+# SINCE 1.0.0

+realm.ldap.groupBase = OU=Groups,OU=UserControl,OU=MyOrganization,DC=MyDomain

+

+# Filter criteria for LDAP groups

+#

+# Query pattern to use when searching for a team. This may be any valid 

+# LDAP query expression, including the standard (&) and (|) operators.

+#

+# Variables may be injected via the ${variableName} syntax.

+# Recognized variables are:

+#    ${username} - The text entered as the user name

+#    ${dn} - The Distinguished Name of the user logged in

+#

+# All attributes from the LDAP User record are available. For example, if a user

+# has an attribute "fullName" set to "John", "(fn=${fullName})" will be 

+# translated to "(fn=John)".

+#

+# SINCE 1.0.0

+realm.ldap.groupMemberPattern = (&(objectClass=group)(member=${dn}))

+

+# LDAP users or groups that should be given administrator privileges.

+#

+# Teams are specified with a leading '@' character.  Groups with spaces in the

+# name can be entered as "@team name".

+#

+# e.g. realm.ldap.admins = john @git_admins "@git admins"

+#

+# SPACE-DELIMITED

+# SINCE 1.0.0

+realm.ldap.admins = @Git_Admins

+

+# Attribute(s) on the USER record that indicate their display (or full) name.

+# Leave blank for no mapping available in LDAP.

+#

+# This may be a single attribute, or a string of multiple attributes.  Examples:

+#  displayName - Uses the attribute 'displayName' on the user record

+#  ${personalTitle}. ${givenName} ${surname} - Will concatenate the 3 

+#       attributes together, with a '.' after personalTitle

+#

+# SINCE 1.0.0

+realm.ldap.displayName = displayName

+

+# Attribute(s) on the USER record that indicate their email address.

+# Leave blank for no mapping available in LDAP.

+#

+# This may be a single attribute, or a string of multiple attributes.  Examples:

+#  email - Uses the attribute 'email' on the user record

+#  ${givenName}.${surname}@gitblit.com -Will concatenate the 2 attributes

+#       together with a '.' and '@' creating something like first.last@gitblit.com 

+#

+# SINCE 1.0.0

+realm.ldap.email = email

+

+#

 # Server Settings

 #

 

diff --git a/docs/04_releases.mkd b/docs/04_releases.mkd
index 9e61a824..d20000bf 100644
--- a/docs/04_releases.mkd
+++ b/docs/04_releases.mkd
@@ -16,6 +16,13 @@
 

 #### additions

 

+- Exposed JGit's internal configuration settings in gitblit.properties/web.xml (issue 93)  

+    **New:** *git.packedGitWindowSize = 8k*  

+    **New:** *git.packedGitLimit = 10m*  

+    **New:** *git.deltaBaseCacheLimit = 10m*  

+    **New:** *git.packedGitOpenFiles = 128*  

+    **New:** *git.streamFileThreshold = 50m*  

+    **New:** *git.packedGitMmap = false*  

 - Added default access restriction.  Applies to new repositories and repositories that have not been configured with Gitblit. (issue 88)  

     **New:** *git.defaultAccessRestriction = NONE*  

 - Added LDAP User Service with many new *realm.ldap* keys (Github/jcrygier)

diff --git a/src/com/gitblit/GitBlit.java b/src/com/gitblit/GitBlit.java
index 969dc537..f96340ae 100644
--- a/src/com/gitblit/GitBlit.java
+++ b/src/com/gitblit/GitBlit.java
@@ -56,6 +56,8 @@ import org.eclipse.jgit.errors.RepositoryNotFoundException;
 import org.eclipse.jgit.lib.Repository;

 import org.eclipse.jgit.lib.RepositoryCache.FileKey;

 import org.eclipse.jgit.lib.StoredConfig;

+import org.eclipse.jgit.storage.file.WindowCache;

+import org.eclipse.jgit.storage.file.WindowCacheConfig;

 import org.eclipse.jgit.transport.resolver.FileResolver;

 import org.eclipse.jgit.transport.resolver.RepositoryResolver;

 import org.eclipse.jgit.transport.resolver.ServiceNotAuthorizedException;

@@ -1929,7 +1931,29 @@ public class GitBlit implements ServletContextListener {
 		scheduledExecutor.scheduleAtFixedRate(luceneExecutor, 1, 2, TimeUnit.MINUTES);

 		if (startFederation) {

 			configureFederation();

-		}		

+		}

+		

+		// Configure JGit

+		WindowCacheConfig cfg = new WindowCacheConfig();

+		

+		cfg.setPackedGitWindowSize(settings.getFilesize(Keys.git.packedGitWindowSize, cfg.getPackedGitWindowSize()));

+		cfg.setPackedGitLimit(settings.getFilesize(Keys.git.packedGitLimit, cfg.getPackedGitLimit()));

+		cfg.setDeltaBaseCacheLimit(settings.getFilesize(Keys.git.deltaBaseCacheLimit, cfg.getDeltaBaseCacheLimit()));

+		cfg.setPackedGitOpenFiles(settings.getFilesize(Keys.git.packedGitOpenFiles, cfg.getPackedGitOpenFiles()));

+		cfg.setStreamFileThreshold(settings.getFilesize(Keys.git.streamFileThreshold, cfg.getStreamFileThreshold()));

+		cfg.setPackedGitMMAP(settings.getBoolean(Keys.git.packedGitMmap, cfg.isPackedGitMMAP()));

+		

+		try {

+			WindowCache.reconfigure(cfg);

+			logger.debug(MessageFormat.format("{0} = {1,number,0}", Keys.git.packedGitWindowSize, cfg.getPackedGitWindowSize()));

+			logger.debug(MessageFormat.format("{0} = {1,number,0}", Keys.git.packedGitLimit, cfg.getPackedGitLimit()));

+			logger.debug(MessageFormat.format("{0} = {1,number,0}", Keys.git.deltaBaseCacheLimit, cfg.getDeltaBaseCacheLimit()));

+			logger.debug(MessageFormat.format("{0} = {1,number,0}", Keys.git.packedGitOpenFiles, cfg.getPackedGitOpenFiles()));

+			logger.debug(MessageFormat.format("{0} = {1,number,0}", Keys.git.streamFileThreshold, cfg.getStreamFileThreshold()));

+			logger.debug(MessageFormat.format("{0} = {1}", Keys.git.packedGitMmap, cfg.isPackedGitMMAP()));

+		} catch (IllegalArgumentException e) {

+			logger.error("Failed to configure JGit parameters!", e);

+		}

 	}

 	

 	private void logTimezone(String type, TimeZone zone) {

diff --git a/src/com/gitblit/IStoredSettings.java b/src/com/gitblit/IStoredSettings.java
index e0600919..790f8b68 100644
--- a/src/com/gitblit/IStoredSettings.java
+++ b/src/com/gitblit/IStoredSettings.java
@@ -120,6 +120,61 @@ public abstract class IStoredSettings {
 	}

 

 	/**

+	 * Returns the long value for the specified key. If the key does not

+	 * exist or the value for the key can not be interpreted as an long, the

+	 * defaultValue is returned.

+	 * 

+	 * @param key

+	 * @param defaultValue

+	 * @return key value or defaultValue

+	 */

+	public long getLong(String name, long defaultValue) {

+		Properties props = getSettings();

+		if (props.containsKey(name)) {

+			try {

+				String value = props.getProperty(name);

+				if (!StringUtils.isEmpty(value)) {

+					return Long.parseLong(value.trim());

+				}

+			} catch (NumberFormatException e) {

+				logger.warn("Failed to parse long for " + name + " using default of "

+						+ defaultValue);

+			}

+		}

+		return defaultValue;

+	}

+	

+	/**

+	 * Returns an int filesize from a string value such as 50m or 50mb

+	 * @param name

+	 * @param defaultValue

+	 * @return an int filesize or defaultValue if the key does not exist or can

+	 *         not be parsed

+	 */

+	public int getFilesize(String name, int defaultValue) {

+		String val = getString(name, null);

+		if (StringUtils.isEmpty(val)) {

+			return defaultValue;

+		}

+		return com.gitblit.utils.FileUtils.convertSizeToInt(val, defaultValue);

+	}

+	

+	/**

+	 * Returns an long filesize from a string value such as 50m or 50mb

+	 * @param name

+	 * @param defaultValue

+	 * @return a long filesize or defaultValue if the key does not exist or can

+	 *         not be parsed

+	 */

+	public long getFilesize(String key, long defaultValue) {

+		String val = getString(key, null);

+		if (StringUtils.isEmpty(val)) {

+			return defaultValue;

+		}

+		return com.gitblit.utils.FileUtils.convertSizeToLong(val, defaultValue);

+	}

+

+	/**

 	 * Returns the char value for the specified key. If the key does not exist

 	 * or the value for the key can not be interpreted as a char, the

 	 * defaultValue is returned.

diff --git a/src/com/gitblit/utils/FileUtils.java b/src/com/gitblit/utils/FileUtils.java
index f8d35c84..a14928f2 100644
--- a/src/com/gitblit/utils/FileUtils.java
+++ b/src/com/gitblit/utils/FileUtils.java
@@ -34,6 +34,71 @@ import java.nio.charset.Charset;
  * 

  */

 public class FileUtils {

+	

+	/** 1024 (number of bytes in one kilobyte) */

+	public static final int KB = 1024;

+

+	/** 1024 {@link #KB} (number of bytes in one megabyte) */

+	public static final int MB = 1024 * KB;

+

+	/** 1024 {@link #MB} (number of bytes in one gigabyte) */

+	public static final int GB = 1024 * MB;

+

+	/**

+	 * Returns an int from a string representation of a file size.

+	 * e.g. 50m = 50 megabytes

+	 * 

+	 * @param aString

+	 * @param defaultValue

+	 * @return an int value or the defaultValue if aString can not be parsed

+	 */

+	public static int convertSizeToInt(String aString, int defaultValue) {

+		return (int) convertSizeToLong(aString, defaultValue);

+	}

+	

+	/**

+	 * Returns a long from a string representation of a file size.

+	 * e.g. 50m = 50 megabytes

+	 * 

+	 * @param aString

+	 * @param defaultValue

+	 * @return a long value or the defaultValue if aString can not be parsed

+	 */

+	public static long convertSizeToLong(String aString, long defaultValue) {

+		// trim string and remove all spaces 

+		aString = aString.toLowerCase().trim();

+		StringBuilder sb = new StringBuilder();

+		for (String a : aString.split(" ")) {

+			sb.append(a);

+		}

+		aString = sb.toString();

+		

+		// identify value and unit

+		int idx = 0;

+		int len = aString.length();

+		while (Character.isDigit(aString.charAt(idx))) {

+			idx++;

+			if (idx == len) {

+				break;

+			}

+		}

+		long value = 0;

+		String unit = null;

+		try {

+			value = Long.parseLong(aString.substring(0, idx));

+			unit = aString.substring(idx);

+		} catch (Exception e) {

+			return defaultValue;

+		}

+		if (unit.equals("g") || unit.equals("gb")) {

+			return value * GB;

+		} else if (unit.equals("m") || unit.equals("mb")) {

+			return value * MB;

+		} else if (unit.equals("k") || unit.equals("kb")) {

+			return value * KB;

+		}

+		return defaultValue;

+	}

 

 	/**

 	 * Returns the string content of the specified file.

diff --git a/tests/com/gitblit/tests/FileUtilsTest.java b/tests/com/gitblit/tests/FileUtilsTest.java
index 12161bc2..8e5cf8a6 100644
--- a/tests/com/gitblit/tests/FileUtilsTest.java
+++ b/tests/com/gitblit/tests/FileUtilsTest.java
@@ -55,4 +55,32 @@ public class FileUtilsTest {
 		size = FileUtils.folderSize(file);

 		assertEquals("size is actually " + size, 11556L, size);

 	}

+	

+	@Test

+	public void testStringSizes() throws Exception {

+		assertEquals(50 * FileUtils.KB, FileUtils.convertSizeToInt("50k", 0));

+		assertEquals(50 * FileUtils.MB, FileUtils.convertSizeToInt("50m", 0));

+		assertEquals(2 * FileUtils.GB, FileUtils.convertSizeToInt("2g", 0));

+

+		assertEquals(50 * FileUtils.KB, FileUtils.convertSizeToInt("50kb", 0));

+		assertEquals(50 * FileUtils.MB, FileUtils.convertSizeToInt("50mb", 0));

+		assertEquals(2 * FileUtils.GB, FileUtils.convertSizeToInt("2gb", 0));

+

+		assertEquals(50L * FileUtils.KB, FileUtils.convertSizeToLong("50k", 0));

+		assertEquals(50L * FileUtils.MB, FileUtils.convertSizeToLong("50m", 0));

+		assertEquals(50L * FileUtils.GB, FileUtils.convertSizeToLong("50g", 0));

+

+		assertEquals(50L * FileUtils.KB, FileUtils.convertSizeToLong("50kb", 0));

+		assertEquals(50L * FileUtils.MB, FileUtils.convertSizeToLong("50mb", 0));

+		assertEquals(50L * FileUtils.GB, FileUtils.convertSizeToLong("50gb", 0));

+		

+		assertEquals(50 * FileUtils.KB, FileUtils.convertSizeToInt("50 k", 0));

+		assertEquals(50 * FileUtils.MB, FileUtils.convertSizeToInt("50 m", 0));

+		assertEquals(2 * FileUtils.GB, FileUtils.convertSizeToInt("2 g", 0));

+

+		assertEquals(50 * FileUtils.KB, FileUtils.convertSizeToInt("50 kb", 0));

+		assertEquals(50 * FileUtils.MB, FileUtils.convertSizeToInt("50 mb", 0));

+		assertEquals(2 * FileUtils.GB, FileUtils.convertSizeToInt("2 gb", 0));

+

+	}

 }
\ No newline at end of file