Merge branch 'jvanhercke-jgit-default-charset' into master

This branch mostly records the commit from PR #1253, so we have it in
the code base. The changes are immediately undone, because the update
to JGit version 4.8.0, as proposed in PR #1252, renders the whole
patching of JGit obsolete.

The update is done now, against reservations in that PR, because we
guess that potential issues have been found and fixed by now. JGit
is updated to the last version on the major version 4 line: 4.11.9.
Also, since upstream JGit is at v6 by now, more updates will have to
happen later on anyways.

To make the protection against illegal character sets complete, an
additional fix for Bugtraq is included, that protects against exceptions
from bugtraq should it encounter an `encoding` that cannot be handled.

.classpath

@@ -58,14 +58,15 @@
 	<classpathentry kind="lib" path="ext/tracwiki-core-1.4.jar" sourcepath="ext/src/tracwiki-core-1.4.jar" />
 	<classpathentry kind="lib" path="ext/mediawiki-core-1.4.jar" sourcepath="ext/src/mediawiki-core-1.4.jar" />
 	<classpathentry kind="lib" path="ext/confluence-core-1.4.jar" sourcepath="ext/src/confluence-core-1.4.jar" />
-	<classpathentry kind="lib" path="ext/org.eclipse.jgit-4.6.1.201703071140-r.jar" sourcepath="ext/src/org.eclipse.jgit-4.6.1.201703071140-r.jar" />
-	<classpathentry kind="lib" path="ext/jsch-0.1.53.jar" sourcepath="ext/src/jsch-0.1.53.jar" />
+	<classpathentry kind="lib" path="ext/org.eclipse.jgit-4.11.9.201909030838-r.jar" sourcepath="ext/src/org.eclipse.jgit-4.11.9.201909030838-r.jar" />
+	<classpathentry kind="lib" path="ext/jsch-0.1.54.jar" sourcepath="ext/src/jsch-0.1.54.jar" />
+	<classpathentry kind="lib" path="ext/jzlib-1.1.1.jar" sourcepath="ext/src/jzlib-1.1.1.jar" />
 	<classpathentry kind="lib" path="ext/JavaEWAH-1.1.6.jar" sourcepath="ext/src/JavaEWAH-1.1.6.jar" />
-	<classpathentry kind="lib" path="ext/httpclient-4.3.6.jar" sourcepath="ext/src/httpclient-4.3.6.jar" />
-	<classpathentry kind="lib" path="ext/httpcore-4.3.3.jar" sourcepath="ext/src/httpcore-4.3.3.jar" />
-	<classpathentry kind="lib" path="ext/commons-logging-1.1.3.jar" sourcepath="ext/src/commons-logging-1.1.3.jar" />
-	<classpathentry kind="lib" path="ext/commons-codec-1.7.jar" sourcepath="ext/src/commons-codec-1.7.jar" />
-	<classpathentry kind="lib" path="ext/org.eclipse.jgit.http.server-4.6.1.201703071140-r.jar" sourcepath="ext/src/org.eclipse.jgit.http.server-4.6.1.201703071140-r.jar" />
+	<classpathentry kind="lib" path="ext/httpclient-4.5.2.jar" sourcepath="ext/src/httpclient-4.5.2.jar" />
+	<classpathentry kind="lib" path="ext/httpcore-4.4.4.jar" sourcepath="ext/src/httpcore-4.4.4.jar" />
+	<classpathentry kind="lib" path="ext/commons-logging-1.2.jar" sourcepath="ext/src/commons-logging-1.2.jar" />
+	<classpathentry kind="lib" path="ext/commons-codec-1.9.jar" sourcepath="ext/src/commons-codec-1.9.jar" />
+	<classpathentry kind="lib" path="ext/org.eclipse.jgit.http.server-4.11.9.201909030838-r.jar" sourcepath="ext/src/org.eclipse.jgit.http.server-4.11.9.201909030838-r.jar" />
 	<classpathentry kind="lib" path="ext/bcprov-jdk15on-1.69.jar" sourcepath="ext/src/bcprov-jdk15on-1.69.jar" />
 	<classpathentry kind="lib" path="ext/bcmail-jdk15on-1.69.jar" sourcepath="ext/src/bcmail-jdk15on-1.69.jar" />
 	<classpathentry kind="lib" path="ext/bcutil-jdk15on-1.69.jar" sourcepath="ext/src/bcutil-jdk15on-1.69.jar" />
@@ -75,13 +76,13 @@
 	<classpathentry kind="lib" path="ext/mina-core-2.0.21.jar" sourcepath="ext/src/mina-core-2.0.21.jar" />
 	<classpathentry kind="lib" path="ext/rome-0.9.jar" sourcepath="ext/src/rome-0.9.jar" />
 	<classpathentry kind="lib" path="ext/jdom-1.0.jar" sourcepath="ext/src/jdom-1.0.jar" />
-	<classpathentry kind="lib" path="ext/gson-2.3.1.jar" sourcepath="ext/src/gson-2.3.1.jar" />
+	<classpathentry kind="lib" path="ext/gson-2.8.2.jar" sourcepath="ext/src/gson-2.8.2.jar" />
 	<classpathentry kind="lib" path="ext/groovy-all-2.4.4.jar" sourcepath="ext/src/groovy-all-2.4.4.jar" />
 	<classpathentry kind="lib" path="ext/unboundid-ldapsdk-2.3.8.jar" sourcepath="ext/src/unboundid-ldapsdk-2.3.8.jar" />
 	<classpathentry kind="lib" path="ext/ivy-2.2.0.jar" sourcepath="ext/src/ivy-2.2.0.jar" />
 	<classpathentry kind="lib" path="ext/jcalendar-1.3.2.jar" />
-	<classpathentry kind="lib" path="ext/commons-compress-1.4.1.jar" sourcepath="ext/src/commons-compress-1.4.1.jar" />
-	<classpathentry kind="lib" path="ext/xz-1.0.jar" sourcepath="ext/src/xz-1.0.jar" />
+	<classpathentry kind="lib" path="ext/commons-compress-1.15.jar" sourcepath="ext/src/commons-compress-1.15.jar" />
+	<classpathentry kind="lib" path="ext/objenesis-2.6.jar" sourcepath="ext/src/objenesis-2.6.jar" />
 	<classpathentry kind="lib" path="ext/commons-io-2.2.jar" sourcepath="ext/src/commons-io-2.2.jar" />
 	<classpathentry kind="lib" path="ext/force-partner-api-24.0.0.jar" sourcepath="ext/src/force-partner-api-24.0.0.jar" />
 	<classpathentry kind="lib" path="ext/force-wsc-24.0.0.jar" sourcepath="ext/src/force-wsc-24.0.0.jar" />
@@ -112,7 +113,11 @@
 	<classpathentry kind="lib" path="ext/mockito-core-2.28.2.jar" sourcepath="ext/src/mockito-core-2.28.2.jar" />
 	<classpathentry kind="lib" path="ext/byte-buddy-1.9.10.jar" sourcepath="ext/src/byte-buddy-1.9.10.jar" />
 	<classpathentry kind="lib" path="ext/byte-buddy-agent-1.9.10.jar" sourcepath="ext/src/byte-buddy-agent-1.9.10.jar" />
-	<classpathentry kind="lib" path="ext/objenesis-2.6.jar" sourcepath="ext/src/objenesis-2.6.jar" />
 	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER" />
+	<classpathentry kind="src" path="src/main/dagger">
+		<attributes>
+			<attribute name="optional" value="true"/>
+		</attributes>
+	</classpathentry>
 	<classpathentry kind="output" path="bin/classes" />
 </classpath>

build.moxie

@@ -109,7 +109,7 @@ properties: {
   slf4j.version  : 1.7.29
   wicket.version : 1.4.22
   lucene.version : 5.5.2
-  jgit.version   : 4.6.1.201703071140-r
+  jgit.version   : 4.11.9.201909030838-r
   groovy.version : 2.4.4
   bouncycastle.version : 1.69
   selenium.version : 2.28.0
@@ -175,19 +175,19 @@ dependencies:
 - compile 'org.apache.sshd:sshd-core:${sshd.version}' :war !org.easymock
 - compile 'org.apache.mina:mina-core:${mina.version}' :war !org.easymock
 - compile 'rome:rome:0.9' :war :manager :api
-- compile 'com.google.code.gson:gson:2.3.1' :war :fedclient :manager :api
+- compile 'com.google.code.gson:gson:2.8.2' :war :fedclient :manager :api
 - compile 'org.codehaus.groovy:groovy-all:${groovy.version}' :war
 - compile 'com.unboundid:unboundid-ldapsdk:2.3.8' :war
 - compile 'org.apache.ivy:ivy:2.2.0' :war
 - compile 'com.toedter:jcalendar:1.3.2' :authority
-- compile 'org.apache.commons:commons-compress:1.4.1' :war
+- compile 'org.apache.commons:commons-compress:1.15' :war
 - compile 'commons-io:commons-io:2.2' :war
 - compile 'com.force.api:force-partner-api:24.0.0' :war
 - compile 'org.freemarker:freemarker:2.3.22' :war
 - compile 'com.github.dblock.waffle:waffle-jna:1.7.3' :war
 - compile 'org.kohsuke:libpam4j:1.8' :war
 - compile 'args4j:args4j:2.0.29' :war :fedclient
-- compile 'commons-codec:commons-codec:1.7' :war
+- compile 'commons-codec:commons-codec:1.9' :war
 - compile 'redis.clients:jedis:2.6.2' :war
 - compile 'ro.fortsoft.pf4j:pf4j:0.9.0' :war
 - compile 'org.apache.tika:tika-core:1.5' :war

gitblit.iml

@@ -583,24 +583,35 @@
       </library>
     </orderEntry>
     <orderEntry type="module-library">
-      <library name="org.eclipse.jgit-4.6.1.201703071140-r.jar">
+      <library name="org.eclipse.jgit-4.11.9.201909030838-r.jar">
         <CLASSES>
-          <root url="jar://$MODULE_DIR$/ext/org.eclipse.jgit-4.6.1.201703071140-r.jar!/" />
+          <root url="jar://$MODULE_DIR$/ext/org.eclipse.jgit-4.11.9.201909030838-r.jar!/" />
         </CLASSES>
         <JAVADOC />
         <SOURCES>
-          <root url="jar://$MODULE_DIR$/ext/src/org.eclipse.jgit-4.6.1.201703071140-r.jar!/" />
+          <root url="jar://$MODULE_DIR$/ext/src/org.eclipse.jgit-4.11.9.201909030838-r.jar!/" />
         </SOURCES>
       </library>
     </orderEntry>
     <orderEntry type="module-library">
-      <library name="jsch-0.1.53.jar">
+      <library name="jsch-0.1.54.jar">
         <CLASSES>
-          <root url="jar://$MODULE_DIR$/ext/jsch-0.1.53.jar!/" />
+          <root url="jar://$MODULE_DIR$/ext/jsch-0.1.54.jar!/" />
         </CLASSES>
         <JAVADOC />
         <SOURCES>
-          <root url="jar://$MODULE_DIR$/ext/src/jsch-0.1.53.jar!/" />
+          <root url="jar://$MODULE_DIR$/ext/src/jsch-0.1.54.jar!/" />
+        </SOURCES>
+      </library>
+    </orderEntry>
+    <orderEntry type="module-library">
+      <library name="jzlib-1.1.1.jar">
+        <CLASSES>
+          <root url="jar://$MODULE_DIR$/ext/jzlib-1.1.1.jar!/" />
+        </CLASSES>
+        <JAVADOC />
+        <SOURCES>
+          <root url="jar://$MODULE_DIR$/ext/src/jzlib-1.1.1.jar!/" />
         </SOURCES>
       </library>
     </orderEntry>
@@ -616,57 +627,57 @@
       </library>
     </orderEntry>
     <orderEntry type="module-library">
-      <library name="httpclient-4.3.6.jar">
+      <library name="httpclient-4.5.2.jar">
         <CLASSES>
-          <root url="jar://$MODULE_DIR$/ext/httpclient-4.3.6.jar!/" />
+          <root url="jar://$MODULE_DIR$/ext/httpclient-4.5.2.jar!/" />
         </CLASSES>
         <JAVADOC />
         <SOURCES>
-          <root url="jar://$MODULE_DIR$/ext/src/httpclient-4.3.6.jar!/" />
+          <root url="jar://$MODULE_DIR$/ext/src/httpclient-4.5.2.jar!/" />
         </SOURCES>
       </library>
     </orderEntry>
     <orderEntry type="module-library">
-      <library name="httpcore-4.3.3.jar">
+      <library name="httpcore-4.4.4.jar">
         <CLASSES>
-          <root url="jar://$MODULE_DIR$/ext/httpcore-4.3.3.jar!/" />
+          <root url="jar://$MODULE_DIR$/ext/httpcore-4.4.4.jar!/" />
         </CLASSES>
         <JAVADOC />
         <SOURCES>
-          <root url="jar://$MODULE_DIR$/ext/src/httpcore-4.3.3.jar!/" />
+          <root url="jar://$MODULE_DIR$/ext/src/httpcore-4.4.4.jar!/" />
         </SOURCES>
       </library>
     </orderEntry>
     <orderEntry type="module-library">
-      <library name="commons-logging-1.1.3.jar">
+      <library name="commons-logging-1.2.jar">
         <CLASSES>
-          <root url="jar://$MODULE_DIR$/ext/commons-logging-1.1.3.jar!/" />
+          <root url="jar://$MODULE_DIR$/ext/commons-logging-1.2.jar!/" />
         </CLASSES>
         <JAVADOC />
         <SOURCES>
-          <root url="jar://$MODULE_DIR$/ext/src/commons-logging-1.1.3.jar!/" />
+          <root url="jar://$MODULE_DIR$/ext/src/commons-logging-1.2.jar!/" />
         </SOURCES>
       </library>
     </orderEntry>
     <orderEntry type="module-library">
-      <library name="commons-codec-1.7.jar">
+      <library name="commons-codec-1.9.jar">
         <CLASSES>
-          <root url="jar://$MODULE_DIR$/ext/commons-codec-1.7.jar!/" />
+          <root url="jar://$MODULE_DIR$/ext/commons-codec-1.9.jar!/" />
         </CLASSES>
         <JAVADOC />
         <SOURCES>
-          <root url="jar://$MODULE_DIR$/ext/src/commons-codec-1.7.jar!/" />
+          <root url="jar://$MODULE_DIR$/ext/src/commons-codec-1.9.jar!/" />
         </SOURCES>
       </library>
     </orderEntry>
     <orderEntry type="module-library">
-      <library name="org.eclipse.jgit.http.server-4.6.1.201703071140-r.jar">
+      <library name="org.eclipse.jgit.http.server-4.11.9.201909030838-r.jar">
         <CLASSES>
-          <root url="jar://$MODULE_DIR$/ext/org.eclipse.jgit.http.server-4.6.1.201703071140-r.jar!/" />
+          <root url="jar://$MODULE_DIR$/ext/org.eclipse.jgit.http.server-4.11.9.201909030838-r.jar!/" />
         </CLASSES>
         <JAVADOC />
         <SOURCES>
-          <root url="jar://$MODULE_DIR$/ext/src/org.eclipse.jgit.http.server-4.6.1.201703071140-r.jar!/" />
+          <root url="jar://$MODULE_DIR$/ext/src/org.eclipse.jgit.http.server-4.11.9.201909030838-r.jar!/" />
         </SOURCES>
       </library>
     </orderEntry>
@@ -770,13 +781,13 @@
       </library>
     </orderEntry>
     <orderEntry type="module-library">
-      <library name="gson-2.3.1.jar">
+      <library name="gson-2.8.2.jar">
         <CLASSES>
-          <root url="jar://$MODULE_DIR$/ext/gson-2.3.1.jar!/" />
+          <root url="jar://$MODULE_DIR$/ext/gson-2.8.2.jar!/" />
         </CLASSES>
         <JAVADOC />
         <SOURCES>
-          <root url="jar://$MODULE_DIR$/ext/src/gson-2.3.1.jar!/" />
+          <root url="jar://$MODULE_DIR$/ext/src/gson-2.8.2.jar!/" />
         </SOURCES>
       </library>
     </orderEntry>
@@ -823,24 +834,24 @@
       </library>
     </orderEntry>
     <orderEntry type="module-library">
-      <library name="commons-compress-1.4.1.jar">
+      <library name="commons-compress-1.15.jar">
         <CLASSES>
-          <root url="jar://$MODULE_DIR$/ext/commons-compress-1.4.1.jar!/" />
+          <root url="jar://$MODULE_DIR$/ext/commons-compress-1.15.jar!/" />
         </CLASSES>
         <JAVADOC />
         <SOURCES>
-          <root url="jar://$MODULE_DIR$/ext/src/commons-compress-1.4.1.jar!/" />
+          <root url="jar://$MODULE_DIR$/ext/src/commons-compress-1.15.jar!/" />
         </SOURCES>
       </library>
     </orderEntry>
     <orderEntry type="module-library">
-      <library name="xz-1.0.jar">
+      <library name="objenesis-2.6.jar">
         <CLASSES>
-          <root url="jar://$MODULE_DIR$/ext/xz-1.0.jar!/" />
+          <root url="jar://$MODULE_DIR$/ext/objenesis-2.6.jar!/" />
         </CLASSES>
         <JAVADOC />
         <SOURCES>
-          <root url="jar://$MODULE_DIR$/ext/src/xz-1.0.jar!/" />
+          <root url="jar://$MODULE_DIR$/ext/src/objenesis-2.6.jar!/" />
         </SOURCES>
       </library>
     </orderEntry>
@@ -1174,17 +1185,6 @@
         </SOURCES>
       </library>
     </orderEntry>
-    <orderEntry type="module-library" scope="TEST">
-      <library name="objenesis-2.6.jar">
-        <CLASSES>
-          <root url="jar://$MODULE_DIR$/ext/objenesis-2.6.jar!/" />
-        </CLASSES>
-        <JAVADOC />
-        <SOURCES>
-          <root url="jar://$MODULE_DIR$/ext/src/objenesis-2.6.jar!/" />
-        </SOURCES>
-      </library>
-    </orderEntry>
     <orderEntry type="inheritedJdk" />
   </component>
 </module>

src/main/bugtraq/com/syntevo/bugtraq/BugtraqConfig.java

@@ -31,6 +31,9 @@ package com.syntevo.bugtraq;
 
 import java.io.File;
 import java.io.IOException;
+import java.nio.charset.Charset;
+import java.nio.charset.IllegalCharsetNameException;
+import java.nio.charset.UnsupportedCharsetException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashSet;
@@ -55,6 +58,8 @@ import org.eclipse.jgit.treewalk.filter.PathFilterGroup;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
+
 public final class BugtraqConfig {
 
 	// Constants ==============================================================
@@ -229,7 +234,7 @@ public final class BugtraqConfig {
 					FileMode entmode = tw.getFileMode(0);
 					if (FileMode.REGULAR_FILE == entmode) {
 						ObjectLoader ldr = repository.open(entid, Constants.OBJ_BLOB);
-						content = new String(ldr.getCachedBytes(), commit.getEncoding());
+						content = new String(ldr.getCachedBytes(), guessEncoding(commit));
 						break;
 					}
 				}
@@ -265,6 +270,15 @@ public final class BugtraqConfig {
 		return baseConfig;
 	}
 
+	@NotNull
+	private static Charset guessEncoding(RevCommit commit) {
+		try {
+			return commit.getEncoding();
+		} catch (IllegalCharsetNameException | UnsupportedCharsetException e) {
+			return UTF_8;
+		}
+	}
+
 	@Nullable
 	private static String getString(@Nullable String subsection, @NotNull String key, @NotNull Config config, @Nullable Config baseConfig) {
 		final String value = config.getString(BUGTRAQ, subsection, key);

src/main/java/com/gitblit/manager/RepositoryManager.java

@@ -21,7 +21,6 @@ import java.io.IOException;
 import java.lang.reflect.Field;
 import java.net.URI;
 import java.net.URISyntaxException;
-import java.nio.charset.Charset;
 import java.text.MessageFormat;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
@@ -53,7 +52,6 @@ import org.eclipse.jgit.storage.file.FileBasedConfig;
 import org.eclipse.jgit.storage.file.WindowCacheConfig;
 import org.eclipse.jgit.util.FS;
 import org.eclipse.jgit.util.FileUtils;
-import org.eclipse.jgit.util.RawParseUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -1944,20 +1942,6 @@ public class RepositoryManager implements IRepositoryManager {
 		} catch (IllegalArgumentException e) {
 			logger.error("Failed to configure JGit parameters!", e);
 		}
-
-		try {
-			// issue-486/ticket-151: UTF-9 & UTF-18
-			// issue-560/ticket-237: 'UTF8'
-			Field field = RawParseUtils.class.getDeclaredField("encodingAliases");
-			field.setAccessible(true);
-			Map<String, Charset> encodingAliases = (Map<String, Charset>) field.get(null);
-			encodingAliases.put("'utf8'", RawParseUtils.UTF8_CHARSET);
-			encodingAliases.put("utf-9", RawParseUtils.UTF8_CHARSET);
-			encodingAliases.put("utf-18", RawParseUtils.UTF8_CHARSET);
-			logger.info("Alias 'UTF8', UTF-9 & UTF-18 encodings as UTF-8 in JGit");
-		} catch (Throwable t) {
-			logger.error("Failed to inject UTF-9 & UTF-18 encoding aliases into JGit", t);
-		}
 	}
 
 	protected void configureCommitCache() {

src/main/java/com/gitblit/utils/BugtraqProcessor.java

@@ -155,10 +155,10 @@ public class BugtraqProcessor {
 				formatter.formatLogMessage(text, new BugtraqOutputHandler(sb));
 				text = sb.toString();
 			}
-		} catch (IOException e) {
-			logger.error(MessageFormat.format("Bugtraq config for {0} is invalid!", repositoryName), e);
 		} catch (ConfigInvalidException e) {
-			logger.error(MessageFormat.format("Bugtraq config for {0} is invalid!", repositoryName), e);
+			logger.warn("Bugtraq config for {} is invalid!", repositoryName, e);
+		} catch (Exception e) {
+			logger.warn("Failed to parse message through Bugtraq.", e);
 		}
 
 		return text;