Enhance RepeatRule to report number of failures at the end

In order to enable counting how frequently a test fails if repeated add
option abortOnFailure. If it is true the test aborts on the first
failure. Otherwise it runs the configured number of repetitions and, if
there was any failure, throws a RepeatException reporting how many of
the test repetitions failed.

Change-Id: Ic47de44d4a6273fddf04b9993ad989903efb40c3
Signed-off-by: Matthias Sohn <matthias.sohn@sap.com>

2 files changed, 34 insertions, 3 deletions

diff --git a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/Repeat.java b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/Repeat.java
index 08220ce245..94df554aec 100644
--- a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/Repeat.java
+++ b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/Repeat.java
@@ -56,4 +56,13 @@ public @interface Repeat {
 	 * Number of repetitions
 	 */
 	public abstract int n();
+
+	/**
+	 * Whether to abort execution on first test failure
+	 *
+	 * @return {@code true} if execution should be aborted on the first failure,
+	 *         otherwise count failures and continue execution
+	 * @since 5.1.9
+	 */
+	public boolean abortOnFailure() default true;
 }
diff --git a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/RepeatRule.java b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/RepeatRule.java
index 8165738ed8..a76f348536 100644
--- a/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/RepeatRule.java
+++ b/org.eclipse.jgit.junit/src/org/eclipse/jgit/junit/RepeatRule.java
@@ -84,6 +84,10 @@ public class RepeatRule implements TestRule {
 	public static class RepeatedTestException extends RuntimeException {
 		private static final long serialVersionUID = 1L;
 
+		public RepeatedTestException(String message) {
+			super(message);
+		}
+
 		public RepeatedTestException(String message, Throwable cause) {
 			super(message, cause);
 		}
@@ -93,28 +97,45 @@ public class RepeatRule implements TestRule {
 
 		private final int repetitions;
 
+		private boolean abortOnFailure;
+
 		private final Statement statement;
 
-		private RepeatStatement(int repetitions, Statement statement) {
+		private RepeatStatement(int repetitions, boolean abortOnFailure,
+				Statement statement) {
 			this.repetitions = repetitions;
+			this.abortOnFailure = abortOnFailure;
 			this.statement = statement;
 		}
 
 		@Override
 		public void evaluate() throws Throwable {
+			int failures = 0;
 			for (int i = 0; i < repetitions; i++) {
 				try {
 					statement.evaluate();
 				} catch (Throwable e) {
+					failures += 1;
 					RepeatedTestException ex = new RepeatedTestException(
 							MessageFormat.format(
 									"Repeated test failed when run for the {0}. time",
 									Integer.valueOf(i + 1)),
 							e);
 					LOG.log(Level.SEVERE, ex.getMessage(), ex);
-					throw ex;
+					if (abortOnFailure) {
+						throw ex;
+					}
 				}
 			}
+			if (failures > 0) {
+				RepeatedTestException e = new RepeatedTestException(
+						MessageFormat.format(
+								"Test failed {0} times out of {1} repeated executions",
+								Integer.valueOf(failures),
+								Integer.valueOf(repetitions)));
+				LOG.log(Level.SEVERE, e.getMessage(), e);
+				throw e;
+			}
 		}
 	}
 
@@ -125,7 +146,8 @@ public class RepeatRule implements TestRule {
 		Repeat repeat = description.getAnnotation(Repeat.class);
 		if (repeat != null) {
 			int n = repeat.n();
-			result = new RepeatStatement(n, statement);
+			boolean abortOnFailure = repeat.abortOnFailure();
+			result = new RepeatStatement(n, abortOnFailure, statement);
 		}
 		return result;
 	}