running negative harness tests, to verify that expected-fails actually fail

3 files changed, 26 insertions, 7 deletions

diff --git a/testing-drivers/testsrc/org/aspectj/testing/drivers/AjcHarnessTestsUsingJUnit.java b/testing-drivers/testsrc/org/aspectj/testing/drivers/AjcHarnessTestsUsingJUnit.java
index 009f266ab..d91419c15 100644
--- a/testing-drivers/testsrc/org/aspectj/testing/drivers/AjcHarnessTestsUsingJUnit.java
+++ b/testing-drivers/testsrc/org/aspectj/testing/drivers/AjcHarnessTestsUsingJUnit.java
@@ -29,9 +29,7 @@ public class AjcHarnessTestsUsingJUnit extends TestCase {
         result.addTest(
             HarnessJUnitUtil.suite("harness", 
                 new String[] {"../tests/ajcHarnessTests.xml"},
-                new String[][] { 
-                    new String[] {"-ajctestSkipKeywords=expect-fail"}
-                    }
+                null
                 ));
         result.addTest(
             HarnessJUnitUtil.suite("harness selection tests", 
diff --git a/testing-drivers/testsrc/org/aspectj/testing/drivers/AjctestsAdapter.java b/testing-drivers/testsrc/org/aspectj/testing/drivers/AjctestsAdapter.java
index fb221a246..dca57adf3 100644
--- a/testing-drivers/testsrc/org/aspectj/testing/drivers/AjctestsAdapter.java
+++ b/testing-drivers/testsrc/org/aspectj/testing/drivers/AjctestsAdapter.java
@@ -207,7 +207,8 @@ public class AjctestsAdapter extends TestSuite {
     }
 
     /** Wrap AjcTest.Spec as a TestCase. Run by delegation to suite */
-    private static class AjcTestSpecAsTest extends TestCase {
+    private static class AjcTestSpecAsTest extends TestCase 
+        implements HarnessJUnitUtil.IHasAjcSpec {
         // this could implement Test, but Ant batchtest fails to pull name 
         final String name;
         final AjcTest.Spec spec;
@@ -222,6 +223,9 @@ public class AjctestsAdapter extends TestSuite {
         public int countTestCases() {
             return 1;
         }
+        public AjcTest.Spec getAjcTestSpec() {
+            return spec;
+        }
         public void run(TestResult result) {
             if (null == suite) {
                 throw new Error("need to re-init");
diff --git a/testing-drivers/testsrc/org/aspectj/testing/drivers/HarnessJUnitUtil.java b/testing-drivers/testsrc/org/aspectj/testing/drivers/HarnessJUnitUtil.java
index 287ea53d9..84e9d7d91 100644
--- a/testing-drivers/testsrc/org/aspectj/testing/drivers/HarnessJUnitUtil.java
+++ b/testing-drivers/testsrc/org/aspectj/testing/drivers/HarnessJUnitUtil.java
@@ -91,6 +91,8 @@ public class HarnessJUnitUtil {
      * FAIL is a failure,
      * ERROR and ABORT are errors,
      * and INFO, WARNING, and DEBUG are ignored.
+     * If test instanceof IHasAjcSpec, and the keywords contain "expect-fail",
+     * then failures are not reported (but passes are reported as a failure).
      * @param result the TestResult sink
      * @param status the IRunStatus source
      * @param test the Test to associate with the results
@@ -102,12 +104,24 @@ public class HarnessJUnitUtil {
             IRunStatus status,
             Test test,
             int numIncomplete) {
-      if (!status.runResult()) {
-          if (status.hasAnyMessage(IMessage.FAIL, false)) {
+      boolean expectFail = false;
+      if (test instanceof IHasAjcSpec) {
+          AjcTest.Spec spec = ((IHasAjcSpec) test).getAjcTestSpec();
+          expectFail = spec.getKeywordsList().contains("expect-fail");
+      }
+      if (status.runResult()) {
+          if (expectFail) {
+              String m = "did not fail as expected per expect-fail keyword";
+              AssertionFailedError failure = new AssertionFailedError(m);
+              result.addFailure(test, failure);
+          }
+      } else if (!expectFail) {
+          final boolean includeChildren = true;
+          if (status.hasAnyMessage(IMessage.FAIL, false, includeChildren)) {
               String m = render(status, null);
               AssertionFailedError failure = new AssertionFailedError(m);
               result.addFailure(test, failure);
-          } else if (status.hasAnyMessage(IMessage.ERROR, true)) {
+          } else if (status.hasAnyMessage(IMessage.ERROR, true, includeChildren)) {
               String m = render(status, null);
               AssertionFailedError failure = new AssertionFailedError(m);
               result.addError(test, failure);
@@ -265,5 +279,8 @@ public class HarnessJUnitUtil {
             return children;
         }
     }
+    public interface IHasAjcSpec {
+        AjcTest.Spec getAjcTestSpec();
+    }
 }