mavenizing testing-drivers - wip

7 files changed, 1276 insertions, 0 deletions

diff --git a/testing-drivers/src/test/java/org/aspectj/testing/TestingDriversModuleTests.java b/testing-drivers/src/test/java/org/aspectj/testing/TestingDriversModuleTests.java
new file mode 100644
index 000000000..8cb8ddf30
--- /dev/null
+++ b/testing-drivers/src/test/java/org/aspectj/testing/TestingDriversModuleTests.java
@@ -0,0 +1,36 @@
+package org.aspectj.testing;
+/* *******************************************************************
+ * Copyright (c) 1999-2001 Xerox Corporation, 
+ *               2002 Palo Alto Research Center, Incorporated (PARC).
+ * All rights reserved. 
+ * This program and the accompanying materials are made available 
+ * under the terms of the Eclipse Public License v1.0 
+ * which accompanies this distribution and is available at 
+ * http://www.eclipse.org/legal/epl-v10.html 
+ *  
+ * Contributors: 
+ *     Xerox/PARC     initial implementation 
+ * ******************************************************************/
+
+
+// default package
+
+import org.aspectj.testing.drivers.DriversTests;
+
+import junit.framework.*;
+import junit.framework.Test;
+
+/**
+ * TODO weaver dumping ajcore files when using this interface
+ */
+public class TestingDriversModuleTests extends TestCase {
+
+    public static Test suite() { 
+        TestSuite suite = new TestSuite(TestingDriversModuleTests.class.getName());
+        suite.addTest(DriversTests.suite()); 
+        return suite;
+    }
+
+    public TestingDriversModuleTests(String name) { super(name); }
+
+}  
diff --git a/testing-drivers/src/test/java/org/aspectj/testing/drivers/AjcHarnessTestsUsingJUnit.java b/testing-drivers/src/test/java/org/aspectj/testing/drivers/AjcHarnessTestsUsingJUnit.java
new file mode 100644
index 000000000..efaae5c25
--- /dev/null
+++ b/testing-drivers/src/test/java/org/aspectj/testing/drivers/AjcHarnessTestsUsingJUnit.java
@@ -0,0 +1,60 @@
+/* *******************************************************************
+ * Copyright (c) 2003 Contributors
+ * All rights reserved. 
+ * This program and the accompanying materials are made available 
+ * under the terms of the Eclipse Public License v1.0 
+ * which accompanies this distribution and is available at 
+ * http://www.eclipse.org/legal/epl-v10.html 
+ *  
+ * Contributors: 
+ *     Wes Isberg     initial implementation
+ * ******************************************************************/
+
+package org.aspectj.testing.drivers;
+
+import org.aspectj.ajdt.internal.core.builder.AjState;
+
+import junit.framework.TestCase;
+import junit.framework.TestSuite;
+
+/*
+ * Run harness tests as JUnit test suites.
+ */
+public class AjcHarnessTestsUsingJUnit extends TestCase {
+
+    /**
+     * Create TestSuite with harness validation tests.
+     * @return Test with all TestSuites and TestCases
+     *         for the harness itself.
+     */
+    public static TestSuite suite() {
+        TestSuite result = HarnessJUnitUtil.suite(null, null, null);
+        result.addTest(
+            HarnessJUnitUtil.suite("harness", 
+                new String[] {"../tests/ajcHarnessTests.xml"},
+                null
+                ));
+        result.addTest(
+            HarnessJUnitUtil.suite("harness selection tests", 
+                new String[] {"testdata/incremental/harness/selectionTest.xml"},
+                null
+                ));
+        return result;
+    }
+        
+	public AjcHarnessTestsUsingJUnit(String name) {
+		super(name);
+	}
+	
+	@Override
+	protected void setUp() throws Exception {
+		super.setUp();
+		AjState.FORCE_INCREMENTAL_DURING_TESTING = true;
+	}
+	
+	@Override
+	protected void tearDown() throws Exception {
+		super.tearDown();
+		AjState.FORCE_INCREMENTAL_DURING_TESTING = false;
+	}
+}
diff --git a/testing-drivers/src/test/java/org/aspectj/testing/drivers/AjcTestsUsingJUnit.java b/testing-drivers/src/test/java/org/aspectj/testing/drivers/AjcTestsUsingJUnit.java
new file mode 100644
index 000000000..423404ee1
--- /dev/null
+++ b/testing-drivers/src/test/java/org/aspectj/testing/drivers/AjcTestsUsingJUnit.java
@@ -0,0 +1,48 @@
+/* *******************************************************************
+ * Copyright (c) 2003 Contributors
+ * All rights reserved. 
+ * This program and the accompanying materials are made available 
+ * under the terms of the Eclipse Public License v1.0 
+ * which accompanies this distribution and is available at 
+ * http://www.eclipse.org/legal/epl-v10.html 
+ *  
+ * Contributors: 
+ *     Wes Isberg     initial implementation 
+ * ******************************************************************/
+
+package org.aspectj.testing.drivers;
+
+import junit.framework.*;
+
+/**
+ * Run ajc tests as JUnit test suites.
+ * This class is named to avoid automatic inclusion in 
+ * most JUnit test runs.
+ */
+public class AjcTestsUsingJUnit extends TestCase {
+    private static final String[] SUITES = new String[]
+        { "../tests/ajcTestsFailing.xml",
+         "../tests/ajcTests.xml"
+        };
+
+    private static final String SKIPS = 
+        "-ajctestSkipKeywords=purejava,knownLimitation";
+    private static final String[][] OPTIONS = new String[][]
+        { new String[] { SKIPS },
+          new String[] { SKIPS, "-emacssym" }
+        };
+     
+    /**
+     * Create TestSuite with all SUITES running all OPTIONS.
+     * @return Test with all TestSuites and TestCases
+     *         specified in SUITES and OPTIONS.
+     */
+    public static Test suite() {
+        String name = AjcTestsUsingJUnit.class.getName();
+        return HarnessJUnitUtil.suite(name, SUITES, OPTIONS);
+    }
+        
+	public AjcTestsUsingJUnit(String name) {
+		super(name);
+	}
+}
diff --git a/testing-drivers/src/test/java/org/aspectj/testing/drivers/AjctestsAdapter.java b/testing-drivers/src/test/java/org/aspectj/testing/drivers/AjctestsAdapter.java
new file mode 100644
index 000000000..05efb60e6
--- /dev/null
+++ b/testing-drivers/src/test/java/org/aspectj/testing/drivers/AjctestsAdapter.java
@@ -0,0 +1,382 @@
+/* *******************************************************************
+ * Copyright (c) 2003 Contributors
+ * All rights reserved. 
+ * This program and the accompanying materials are made available 
+ * under the terms of the Eclipse Public License v1.0 
+ * which accompanies this distribution and is available at 
+ * http://www.eclipse.org/legal/epl-v10.html 
+ *  
+ * Contributors: 
+ *     Wes Isberg     initial implementation
+ * ******************************************************************/
+
+package org.aspectj.testing.drivers;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import junit.framework.Test;
+import junit.framework.TestCase;
+import junit.framework.TestResult;
+import junit.framework.TestSuite;
+
+import org.aspectj.ajdt.internal.core.builder.AjState;
+import org.aspectj.bridge.IMessage;
+import org.aspectj.bridge.IMessageHolder;
+import org.aspectj.bridge.MessageHandler;
+import org.aspectj.bridge.MessageUtil;
+import org.aspectj.testing.harness.bridge.AjcTest;
+import org.aspectj.testing.harness.bridge.RunSpecIterator;
+import org.aspectj.testing.harness.bridge.Sandbox;
+import org.aspectj.testing.harness.bridge.Validator;
+import org.aspectj.testing.run.IRunIterator;
+import org.aspectj.testing.run.RunStatus;
+import org.aspectj.testing.run.Runner;
+
+/*
+ * Adapt Harness tests to JUnit driver. This renders suite files as TestSuite
+ * and AjcTest as TestCase. When run, aborts are reported as error and fails as
+ * failures, with all messages stuffed into the one JUnit exception message.
+ * Test options are supported, but no harness options. The TestSuite
+ * implementation prevents us from re-running tests. In the Eclipse JUnit test
+ * runner, the tests display hierarchically and with annotations and with
+ * messages. You can stop the tests, but not traverse to the source or re-run
+ * the test.
+ */
+public class AjctestsAdapter extends TestSuite {
+    public static final String VERBOSE_NAME = AjctestsAdapter.class.getName()
+            + ".VERBOSE";
+
+    private static final boolean VERBOSE = HarnessJUnitUtil
+            .readBooleanSystemProperty(VERBOSE_NAME);
+
+    /**
+     * Factory to make and populate suite without options.
+     * 
+     * @param suitePath
+     *            the String path to a harness suite file
+     * @return AjctestJUnitSuite populated with tests
+     */
+    public static AjctestsAdapter make(String suitePath) {
+        return make(suitePath, null);
+    }
+
+    /**
+     * Factory to make and populate suite
+     * 
+     * @param suitePath
+     *            the String path to a harness suite file
+     * @param options
+     *            the String[] options to use when creating tests
+     * @return AjctestJUnitSuite populated with tests
+     */
+    public static AjctestsAdapter make(String suitePath, String[] options) {
+        AjctestsAdapter result = new AjctestsAdapter(suitePath, options);
+        AjcTest.Spec[] tests = AjcTest.Suite.getTests(result.getSpec());
+        if (VERBOSE) {
+            log("loading " + tests.length + " tests in " + suitePath);
+        }
+        for (int i = 0; i < tests.length; i++) {
+            AjcTest.Spec ajcTest = tests[i];
+            result.addTest(new AjcTestSpecAsTest(ajcTest, result));
+        }
+        return result;
+    }
+    
+    private static void log(String message) {
+        System.err.println(message);
+        System.err.flush();
+    }
+
+    private final String suitePath;
+
+    private final String[] options;
+
+    private AjcTest.Suite.Spec spec;
+
+    private Runner runner;
+
+    private Validator validator;
+
+    private IMessageHolder holder;
+
+    private Sandbox sandbox;
+
+    private File suiteDir;
+
+    private String name;
+
+    private AjctestsAdapter(String suitePath, String[] options) {
+        this.suitePath = suitePath;
+        String[] opts = new String[0];
+        if (!HarnessJUnitUtil.isEmpty(options)) {
+            opts = new String[options.length];
+            System.arraycopy(options, 0, opts, 0, opts.length);
+        }
+        this.options = opts;
+    }
+
+    public void addTest(Test test) {
+        if (!(test instanceof AjcTestSpecAsTest)) {
+            String m = "expecting AjcTestSpecAsTest, got "
+                    + (null == test ? "null test" : test.getClass().getName()
+                            + ": " + test);
+            throw new IllegalArgumentException(m);
+        }
+        super.addTest(test);
+    }
+
+    public void addTestSuite(Class testClass) {
+        throw new Error("unimplemented");
+    }
+
+    public String getName() {
+        if (null == name) {
+            name = HarnessJUnitUtil.cleanTestName(suitePath
+                    + Arrays.asList(options));
+        }
+        return name;
+    }
+
+    /**
+     * Callback from test to run it using suite-wide holder, etc. The caller is
+     * responsible for calling result.startTest(test) and result.endTest(test);
+     * 
+     * @param test
+     *            the AjcTestSpecAsTest to run
+     * @param result
+     *            the TestResult for any result messages (may be null)
+     */
+    protected void runTest(AjcTestSpecAsTest test, TestResult result) {
+        final Runner runner = getRunner();
+        final Sandbox sandbox = getSandbox();
+        final Validator validator = getValidator();
+        int numIncomplete = 0;
+        final RunStatus status = new RunStatus(new MessageHandler(), runner);
+        status.setIdentifier(test.toString());
+        try {
+            IMessageHolder holder = getHolder();
+            holder.clearMessages();
+            IRunIterator steps = test.spec.makeRunIterator(sandbox, validator);
+            if (0 < holder.numMessages(IMessage.ERROR, true)) {
+                MessageUtil.handleAll(status, holder, IMessage.INFO, true,
+                        false);
+            } else {
+                runner.runIterator(steps, status, null);
+            }
+            if (steps instanceof RunSpecIterator) {
+                numIncomplete = ((RunSpecIterator) steps).getNumIncomplete();
+            }
+        } finally {
+            try {
+                // reportResult handles null TestResult
+                HarnessJUnitUtil
+                        .reportResult(null, status, test, numIncomplete);
+            } finally {
+                validator.deleteTempFiles(true);
+            }
+        }
+    }
+
+    private File getSuiteDir() {
+        if (null == suiteDir) {
+            File file = new File(suitePath);
+            file = file.getParentFile();
+            if (null == file) {
+                file = new File(".");
+            }
+            suiteDir = file;
+        }
+        return suiteDir;
+    }
+
+    private Validator getValidator() {
+        if (null == validator) {
+            validator = new Validator(getHolder());
+            // XXX lock if keepTemp?
+        }
+        return validator;
+    }
+
+    private Runner getRunner() {
+        if (null == runner) {
+            runner = new Runner();
+        }
+        return runner;
+    }
+
+    private IMessageHolder getHolder() {
+        if (null == holder) {
+            holder = new MessageHandler();
+        }
+        return holder;
+    }
+
+    private AjcTest.Suite.Spec getSpec() {
+        if (null == spec) {
+            IMessageHolder holder = getHolder();
+            spec = HarnessJUnitUtil.getSuiteSpec(suitePath, options,
+                    getHolder());
+            if (VERBOSE && holder.hasAnyMessage(null, true)) {
+                MessageUtil.print(System.err, holder, "skip ",
+                        MessageUtil.MESSAGE_MOST);
+            }
+            holder.clearMessages();
+        }
+        return spec;
+    }
+
+    private Sandbox getSandbox() {
+        if (null == sandbox) {
+            sandbox = new Sandbox(spec.getSuiteDirFile(), getValidator());
+        }
+        return sandbox;
+    }
+
+    /**
+     * Wrap AjcTest.Spec for lookup by description
+     * 
+     * @author wes
+     */
+    public static class SpecTests {
+        private static final HashMap TESTS = new HashMap();
+
+//        private static void putSpecTestsFor(String id, SpecTests tests) {
+//            TESTS.put(id, tests);
+//        }
+
+        private static SpecTests getSpecTestsFor(String id) {
+            SpecTests result = (SpecTests) TESTS.get(id);
+            if (null == result) {
+                throw new Error("no tests found for " + id);
+            }
+            return result;
+        }
+        
+        // ------------------------------------
+        final AjctestsAdapter mAjctestsAdapter;
+
+        private final Map mDescriptionToAjcTestSpec;
+
+        // ------------------------------------
+        private SpecTests(AjctestsAdapter ajctestsAdapter, AjcTest.Spec[] tests) {
+            mAjctestsAdapter = ajctestsAdapter;
+            Map map = new HashMap();
+            for (int i = 0; i < tests.length; i++) {
+                map.put(tests[i].getDescription(), tests[i]);
+            }
+            
+            mDescriptionToAjcTestSpec = Collections.unmodifiableMap(map);
+        }
+
+        /**
+         * @param description
+         *            the String description of the test
+         * @throws IllegalArgumentException
+         *             if testName is not found
+         */
+        protected void runTest(String description) {
+            AjcTest.Spec spec = getSpec(description);
+            AjctestsAdapter.AjcTestSpecAsTest ajcTestAsSpec = new AjctestsAdapter.AjcTestSpecAsTest(
+                    spec, mAjctestsAdapter);
+            // runTest handles null TestResult
+            mAjctestsAdapter.runTest(ajcTestAsSpec, null);
+        }
+
+        /**
+         * @param description
+         *            the String description of the test
+         * @throws IllegalArgumentException
+         *             if testName is not found
+         */
+        private AjcTest.Spec getSpec(String description) {
+            AjcTest.Spec spec = (AjcTest.Spec) mDescriptionToAjcTestSpec
+                    .get(description);
+            if (null == spec) {
+                throw new IllegalArgumentException("no test for " + description);
+            }
+            return spec;
+        }
+
+        /**
+         * makeUsingTestClass(..) extends this to create TestCase with
+         * test_{name} for each test case.
+         */
+        public static class TestClass extends TestCase {
+            public TestClass() {                
+            }
+            private SpecTests mTests;
+
+            /**
+             * Called by code generated in makeUsingTestClass(..)
+             * 
+             * @param description
+             *            the String identifier of the test stored in SpecTests
+             *            mTests.
+             * @throws Error
+             *             on first and later uses if getTestsFor() returns
+             *             null.
+             */
+            public final void runTest(String description) {
+                if (null == mTests) {
+                    String classname = getClass().getName();
+                    mTests = getSpecTestsFor(classname);
+                }
+                mTests.runTest(description);
+            }
+        }
+    }
+
+    /** Wrap AjcTest.Spec as a TestCase. Run by delegation to suite */
+    private static class AjcTestSpecAsTest extends TestCase implements
+            HarnessJUnitUtil.IHasAjcSpec {
+        // this could implement Test, but Ant batchtest fails to pull name
+        final String name;
+
+        final AjcTest.Spec spec;
+
+        AjctestsAdapter suite;
+
+        AjcTestSpecAsTest(AjcTest.Spec spec, AjctestsAdapter suite) {
+            super(HarnessJUnitUtil.cleanTestName(spec.getDescription()));
+            this.name = HarnessJUnitUtil.cleanTestName(spec.getDescription());
+            this.suite = suite;
+            this.spec = spec;
+            spec.setSuiteDir(suite.getSuiteDir());
+        }
+
+        public int countTestCases() {
+            return 1;
+        }
+
+        public AjcTest.Spec getAjcTestSpec() {
+            return spec;
+        }
+
+        public void run(TestResult result) {
+            if (null == suite) {
+                throw new Error("need to re-init");
+            }
+            try {
+            	AjState.FORCE_INCREMENTAL_DURING_TESTING = true;
+                result.startTest(this);
+                suite.runTest(this, result);
+            } finally {
+                result.endTest(this);
+                suite = null;
+                AjState.FORCE_INCREMENTAL_DURING_TESTING = false;
+            }
+        }
+
+        public String getName() {
+            return name;
+        }
+
+        public String toString() {
+            return name;
+        }
+    }
+}
\ No newline at end of file
diff --git a/testing-drivers/src/test/java/org/aspectj/testing/drivers/DriversTests.java b/testing-drivers/src/test/java/org/aspectj/testing/drivers/DriversTests.java
new file mode 100644
index 000000000..da277f135
--- /dev/null
+++ b/testing-drivers/src/test/java/org/aspectj/testing/drivers/DriversTests.java
@@ -0,0 +1,35 @@
+/* *******************************************************************
+ * Copyright (c) 1999-2001 Xerox Corporation, 
+ *               2002 Palo Alto Research Center, Incorporated (PARC)
+ *               2003 Contributors.
+ * All rights reserved. 
+ * This program and the accompanying materials are made available 
+ * under the terms of the Eclipse Public License v1.0 
+ * which accompanies this distribution and is available at 
+ * http://www.eclipse.org/legal/epl-v10.html 
+ *  
+ * Contributors: 
+ *     Xerox/PARC     initial implementation 
+ *     Wes Isberg     added JUnit harness adapters
+ * ******************************************************************/
+
+
+package org.aspectj.testing.drivers;
+
+import junit.framework.*;
+
+public class DriversTests extends TestCase {
+
+    public static Test suite() { 
+        TestSuite suite = new TestSuite(DriversTests.class.getName());
+        // AjcTestsUsingJUnit takes too long to include by default
+        //$JUnit-BEGIN$
+        suite.addTestSuite(HarnessSelectionTest.class); 
+        suite.addTest(AjcHarnessTestsUsingJUnit.suite()); 
+        //$JUnit-END$
+        return suite;
+    }
+
+    public DriversTests(String name) { super(name); }
+
+}  
diff --git a/testing-drivers/src/test/java/org/aspectj/testing/drivers/HarnessJUnitUtil.java b/testing-drivers/src/test/java/org/aspectj/testing/drivers/HarnessJUnitUtil.java
new file mode 100644
index 000000000..d3e05f84c
--- /dev/null
+++ b/testing-drivers/src/test/java/org/aspectj/testing/drivers/HarnessJUnitUtil.java
@@ -0,0 +1,320 @@
+/* *******************************************************************
+ * Copyright (c) 2003 Contributors
+ * All rights reserved. 
+ * This program and the accompanying materials are made available 
+ * under the terms of the Eclipse Public License v1.0 
+ * which accompanies this distribution and is available at 
+ * http://www.eclipse.org/legal/epl-v10.html 
+ *  
+ * Contributors: 
+ *     Wes Isberg     initial implementation
+ * ******************************************************************/
+
+package org.aspectj.testing.drivers;
+
+import java.io.*;
+import java.util.*;
+
+import junit.framework.*;
+
+import org.aspectj.bridge.*;
+//import org.aspectj.bridge.MessageHandler;
+import org.aspectj.testing.harness.bridge.*;
+import org.aspectj.testing.run.IRunStatus;
+import org.aspectj.testing.util.RunUtils;
+import org.aspectj.testing.util.RunUtils.IRunStatusPrinter;
+import org.aspectj.testing.xml.AjcSpecXmlReader;
+
+/**
+ * Utilities for adapting AjcTest.{Suite.}Spec to JUnit.
+ */
+public class HarnessJUnitUtil {
+    /** bug?: eclipse RemoteTestRunner hangs if n>1 */
+    public static final boolean ONE_ERROR_PER_TEST = true;
+    public static final boolean FLATTEN_RESULTS = true;
+    public static final boolean PRINT_OTHER_MESSAGES = false;
+
+    /**
+     * Create TestSuite with all suites running all options.
+     * @param suites the String[] of paths to harness test suite files
+     * @param options the String[][] of option sets to run (may be null)
+     * @return Test with all TestSuites and TestCases
+     *         specified in suites and options.
+     */
+    public static TestSuite suite(String name, String[] suites, String[][] options) {
+        if (null == name) {
+            name = AjcHarnessTestsUsingJUnit.class.getName();
+        }
+        TestSuite suite = new TestSuite(name);
+        if (!HarnessJUnitUtil.isEmpty(suites)) {
+            if (HarnessJUnitUtil.isEmpty(options)) {
+                options = new String[][] {new String[0]};
+            }
+            for (int i = 0; i < suites.length; i++) {
+                for (int j = 0; j < options.length; j++) {
+                    Test t = AjctestsAdapter.make(suites[i], options[j]);
+                    suite.addTest(t);
+                }
+            }
+        }
+        return suite;
+    }
+
+    public static boolean isEmpty(Object[] ra) {
+        return ((null == ra) || (0 == ra.length));
+    }
+    /**
+     * Render status using a given printer.
+     * @param status the IRunStatus to render to String
+     * @param printer the IRunStatusPrinter to use
+     *        (defaults to AJC_PRINTER if null)
+     * @return the String rendering of the status,
+     *         or "((IRunStatus) null)" if null
+     */
+    public static String render(IRunStatus status, IRunStatusPrinter printer) {
+        if (null == status) {
+            return "((IRunStatus) null)";
+        }
+        if (null == printer) {
+            printer = RunUtils.AJC_PRINTER;
+        }
+        ByteArrayOutputStream outStream = new ByteArrayOutputStream();
+        PrintStream out = new PrintStream(outStream);
+        printer.printRunStatus(out, status);
+        out.flush();
+          
+        return outStream.toString();
+    }
+    
+    /**
+     * Dump results for Test from status into TestResult.
+     * FAIL is a failure,
+     * ERROR and ABORT are errors,
+     * and INFO, WARNING, and DEBUG are ignored.
+     * If test instanceof IHasAjcSpec, and the keywords contain "expect-fail",
+     * then failures are not reported (but passes are reported as a failure).
+     * @param result the TestResult sink
+     * @param status the IRunStatus source
+     * @param test the Test to associate with the results
+     * @param numIncomplete ignored
+     * @return 0 (ignored)
+     */
+    public static int reportResult(
+            TestResult result, 
+            IRunStatus status,
+            Test test,
+            int numIncomplete) {
+      boolean expectFail = false;
+      if (test instanceof IHasAjcSpec) {
+          AjcTest.Spec spec = ((IHasAjcSpec) test).getAjcTestSpec();
+          expectFail = spec.getKeywordsList().contains("expect-fail");
+      }
+      if (status.runResult()) {
+          if (expectFail) {
+              String m = "did not fail as expected per expect-fail keyword";
+              reportResultToJUnit(m, false, true, test, result);
+          }
+      } else if (!expectFail) {
+          final boolean includeChildren = true;
+          if (status.hasAnyMessage(IMessage.FAIL, false, includeChildren)) {
+              String m = render(status, null);
+              reportResultToJUnit(m, false, true, test, result);
+          } else if (status.hasAnyMessage(IMessage.ERROR, true, includeChildren)) {
+              String m = render(status, null);
+              reportResultToJUnit(m, true, false, test, result);
+          } // /XXX  skip INFO, DEBUG
+      }
+      return 0; // XXX not doing incomplete
+    }
+    
+    /**
+     * Report results as error, failure, or success (ignored),
+     * differently if result is null
+     * @param description the String description of the result
+     * @param isError if true, report as failure
+     * @param isFailure if true and not isError, report as failure
+     * @param test the Test case
+     * @param result the TestResult sink - ignored if null
+     * @return 0
+     */
+    private static int reportResultToJUnit(String description, boolean isError, boolean isFailure, Test test, TestResult result) {
+        if (null != result) {
+            if (isError) {
+                result.addError(test, new AssertionFailedError(description));
+            } else if (isFailure) {
+                result.addFailure(test, new AssertionFailedError(description));
+            } // no need to log success
+        } else { // have to throw failure
+            if (isError) {
+                String m = safeTestName(test) + " " + description;
+                throw new Error(m);
+            } else if (isFailure) {
+//                String m = safeTestName(test) + " " + description;
+                throw new AssertionFailedError(description);
+            } // no need to log success
+        }
+        return 0;
+    }
+    
+//    public static int reportResultComplex(
+//        TestResult result, 
+//        IRunStatus status,
+//        Test test,
+//        int numIncomplete) {
+//        int errs = 0;
+//        if (FLATTEN_RESULTS) {
+//            IRunStatus[] kids = status.getChildren();
+//            for (int i = 0; i < kids.length; i++) {
+//                errs += reportResult(result, kids[i], test, 0);
+//                if ((errs > 0) && ONE_ERROR_PER_TEST) {
+//                    return errs;
+//                }
+//            }
+//        }
+//            
+//        Throwable thrown = status.getThrown();
+//        if (null != thrown) { // always report this? XXX what if expected?
+//            result.addError(test, thrown);
+//            errs++;
+//        }
+//        boolean previewPass = status.runResult();
+//        IMessage[] errors = status.getMessages(null, true);
+//        for (int i = 0; ((errs == 0) || !ONE_ERROR_PER_TEST)
+//                        && i < errors.length; i++) {
+//            IMessage message = errors[i];
+//            if (message.isAbort()) {
+//                result.addError(test, new ErrorMessage(message));
+//                errs++;
+//            } else if (message.isFailed()) {
+//                result.addFailure(test, new ErrorMessage(message));
+//                errs++;
+//            } else if (PRINT_OTHER_MESSAGES || !previewPass) {
+//                System.out.println("#### message for " + test + ": ");
+//                System.out.println(message);
+//            }
+//        }
+//        if (((errs == 0) || !ONE_ERROR_PER_TEST)
+//            && ((errs == 0) != status.runResult())) {
+//            String s = "expected pass=" + (errs == 0);
+//            result.addFailure(test, new ErrorMessage(s));
+//            errs++;
+//        }
+//        if (((errs == 0) || !ONE_ERROR_PER_TEST)
+//            && !status.isCompleted()) {
+//            result.addFailure(test, new ErrorMessage("test incomplete? "));
+//            errs++;
+//        }
+//        if (((errs == 0) || !ONE_ERROR_PER_TEST)
+//            && (0 < numIncomplete)) {
+//            result.addFailure(test, new ErrorMessage("incomplete steps: " + numIncomplete));
+//            errs++;
+//        }
+//        return errs;
+//    }
+
+    /**
+     * @return TestCase.getName() or Test.toString() or "nullTest"
+     */
+    public static String safeTestName(Test test) {
+        if (test instanceof TestCase) {
+            return ((TestCase) test).getName();
+        } else if (null != test) {
+            return test.toString();
+        } else {
+            return "nullTest";
+        }
+    }
+    
+    /**
+     * Fix up test names for JUnit.
+     * (i.e., workaround eclipse JUnit bugs)
+     * @param name the String identifier for the test
+     * @return the String permitted by (Eclipse) JUnit support
+     */
+    public static String cleanTestName(String name) { 
+        name = name.replace(',', ' ');
+        name = name.replace('[', ' ');
+        name = name.replace(']', ' ');
+        name = name.replace('-', ' ');
+        return name;
+    }
+
+    public static boolean readBooleanSystemProperty(String name) {
+        boolean result = false;
+        try {
+            result = Boolean.getBoolean(name);
+        } catch (Throwable t) {
+            // ignore
+        }
+        return result;
+    }
+    
+    /**
+     * Get the test suite specifications from the suite file,
+     * apply the options to all,
+     * and report any messages to the holder.
+     * @param suitePath the String path to the harness suite file
+     * @param options the String[] options for the tests - may be null
+     * @param holder the IMessageHolder for any messages - may be null
+     * @return AjcTest.Suite.Spec test descriptions
+     *   (non-null but empty if some error)
+     */
+    public static AjcTest.Suite.Spec getSuiteSpec(
+            String suitePath, 
+            String[] options,
+            IMessageHolder holder) {
+        if (null == suitePath) {
+            MessageUtil.fail(holder, "null suitePath");
+            return EmptySuite.ME;
+        }
+        File suiteFile = new File(suitePath);
+        if (!suiteFile.canRead() || !suiteFile.isFile()) {
+            MessageUtil.fail(holder, "unable to read file " + suitePath);
+            return EmptySuite.ME;
+        }
+        try {
+            AjcTest.Suite.Spec tempSpec;
+            AbstractRunSpec.RT runtime = new AbstractRunSpec.RT();
+            tempSpec = AjcSpecXmlReader.getReader().
+                        readAjcSuite(suiteFile);
+            tempSpec.setSuiteDirFile(suiteFile.getParentFile());
+            if (null == options) {
+                options = new String[0];
+            }
+            runtime.setOptions(options);
+            boolean skip = !tempSpec.adoptParentValues(runtime, holder);                
+            if (skip) {
+                tempSpec = EmptySuite.ME;
+            }
+            return tempSpec;
+        } catch (IOException e) {
+            MessageUtil.abort(holder, "IOException", e);
+            return EmptySuite.ME;
+        }
+    }
+    
+    private static class EmptySuite extends AjcTest.Suite.Spec {
+        static final EmptySuite ME = new EmptySuite();
+        final ArrayList children;
+        private EmptySuite(){
+            children = new ArrayList() {
+                // XXX incomplete...
+                public void add(int arg0, Object arg1) { fail();}
+                public boolean addAll(int arg0, Collection arg1) { return fail();}
+                public boolean addAll(Collection o) { return fail(); }
+                public boolean add(Object o) { return fail(); }
+                public boolean remove(Object o) { return fail(); }
+                private boolean fail() {
+                    throw new Error("unmodifiable");
+                }
+            };
+        }
+        public ArrayList getChildren() {
+            return children;
+        }
+    }
+    public interface IHasAjcSpec {
+        AjcTest.Spec getAjcTestSpec();
+    }
+}
+
diff --git a/testing-drivers/src/test/java/org/aspectj/testing/drivers/HarnessSelectionTest.java b/testing-drivers/src/test/java/org/aspectj/testing/drivers/HarnessSelectionTest.java
new file mode 100644
index 000000000..ac937ac44
--- /dev/null
+++ b/testing-drivers/src/test/java/org/aspectj/testing/drivers/HarnessSelectionTest.java
@@ -0,0 +1,395 @@
+/* *******************************************************************
+ * Copyright (c) 1999-2001 Xerox Corporation, 
+ *               2002 Palo Alto Research Center, Incorporated (PARC).
+ * All rights reserved. 
+ * This program and the accompanying materials are made available 
+ * under the terms of the Eclipse Public License v1.0 
+ * which accompanies this distribution and is available at 
+ * http://www.eclipse.org/legal/epl-v10.html 
+ *  
+ * Contributors: 
+ *     Xerox/PARC     initial implementation 
+ * ******************************************************************/
+
+package org.aspectj.testing.drivers;
+
+import org.aspectj.bridge.IMessage;
+import org.aspectj.bridge.MessageHandler;
+import org.aspectj.bridge.MessageUtil;
+import org.aspectj.testing.harness.bridge.AbstractRunSpec;
+import org.aspectj.testing.harness.bridge.AjcTest;
+import org.aspectj.testing.harness.bridge.IRunSpec;
+import org.aspectj.testing.run.IRunStatus;
+import org.aspectj.testing.run.RunValidator;
+import org.aspectj.testing.util.BridgeUtil;
+import org.aspectj.testing.util.RunUtils;
+import org.aspectj.testing.xml.AjcSpecXmlReader;
+import org.aspectj.util.LangUtil;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Hashtable;
+import java.util.List;
+
+import junit.framework.TestCase;
+
+/**
+ * The harness supports custom-coded queries based on
+ * -ajctest{kind}={query} parameters
+ * (until we move to an XML database with real queries).
+ */
+public class HarnessSelectionTest extends TestCase {
+    private static final String TESTDATA = "../testing-drivers/testdata";
+    private static final String INC_HARNESS_DIR 
+        = TESTDATA + "/incremental/harness";
+    private static final String SELECT 
+        = INC_HARNESS_DIR + "/selectionTest.xml";
+        
+    /** @see testIncrementalSuite() */
+    private static final String INCREMENTAL 
+        = INC_HARNESS_DIR + "/suite.xml";
+
+    private static final String TITLE_LIST_ONE 
+        = INC_HARNESS_DIR + "/titleListOne.txt";
+
+    private static final String TITLE_LIST_PLURAL 
+        = INC_HARNESS_DIR + "/titleListPlural.txt";
+    
+    private static Hashtable SPECS = new Hashtable();    
+    
+    private static AjcTest.Suite.Spec getSpec(String suiteFile) {
+        AjcTest.Suite.Spec result = (AjcTest.Suite.Spec) SPECS.get(suiteFile);
+        if (null == result) {
+            try {
+                result = AjcSpecXmlReader.getReader().readAjcSuite(new File(suiteFile));
+                SPECS.put(suiteFile, result);
+            } catch (IOException e) {
+                e.printStackTrace(System.err);
+            }
+        }        
+        try {
+            return (AjcTest.Suite.Spec) result.clone();
+        } catch (CloneNotSupportedException e) {
+            e.printStackTrace(System.err);
+            assertTrue("clone failed: " + e, false);
+            return null; // keep compiler happy
+        }
+    }
+    
+    private boolean verbose;
+
+	public HarnessSelectionTest(String name) {
+		super(name);
+	}
+    
+    public void testFilesAvailable() {
+        String[] files = new String[] {
+            SELECT, INCREMENTAL, TITLE_LIST_ONE, TITLE_LIST_PLURAL
+        };
+        for (int i = 0; i < files.length; i++) {
+            assertTrue(files[i], new File(files[i]).canRead());
+        }
+    }
+
+    public void testIncrementalSuite() {
+        System.err.println("skipping testIncrementalSuite - too long");
+        if (true) return;
+        if (!eclipseAvailable()) {
+            System.err.println("skipping test - eclipse classes not available");
+            return;
+        }
+        String[] options = new String[] 
+            { "!verbose", "!eclipse",
+            };
+        Exp exp = new Exp(6, 6, 0, 6, 0, 0, 0);
+        checkSelection(INCREMENTAL, options, "INFIX IGNORED", exp);
+    }
+    
+    public void testKeywordSelectionBoth() {
+        if (!eclipseAvailable()) {
+            System.err.println("skipping test - eclipse classes not available");
+            return;
+        }
+        String[] options = new String[] 
+            { "-ajctestRequireKeywords=requireKeyword", 
+            "-ajctestSkipKeywords=skipKeyword,skipUnenforcedAjcLimit",
+            "!verbose",
+            "-eclipse",
+            };
+        Exp exp = new Exp(17, 1, 16, 1, 0, 0, 1);
+        checkSelection(SELECT, options, "keyword skipKeyword was found", exp);
+    }
+    
+    public void testKeywordSelectionRequire() {
+        if (!eclipseAvailable()) {
+            System.err.println("skipping test - eclipse classes not available");
+            return;
+        }
+        String[] options = new String[] 
+            { "-ajctestRequireKeywords=skipKeyword", 
+            "!verbose",
+            "-eclipse",
+            };
+        Exp exp = new Exp(17, 1, 16, 1, 0, 0, 16);
+        checkSelection(SELECT, options, "keyword skipKeyword was not found", exp);
+    }
+
+    public void testKeywordSelectionSkip() {
+        if (!eclipseAvailable()) {
+            System.err.println("skipping test - eclipse classes not available");
+            return;
+        }
+        String[] options = new String[] 
+            { "-ajctestSkipKeywords=requireKeyword", 
+            "!verbose",
+            "-eclipse",
+            };
+        Exp exp = new Exp(17, 0, 17, 0, 0, 0, 17);
+        checkSelection(SELECT, options, "keyword requireKeyword was found", exp);
+    }
+    
+    public void testNoOptions() {
+        if (!ajcAvailable()) {
+            System.err.println("skipping test - ajc classes not available");
+            return;
+        }
+        String[] options = new String[] 
+            { "!ajc"
+            };
+        Exp exp = new Exp(17, 3, 14, 3, 0, 0, 4);
+        checkSelection(SELECT, options, "normally-valid", exp);
+    }
+
+    public void testEclipseOptionsSkip() {
+        String[] options = new String[] 
+            { "-eclipse",
+                "-ajctestRequireKeywords=eclipseOptionSkip"
+            };
+        Exp exp = new Exp(17, 0, 17, 0, 0, 0, 6);
+        checkSelection(SELECT, options, "old ajc 1.0 option", exp);
+    }
+    
+    public void testAjcEclipseConflict() {
+        if (!ajcAvailable()) {
+            System.err.println("skipping test - ajc classes not available");
+            return;
+        }
+        String[] options = new String[] 
+            { "!ajc"
+            };
+        Exp exp = new Exp(17, 3, 14, 3, 0, 0, 6);
+        checkSelection(SELECT, options, "conflict between !eclipse and !ajc", exp);
+    }
+    
+    public void testEclipseConflict() {
+        String[] options = new String[] 
+            { "^eclipse",
+              "-ajctestSkipKeywords=skipUnenforcedAjcLimit"
+            };            
+        Exp exp = new Exp(17, 3, 14, 3, 0, 0, 6);
+        checkSelection(SELECT, options, "force conflict between eclipse", exp);
+    }
+    
+    public void testSinglePR() {
+        String[] options = new String[] 
+            { "-eclipse", "-ajctestPR=100"
+            };
+        Exp exp = new Exp(17, 1, 16, 1, 0, 0, 16);
+        checkSelection(SELECT, options, "bugId required", exp);
+    }
+    
+    public void testTwoPR() {
+        String[] options = new String[] 
+            { "-eclipse", "-ajctestPR=100,101"
+            };
+        Exp exp = new Exp(17, 2, 15, 2, 0, 0, 15);
+        checkSelection(SELECT, options, "bugId required", exp);
+    }
+    
+    public void testTitleContainsSubstringSelection() {
+        String[] options = new String[] 
+            { "-ajctestTitleContains=run and ",
+                "-eclipse" 
+            };
+        Exp exp = new Exp(17, 1, 16, 1, 0, 0, 16);
+        checkSelection(SELECT, options, "run and", exp);
+    }
+    
+    public void testTitleContainsSubstringSelectionPlural() {
+        String[] options = new String[] 
+            { "-ajctestTitleContains= run and , if skipKeyword ",
+                "-eclipse" 
+            };
+        Exp exp = new Exp(17, 2, 15, 2, 0, 0, 15);
+        checkSelection(SELECT, options, "title", exp);
+    }
+
+    public void testTitleContainsExactSelection() {
+        String[] options = new String[] 
+            { "-ajctestTitleContains=run and pass",
+                "-eclipse" 
+            };
+        Exp exp = new Exp(17, 1, 16, 1, 0, 0, 16);
+        checkSelection(SELECT, options, "run and pass", exp);
+    }
+    
+    public void testTitleContainsExactSelectionPlural() {
+        String[] options = new String[] 
+            { "-ajctestTitleContains= run and pass , omit if skipKeyword ",
+                "-eclipse" 
+            };
+        Exp exp = new Exp(17, 2, 15, 2, 0, 0, 15);
+        checkSelection(SELECT, options, "title", exp);
+    }
+
+    public void testTitleListSelection() {
+        String[] options = new String[] 
+            { "-ajctestTitleList=run and pass",
+                "-eclipse" 
+            };
+        Exp exp = new Exp(17, 1, 16, 1, 0, 0, 16);
+        checkSelection(SELECT, options, "run and pass", exp);
+    }
+    
+    public void testTitleListSelectionPlural() {
+        String[] options = new String[] 
+            { "-ajctestTitleList= run and pass , omit if skipKeyword ",
+                "-eclipse" 
+            };
+        Exp exp = new Exp(17, 2, 15, 2, 0, 0, 15);
+        checkSelection(SELECT, options, "title", exp);
+    }
+
+    public void testTitleListFileSelection() {
+        String[] options = new String[] 
+            { "-ajctestTitleList=" + TITLE_LIST_ONE,
+                "-eclipse"
+            };
+        Exp exp = new Exp(17, 1, 16, 1, 0, 0, 16);
+        checkSelection(SELECT, options, TITLE_LIST_ONE, exp);
+    }
+
+   public void testTitleListFileSelectionPlural() {
+       String[] options = new String[] 
+           { "-ajctestTitleList=" + TITLE_LIST_PLURAL,
+               "-eclipse"
+           };
+       Exp exp = new Exp(17, 2, 15, 2, 0, 0, 15);
+       checkSelection(SELECT, options, TITLE_LIST_PLURAL, exp);
+       
+       // Now check the "fail only" path 
+       options = new String[] 
+           { "-ajctestTitleFailList=" + TITLE_LIST_PLURAL,
+               "-eclipse"
+           };
+       // 1 messages skipped when run under 1.4 for other reasons,
+       // so count "skip" instead of TITLE_LIST_PLURAL
+       exp = new Exp(17, 2, 15, 2, 0, 0, 15);
+       checkSelection(SELECT, options, "skip", exp);
+   }
+
+    /** 
+     * Run the static test suite with the given options.
+     * @param setupHolder the IMessageHolder for any setup messages
+     * @return null if setup failed or Harness.RunResult if suite completed.
+     */
+    private Harness.RunResult runSuite(String suiteFile, String[] options, MessageHandler setupHolder) {
+        AbstractRunSpec.RT runtime = new AbstractRunSpec.RT();
+        runtime.setOptions(options);
+        AjcTest.Suite.Spec spec = getSpec(suiteFile);
+        assertNotNull(spec);
+        ArrayList<IRunSpec> kids = spec.getChildren();
+        assertNotNull(kids);
+        if ((suiteFile == SELECT) && (17 != kids.size())) {
+            assertTrue("expected 17 kids, got " + kids.size(), false);
+        }
+        if (!spec.adoptParentValues(runtime, setupHolder)) {
+            return null; 
+        } else {
+            class TestHarness extends Harness {
+                public RunResult run(AjcTest.Suite.Spec spec) {
+                    return super.run(spec);
+                } 
+            }
+            TestHarness h = new TestHarness();
+            return h.run(spec);            
+        }
+    }
+
+    class Exp {
+        public final int tests;
+        public final int testsRun;
+        public final int skipped;
+        public final int passed;
+        public final int failed;
+        public final int incomplete;
+        public final int infix;
+        Exp(int tests, int testsRun, int skipped, int passed, int failed, int incomplete, int infix) {
+            this.tests = tests;
+            this.testsRun = testsRun;
+            this.skipped = skipped;
+            this.passed = passed;
+            this.failed = failed;
+            this.incomplete = incomplete;
+            this.infix = infix;
+        }
+    }   
+         
+    public void checkSelection(String suiteFile, String[] options, String infoInfix, Exp exp) {
+        MessageHandler holder = new MessageHandler();
+        Harness.RunResult result = runSuite(suiteFile, options, holder);
+        if (verbose) {
+            MessageUtil.print(System.out, holder, " setup - ");
+        }
+        assertNotNull("Harness.RunResult", result);
+        // XXX sync hack snooping of message text with skip messages, harness
+        final List skipList = MessageUtil.getMessages(holder, IMessage.INFO, false, "skip");
+        final int numSkipped = skipList.size();        
+        IRunStatus status = result.status;
+        assertNotNull(status);
+        if (verbose) {
+            RunUtils.print(System.out, "result - ", status);
+            System.out.println(BridgeUtil.childString(status, numSkipped, result.numIncomplete));
+        }
+        assertEquals("skips", exp.skipped, numSkipped);
+        IRunStatus[] children = status.getChildren();
+        assertNotNull(children);
+        assertTrue(children.length + "!= expRun=" + exp.testsRun, 
+            exp.testsRun == children.length);
+        int actPass = 0;
+        for (int i = 0; i < children.length; i++) {
+            if (RunValidator.NORMAL.runPassed(children[i])) {
+                actPass++;
+            }
+		}
+        if (exp.passed != actPass) {
+            assertTrue("exp.passed=" + exp.passed + " != actPass=" + actPass, false);
+        }
+        if (!LangUtil.isEmpty(infoInfix)) {
+            int actInfix = MessageUtil.getMessages(holder, IMessage.INFO, false, infoInfix).size();
+            if (actInfix != exp.infix) {
+                String s = "for infix \"" + infoInfix 
+                    + "\" actInfix=" + actInfix + " != expInfix=" + exp.infix;
+                assertTrue(s, false);
+            }
+        }
+    }
+    
+    private boolean ajcAvailable() { // XXX util
+        try {
+            return (null != Class.forName("org.aspectj.compiler.base.JavaCompiler"));
+        } catch (ClassNotFoundException e) {
+            return false;
+        }
+    }
+
+    
+    private boolean eclipseAvailable() { // XXX util
+        try {
+            return (null != Class.forName("org.aspectj.ajdt.ajc.AjdtCommand"));
+        } catch (ClassNotFoundException e) {
+            return false;
+        }
+    }
+}