/* ******************************************************************* * Copyright (c) 1999-2001 Xerox Corporation, * 2002 Palo Alto Research Center, Incorporated (PARC). * All rights reserved. * This program and the accompanying materials are made available * under the terms of the Eclipse Public License v 2.0 * which accompanies this distribution and is available at * https://www.eclipse.org/org/documents/epl-2.0/EPL-2.0.txt * * Contributors: * Xerox/PARC initial implementation * ******************************************************************/ package org.aspectj.testing.drivers; import org.aspectj.bridge.IMessage; import org.aspectj.bridge.MessageHandler; import org.aspectj.bridge.MessageUtil; import org.aspectj.testing.harness.bridge.AbstractRunSpec; import org.aspectj.testing.harness.bridge.AjcTest; import org.aspectj.testing.harness.bridge.IRunSpec; import org.aspectj.testing.run.IRunStatus; import org.aspectj.testing.run.RunValidator; import org.aspectj.testing.util.BridgeUtil; import org.aspectj.testing.util.RunUtils; import org.aspectj.testing.xml.AjcSpecXmlReader; import org.aspectj.util.LangUtil; import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.Hashtable; import java.util.List; import junit.framework.TestCase; /** * The harness supports custom-coded queries based on * -ajctest{kind}={query} parameters * (until we move to an XML database with real queries). */ public class HarnessSelectionTest extends TestCase { private static final String TESTDATA = "../testing-drivers/testdata"; private static final String INC_HARNESS_DIR = TESTDATA + "/incremental/harness"; private static final String SELECT = INC_HARNESS_DIR + "/selectionTest.xml"; /** @see testIncrementalSuite() */ private static final String INCREMENTAL = INC_HARNESS_DIR + "/suite.xml"; private static final String TITLE_LIST_ONE = INC_HARNESS_DIR + "/titleListOne.txt"; private static final String TITLE_LIST_PLURAL = INC_HARNESS_DIR + "/titleListPlural.txt"; private static Hashtable SPECS = new Hashtable(); private static AjcTest.Suite.Spec getSpec(String suiteFile) { AjcTest.Suite.Spec result = (AjcTest.Suite.Spec) SPECS.get(suiteFile); if (null == result) { try { result = AjcSpecXmlReader.getReader().readAjcSuite(new File(suiteFile)); SPECS.put(suiteFile, result); } catch (IOException e) { e.printStackTrace(System.err); } } try { return (AjcTest.Suite.Spec) result.clone(); } catch (CloneNotSupportedException e) { e.printStackTrace(System.err); assertTrue("clone failed: " + e, false); return null; // keep compiler happy } } private boolean verbose; public HarnessSelectionTest(String name) { super(name); } public void testFilesAvailable() { String[] files = new String[] { SELECT, INCREMENTAL, TITLE_LIST_ONE, TITLE_LIST_PLURAL }; for (String file : files) { assertTrue(file, new File(file).canRead()); } } public void testIncrementalSuite() { System.err.println("skipping testIncrementalSuite - too long"); if (true) return; if (!eclipseAvailable()) { System.err.println("skipping test - eclipse classes not available"); return; } String[] options = new String[] { "!verbose", "!eclipse", }; Exp exp = new Exp(6, 6, 0, 6, 0, 0, 0); checkSelection(INCREMENTAL, options, "INFIX IGNORED", exp); } public void testKeywordSelectionBoth() { if (!eclipseAvailable()) { System.err.println("skipping test - eclipse classes not available"); return; } String[] options = new String[] { "-ajctestRequireKeywords=requireKeyword", "-ajctestSkipKeywords=skipKeyword,skipUnenforcedAjcLimit", "!verbose", "-eclipse", }; Exp exp = new Exp(17, 1, 16, 1, 0, 0, 1); checkSelection(SELECT, options, "keyword skipKeyword was found", exp); } public void testKeywordSelectionRequire() { if (!eclipseAvailable()) { System.err.println("skipping test - eclipse classes not available"); return; } String[] options = new String[] { "-ajctestRequireKeywords=skipKeyword", "!verbose", "-eclipse", }; Exp exp = new Exp(17, 1, 16, 1, 0, 0, 16); checkSelection(SELECT, options, "keyword skipKeyword was not found", exp); } public void testKeywordSelectionSkip() { if (!eclipseAvailable()) { System.err.println("skipping test - eclipse classes not available"); return; } String[] options = new String[] { "-ajctestSkipKeywords=requireKeyword", "!verbose", "-eclipse", }; Exp exp = new Exp(17, 0, 17, 0, 0, 0, 17); checkSelection(SELECT, options, "keyword requireKeyword was found", exp); } public void testNoOptions() { if (!ajcAvailable()) { System.err.println("skipping test - ajc classes not available"); return; } String[] options = new String[] { "!ajc" }; Exp exp = new Exp(17, 3, 14, 3, 0, 0, 4); checkSelection(SELECT, options, "normally-valid", exp); } public void testEclipseOptionsSkip() { String[] options = new String[] { "-eclipse", "-ajctestRequireKeywords=eclipseOptionSkip" }; Exp exp = new Exp(17, 0, 17, 0, 0, 0, 6); checkSelection(SELECT, options, "old ajc 1.0 option", exp); } public void testAjcEclipseConflict() { if (!ajcAvailable()) { System.err.println("skipping test - ajc classes not available"); return; } String[] options = new String[] { "!ajc" }; Exp exp = new Exp(17, 3, 14, 3, 0, 0, 6); checkSelection(SELECT, options, "conflict between !eclipse and !ajc", exp); } public void testEclipseConflict() { String[] options = new String[] { "^eclipse", "-ajctestSkipKeywords=skipUnenforcedAjcLimit" }; Exp exp = new Exp(17, 3, 14, 3, 0, 0, 6); checkSelection(SELECT, options, "force conflict between eclipse", exp); } public void testSinglePR() { String[] options = new String[] { "-eclipse", "-ajctestPR=100" }; Exp exp = new Exp(17, 1, 16, 1, 0, 0, 16); checkSelection(SELECT, options, "bugId required", exp); } public void testTwoPR() { String[] options = new String[] { "-eclipse", "-ajctestPR=100,101" }; Exp exp = new Exp(17, 2, 15, 2, 0, 0, 15); checkSelection(SELECT, options, "bugId required", exp); } public void testTitleContainsSubstringSelection() { String[] options = new String[] { "-ajctestTitleContains=run and ", "-eclipse" }; Exp exp = new Exp(17, 1, 16, 1, 0, 0, 16); checkSelection(SELECT, options, "run and", exp); } public void testTitleContainsSubstringSelectionPlural() { String[] options = new String[] { "-ajctestTitleContains= run and , if skipKeyword ", "-eclipse" }; Exp exp = new Exp(17, 2, 15, 2, 0, 0, 15); checkSelection(SELECT, options, "title", exp); } public void testTitleContainsExactSelection() { String[] options = new String[] { "-ajctestTitleContains=run and pass", "-eclipse" }; Exp exp = new Exp(17, 1, 16, 1, 0, 0, 16); checkSelection(SELECT, options, "run and pass", exp); } public void testTitleContainsExactSelectionPlural() { String[] options = new String[] { "-ajctestTitleContains= run and pass , omit if skipKeyword ", "-eclipse" }; Exp exp = new Exp(17, 2, 15, 2, 0, 0, 15); checkSelection(SELECT, options, "title", exp); } public void testTitleListSelection() { String[] options = new String[] { "-ajctestTitleList=run and pass", "-eclipse" }; Exp exp = new Exp(17, 1, 16, 1, 0, 0, 16); checkSelection(SELECT, options, "run and pass", exp); } public void testTitleListSelectionPlural() { String[] options = new String[] { "-ajctestTitleList= run and pass , omit if skipKeyword ", "-eclipse" }; Exp exp = new Exp(17, 2, 15, 2, 0, 0, 15); checkSelection(SELECT, options, "title", exp); } public void testTitleListFileSelection() { String[] options = new String[] { "-ajctestTitleList=" + TITLE_LIST_ONE, "-eclipse" }; Exp exp = new Exp(17, 1, 16, 1, 0, 0, 16); checkSelection(SELECT, options, TITLE_LIST_ONE, exp); } public void testTitleListFileSelectionPlural() { String[] options = new String[] { "-ajctestTitleList=" + TITLE_LIST_PLURAL, "-eclipse" }; Exp exp = new Exp(17, 2, 15, 2, 0, 0, 15); checkSelection(SELECT, options, TITLE_LIST_PLURAL, exp); // Now check the "fail only" path options = new String[] { "-ajctestTitleFailList=" + TITLE_LIST_PLURAL, "-eclipse" }; // 1 messages skipped when run under 1.4 for other reasons, // so count "skip" instead of TITLE_LIST_PLURAL exp = new Exp(17, 2, 15, 2, 0, 0, 15); checkSelection(SELECT, options, "skip", exp); } /** * Run the static test suite with the given options. * @param setupHolder the IMessageHolder for any setup messages * @return null if setup failed or Harness.RunResult if suite completed. */ private Harness.RunResult runSuite(String suiteFile, String[] options, MessageHandler setupHolder) { AbstractRunSpec.RT runtime = new AbstractRunSpec.RT(); runtime.setOptions(options); AjcTest.Suite.Spec spec = getSpec(suiteFile); assertNotNull(spec); ArrayList kids = spec.getChildren(); assertNotNull(kids); if ((suiteFile == SELECT) && (17 != kids.size())) { assertTrue("expected 17 kids, got " + kids.size(), false); } if (!spec.adoptParentValues(runtime, setupHolder)) { return null; } else { class TestHarness extends Harness { public RunResult run(AjcTest.Suite.Spec spec) { return super.run(spec); } } TestHarness h = new TestHarness(); return h.run(spec); } } class Exp { public final int tests; public final int testsRun; public final int skipped; public final int passed; public final int failed; public final int incomplete; public final int infix; Exp(int tests, int testsRun, int skipped, int passed, int failed, int incomplete, int infix) { this.tests = tests; this.testsRun = testsRun; this.skipped = skipped; this.passed = passed; this.failed = failed; this.incomplete = incomplete; this.infix = infix; } } public void checkSelection(String suiteFile, String[] options, String infoInfix, Exp exp) { MessageHandler holder = new MessageHandler(); Harness.RunResult result = runSuite(suiteFile, options, holder); if (verbose) { MessageUtil.print(System.out, holder, " setup - "); } assertNotNull("Harness.RunResult", result); // XXX sync hack snooping of message text with skip messages, harness final List skipList = MessageUtil.getMessages(holder, IMessage.INFO, false, "skip"); final int numSkipped = skipList.size(); IRunStatus status = result.status; assertNotNull(status); if (verbose) { RunUtils.print(System.out, "result - ", status); System.out.println(BridgeUtil.childString(status, numSkipped, result.numIncomplete)); } assertEquals("skips", exp.skipped, numSkipped); IRunStatus[] children = status.getChildren(); assertNotNull(children); assertTrue(children.length + "!= expRun=" + exp.testsRun, exp.testsRun == children.length); int actPass = 0; for (IRunStatus child : children) { if (RunValidator.NORMAL.runPassed(child)) { actPass++; } } if (exp.passed != actPass) { assertTrue("exp.passed=" + exp.passed + " != actPass=" + actPass, false); } if (!LangUtil.isEmpty(infoInfix)) { int actInfix = MessageUtil.getMessages(holder, IMessage.INFO, false, infoInfix).size(); if (actInfix != exp.infix) { String s = "for infix \"" + infoInfix + "\" actInfix=" + actInfix + " != expInfix=" + exp.infix; assertTrue(s, false); } } } private boolean ajcAvailable() { // XXX util try { Class.forName("org.aspectj.compiler.base.JavaCompiler"); return true; } catch (ClassNotFoundException e) { return false; } } private boolean eclipseAvailable() { // XXX util try { Class.forName("org.aspectj.ajdt.ajc.AjdtCommand"); return true; } catch (ClassNotFoundException e) { return false; } } }