org.aspectj/testing-drivers/src/test/java/org/aspectj/testing/drivers/Harness.java

/* *******************************************************************
 * Copyright (c) 2002 Palo Alto Research Center, Incorporated (PARC),
 *               2003 Contributors.
 * All rights reserved.
 * This program and the accompanying materials are made available
 * under the terms of the Eclipse Public License v 2.0
 * which accompanies this distribution and is available at
 * https://www.eclipse.org/org/documents/epl-2.0/EPL-2.0.txt
 *
 * Contributors:
 *     Xerox/PARC     initial implementation
 *     Wes Isberg     2003 changes.
 * ******************************************************************/

package org.aspectj.testing.drivers;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.io.PrintStream;
import java.io.PrintWriter;
import java.text.DecimalFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.Set;
import java.util.StringTokenizer;

import org.aspectj.bridge.IMessage;
import org.aspectj.bridge.IMessageHolder;
import org.aspectj.bridge.MessageHandler;
import org.aspectj.bridge.MessageUtil;
import org.aspectj.testing.harness.bridge.AbstractRunSpec;
import org.aspectj.testing.harness.bridge.AjcTest;
import org.aspectj.testing.harness.bridge.CompilerRun;
import org.aspectj.testing.harness.bridge.FlatSuiteReader;
import org.aspectj.testing.harness.bridge.IRunSpec;
import org.aspectj.testing.harness.bridge.IncCompilerRun;
import org.aspectj.testing.harness.bridge.JavaRun;
import org.aspectj.testing.harness.bridge.RunSpecIterator;
import org.aspectj.testing.harness.bridge.Sandbox;
import org.aspectj.testing.harness.bridge.Validator;
import org.aspectj.testing.run.IRun;
import org.aspectj.testing.run.IRunIterator;
import org.aspectj.testing.run.IRunListener;
import org.aspectj.testing.run.IRunStatus;
import org.aspectj.testing.run.IRunValidator;
import org.aspectj.testing.run.RunListener;
import org.aspectj.testing.run.RunStatus;
import org.aspectj.testing.run.RunValidator;
import org.aspectj.testing.run.Runner;
import org.aspectj.testing.util.BridgeUtil;
import org.aspectj.testing.util.RunUtils;
import org.aspectj.testing.util.StreamsHandler;
import org.aspectj.testing.util.StreamsHandler.Result;
import org.aspectj.testing.xml.AjcSpecXmlReader;
import org.aspectj.testing.xml.XMLWriter;
import org.aspectj.util.FileUtil;
import org.aspectj.util.LangUtil;

/**
 * Test harness for running AjcTest.Suite test suites.
 * This can be easily extended by subclassing.
 * <ul>
 * <li>template algorithms for reading arguments, printing syntax,
 *     reading suites, and reporting results all
 *     delegate to methods that subclasses can override to support
 *     additional arguments or different reporting.</li>
 * <li>implement arbitrary features as IRunListeners</li>
 * <li>support single-option aliases to any number of single-options </li>
 * </ul>
 * See {@link report(IRunStatus, int, int)} for an explanation of test result
 * categories.
 */
public class Harness {
	/**
	 * Spaces up to the width that an option should take in the syntax,
	 * including the two-space leader
	 */
	protected static final String SYNTAX_PAD = "                    ";
	protected static final String OPTION_DELIM = ";";
	private static final String JAVA_VERSION;
	private static final String ASPECTJ_VERSION;
	static {
		String version = "UNKNOWN";
		try {  version = System.getProperty("java.version", "UNKNOWN"); }
		catch (Throwable t) {}
		JAVA_VERSION = version;

		version = "UNKNOWN";
		try {
			Class c = Class.forName("org.aspectj.bridge.Version");
			version = (String) c.getField("text").get(null);
		} catch (Throwable t) {
			// ignore
		}
		ASPECTJ_VERSION = version;
	}

	/** factory for the subclass currently anointed as default */
	public static Harness makeHarness() {
		return new FeatureHarness();
	}

	/** @param args String[] like runMain(String[]) args */
	public static void main(String[] args) throws Exception {
		if (LangUtil.isEmpty(args)) {
			File argFile = new File("HarnessArgs.txt");
			if (argFile.canRead()) {
				args = readArgs(argFile);
			} else {
				args = new String[] { "-help" };
			}
		}
		makeHarness().runMain(args, null);
	}

	/**
	 * Get known option aliases.
	 * Subclasses may add new aliases, where the key is the alias option,
	 * and the value is a comma-delimited String of target options.
	 * @return Properties with feature aliases or null
	 */
	protected static Properties getOptionAliases() {
		if (null == optionAliases) {
			optionAliases = new Properties();
			// XXX load from **OptionAliases.properties
		}
		return optionAliases;
	}

	/**
	 * Read argFile contents into String[],
	 * delimiting at any whitespace
	 */
	private static String[] readArgs(File argFile) {
		ArrayList<String> args = new ArrayList<>();
		//        int lineNum = 0;

		try {
			BufferedReader stream =
					new BufferedReader(new FileReader(argFile));
			String line;
			while (null != (line = stream.readLine())) {
				StringTokenizer st = new StringTokenizer(line);
				while (st.hasMoreTokens()) {
					args.add(st.nextToken());
				}
			}
		} catch (IOException e) {
			e.printStackTrace(System.err);
		}
		return args.toArray(new String[0]);
	}

	/** aliases key="option" value="option{,option}" */
	private static Properties optionAliases;

	/** be extra noisy if true */
	private boolean verboseHarness;

	/** be extra quiet if true */
	private boolean quietHarness;

	/** just don't say anything! */
	protected boolean silentHarness;

	private Map<String,Feature> features;

	/** if true, do not delete temporary files. */
	private boolean keepTemp;

	/** if true, delete temporary files as each test completes. */
	private boolean killTemp;

	/** if true, then log results in report(..) when done */
	private boolean logResults;

	/** if true and there were failures, do System.exit({numFailures})*/
	private boolean exitOnFailure;

	protected Harness() {
		features = new HashMap<>();
	}


	/**
	 * Entry point for a test.
	 * This reads in the arguments,
	 * creates the test suite(s) from the input file(s),
	 * and for each suite does setup, run, report, and cleanup.
	 * When arguments are read, any option ending with "-" causes
	 * option variants, a set of args with and another without the
	 * option. See {@link LangUtil.optionVariants(String[])} for
	 * more details.
	 * @param args the String[] for the test suite - use -help to get options,
	 *         and use "-" suffixes for variants.
	 * @param resultList List for IRunStatus results - ignored if null
	 */
	public void runMain(String[] args, List resultList) {
		LangUtil.throwIaxIfFalse(!LangUtil.isEmpty(args), "empty args");
		// read arguments
		final ArrayList<String> globals = new ArrayList<>();
		final List<String> files = new ArrayList<>();
		final LinkedList<String> argList = new LinkedList<>(Arrays.asList(args));
		for (int i = 0; i < argList.size(); i++) {
			String arg = argList.get(i);
			List<String> aliases = aliasOptions(arg);
			if (!LangUtil.isEmpty(aliases)) {
				argList.remove(i);
				argList.addAll(i, aliases);
				i--;
				continue;
			}
			if ("-help".equals(arg)) {
				logln("java " + Harness.class.getName() + " {option|suiteFile}..");
				printSyntax(getLogStream());
				return;
			} else if (isSuiteFile(arg)) {
				files.add(arg);
			} else if (!acceptOption(arg)) {
				globals.add(arg);
			} // else our options absorbed
		}
		if (0 == files.size()) {
			logln("## Error reading arguments: at least 1 suite file required");
			logln("java " + Harness.class.getName() + " {option|suiteFile}..");
			printSyntax(getLogStream());
			return;
		}
		String[] globalOptions = globals.toArray(new String[0]);
		String[][] globalOptionVariants = optionVariants(globalOptions);
		AbstractRunSpec.RT runtime = new AbstractRunSpec.RT();
		if (verboseHarness) {
			runtime.setVerbose(true);
		}

		// run suites read from each file
		AjcTest.Suite.Spec spec;
		for (String string : files) {
			File suiteFile = new File(string);
			if (!suiteFile.canRead()) {
				logln("runMain(..) cannot read file: " + suiteFile);
				continue;
			}
			if (null == (spec = readSuite(suiteFile))) {
				logln("runMain(..) cannot read suite from file: " + suiteFile);
				continue;
			}

			MessageHandler holder = new MessageHandler();
			for (String[] globalOptionVariant : globalOptionVariants) {
				runtime.setOptions(globalOptionVariant);
				holder.init();
				boolean skip = !spec.adoptParentValues(runtime, holder);
				// awful/brittle assumption about number of skips == number of skip messages
				final List<IMessage> skipList = MessageUtil.getMessages(holder, IMessage.INFO, false, "skip");
				if ((verboseHarness || skip || (0 < skipList.size()))) {
					final List<String> curArgs = Arrays.asList(globalOptionVariant);
					logln("runMain(" + suiteFile + ", " + curArgs + ")");
					if (verboseHarness) {
						String format = "yyyy.MM.dd G 'at' hh:mm:ss a zzz";
						SimpleDateFormat formatter = new SimpleDateFormat (format);
						String date = formatter.format(new Date());
						logln("test date: " + date);
						logln("harness features: " + listFeatureNames());
						logln("Java version: " + JAVA_VERSION);
						logln("AspectJ version: " + ASPECTJ_VERSION);
					}
					if (!(quietHarness || silentHarness) && holder.hasAnyMessage(null, true)) {
						MessageUtil.print(getLogStream(), holder, "skip - ");
						MessageUtil.printMessageCounts(getLogStream(), holder, "skip - ");
					}
				}
				if (!skip) {
					doStartSuite(suiteFile);
					long elapsed = 0;
					RunResult result = null;
					try {
						final long startTime = System.currentTimeMillis();
						result = run(spec);
						if (null != resultList) {
							resultList.add(result);
						}
						elapsed = System.currentTimeMillis() - startTime;
						report(result.status, skipList.size(), result.numIncomplete, elapsed);
					} finally {
						doEndSuite(suiteFile,elapsed);
					}
					if (exitOnFailure) {
						int numFailures = RunUtils.numFailures(result.status, true);
						if (0 < numFailures) {
							System.exit(numFailures);
						}
						Object value = result.status.getResult();
						if ((value instanceof Boolean)
								&& !(Boolean) value) {
							System.exit(-1);
						}
					}
				}
			}
		}
	}


	/**
	 * Tell all IRunListeners that we are about to start a test suite
	 * @param suiteFile
	 * @param elapsed
	 */
	private void doEndSuite(File suiteFile, long elapsed) {
		Collection c = features.values();
		for (Object o : c) {
			Feature element = (Feature) o;
			if (element.listener instanceof TestCompleteListener) {
				((TestCompleteListener) element.listener).doEndSuite(suiteFile, elapsed);
			}
		}
	}
	/**
	 * Generate variants of String[] options by creating an extra set for
	 * each option that ends with "-".  If none end with "-", then an
	 * array equal to <code>new String[][] { options }</code> is returned;
	 * if one ends with "-", then two sets are returned,
	 * three causes eight sets, etc.
	 * @return String[][] with each option set.
	 * @throws IllegalArgumentException if any option is null or empty.
	 */
	public static String[][] optionVariants(String[] options) {
		if ((null == options) || (0 == options.length)) {
			return new String[][] { new String[0]};
		}
		// be nice, don't stomp input
		String[] temp = new String[options.length];
		System.arraycopy(options, 0, temp, 0, temp.length);
		options = temp;
		boolean[] dup = new boolean[options.length];
		int numDups = 0;

		for (int i = 0; i < options.length; i++) {
			String option = options[i];
			if (LangUtil.isEmpty(option)) {
				throw new IllegalArgumentException("empty option at " + i);
			}
			if (option.endsWith("-")) {
				options[i] = option.substring(0, option.length()-1);
				dup[i] = true;
				numDups++;
			}
		}
		final String[] NONE = new String[0];
		final int variants = exp(2, numDups);
		final String[][] result = new String[variants][];
		// variant is a bitmap wrt doing extra value when dup[k]=true
		for (int variant = 0; variant < variants; variant++) {
			ArrayList<String> next = new ArrayList<>();
			int nextOption = 0;
			for (int k = 0; k < options.length; k++) {
				if (!dup[k] || (0 != (variant & (1 << (nextOption++))))) {
					next.add(options[k]);
				}
			}
			result[variant] = next.toArray(NONE);
		}
		return result;
	}

	private static int exp(int base, int power) { // not in Math?
		if (0 > power) {
			throw new IllegalArgumentException("negative power: " + power);
		}
		int result = 1;
		while (0 < power--) {
			result *= base;
		}
		return result;
	}

	/**
	 * @param suiteFile
	 */
	private void doStartSuite(File suiteFile) {
		Collection<Feature> c = features.values();
		for (Feature element : c) {
			if (element.listener instanceof TestCompleteListener) {
				((TestCompleteListener)element.listener).doStartSuite(suiteFile);
			}
		}
	}

	/** Run the test suite specified by the spec */
	protected RunResult run(AjcTest.Suite.Spec spec) {
		LangUtil.throwIaxIfNull(spec, "spec");
		/*
		 * For each run, initialize the runner and validator,
		 * create a new set of IRun{Iterator} tests,
		 * and run them.
		 * Delete all temp files when done.
		 */
		Runner runner = new Runner();
		if (0 != features.size()) {
			for (Entry<String, Feature> entry : features.entrySet()) {
				Feature feature = entry.getValue();
				runner.registerListener(feature.clazz, feature.listener);
			}
		}
		IMessageHolder holder = new MessageHandler();
		int numIncomplete = 0;
		RunStatus status = new RunStatus(holder, runner);
		status.setIdentifier(spec);
		// validator is used for all setup in entire tree...
		Validator validator = new Validator(status);
		if (!killTemp) {
			validator.lock(this);
		}
		Sandbox sandbox = null;
		try {
			sandbox = new Sandbox(spec.getSuiteDirFile(), validator);
			IRunIterator tests = spec.makeRunIterator(sandbox, validator);
			runner.runIterator(tests, status, null);
			if (tests instanceof RunSpecIterator) {
				numIncomplete = ((RunSpecIterator) tests).getNumIncomplete();
			}
		} finally {
			if (!keepTemp) {
				if (!killTemp) {
					validator.unlock(this);
				}
				validator.deleteTempFiles(verboseHarness);
			}
		}
		return new RunResult(status, numIncomplete);
	}

	/**
	 * Report the results of a test run after it is completed.
	 * Clients should be able to identify the number of:
	 * <ul>
	 * <li>tests run and passed</li>
	 * <li>tests failed, i.e., run and not passed (fail, error, etc.)</li>
	 * <li>tests incomplete, i.e., test definition read but test run setup failed</li>
	 * <li>tests skipped, i.e., test definition read and found incompatible with
	 *     the current configuration.</li>
	 * <ul>
	 *
	 * @param status returned from the run
	 * @param numSkipped int tests that were skipped because of
	 *         configuration incompatibilities
	 * @param numIncomplete int tests that failed during setup,
	 *         usually indicating a test definition or configuration error.
	 * @param msElapsed elapsed time in milliseconds
	 * */
	protected void report(IRunStatus status, int numSkipped, int numIncomplete,
			long msElapsed ) {
		if (logResults) {
			RunUtils.AJCSUITE_PRINTER.printRunStatus(getLogStream(), status);
		} else if (!(quietHarness || silentHarness) && (0 < status.numMessages(null, true))) {
			if (!silentHarness) {
				MessageUtil.print(getLogStream(), status, "");
			}
		}

		logln(BridgeUtil.childString(status, numSkipped, numIncomplete)
				+ " " + (msElapsed/1000) + " seconds");

	}

	// --------------- delegate methods
	protected void logln(String s) {
		if (!silentHarness) {
			getLogStream().println(s);
		}
	}

	protected PrintStream getLogStream() {
		return System.out;
	}

	protected boolean isSuiteFile(String arg) {
		return ((null != arg)
				&& (arg.endsWith(".txt") || arg.endsWith(".xml"))
				&& new File(arg).canRead());
	}

	/**
	 * Get the options that the input option is an alias for.
	 * Subclasses may add options directly to the getFeatureAliases result
	 * or override this.
	 * @return null if the input is not an alias for other options,
	 * or a non-empty List (String) of options that this option is an alias for
	 */
	protected List<String> aliasOptions(String option) {
		Properties aliases = Harness.getOptionAliases();
		if (null != aliases) {
			String args = aliases.getProperty(option);
			if (!LangUtil.isEmpty(args)) {
				return LangUtil.anySplit(args, OPTION_DELIM);
			}
		}
		return null;
	}

	/**
	 * Read and implement any of our options.
	 * Options other than this and suite files will be
	 * passed down as parent options through the test spec hierarchy.
	 * Subclasses override this to implement new options.
	 */
	protected boolean acceptOption(String option) {
		//        boolean result = false;
		if (LangUtil.isEmpty(option)) {
			return true; // skip bad input
		} else if ("-verboseHarness".equals(option)) {
			verboseHarness = true;
		} else if ("-quietHarness".equals(option)) {
			quietHarness = true;
		} else if ("-silentHarness".equals(option)) {
			silentHarness = true;
		} else if ("-keepTemp".equals(option)) {
			keepTemp = true;
		} else if ("-killTemp".equals(option)) {
			killTemp = true;
		} else if ("-logResults".equals(option)) {
			logResults = true;
		} else if ("-exitOnFailure".equals(option)) {
			exitOnFailure = true;
		} else {
			return false;
		}
		return true;
	}

	/**
	 * Read a test suite file.
	 * This implementation knows how to read .txt and .xml files
	 * and logs any errors.
	 * Subclasses override this to read new kinds of suites.
	 * @return null if unable to read (logging errors) or AjcTest.Suite.Spec otherwise
	 */
	protected AjcTest.Suite.Spec readSuite(File suiteFile) {
		if (null != suiteFile) {
			String path = suiteFile.getPath();
			try {
				if (path.endsWith(".xml")) {
					return AjcSpecXmlReader.getReader().readAjcSuite(suiteFile);
				} else if (path.endsWith(".txt")) {
					return FlatSuiteReader.ME.readSuite(suiteFile);
				} else {
					logln("unrecognized extension? " + path);
				}
			} catch (IOException e) {
				e.printStackTrace(getLogStream());
			}
		}
		return null;
	}

	/** Add feature to take effect during the next runMain(..) invocation.
	 * @param feature the Feature to add, using feature.name as key.
	 */
	protected void addFeature(Feature feature) {
		if (null != feature) {
			features.put(feature.name, feature);
		}
	}

	/** remove feature by name (same as feature.name) */
	protected void removeFeature(String name) {
		if (!LangUtil.isEmpty(name)) {
			features.remove(name);
		}
	}

	/** @return unmodifiable Set of feature names */
	protected Set listFeatureNames() {
		return Collections.unmodifiableSet(features.keySet());
	}

	/** print detail message for syntax of main(String[]) command-line */
	protected void printSyntax(PrintStream out) {
		out.println("  {??}              unrecognized options are used as test spec globals");
		out.println("  -help             print this help message");
		out.println("  -verboseHarness   harness components log verbosely");
		out.println("  -quietHarness     harness components suppress logging");
		out.println("  -keepTemp         do not delete temporary files");
		out.println("  -logResults       log results at end, verbosely if fail");
		out.println("  -exitOnFailure    do System.exit({num-failures}) if suite fails");
		out.println("  {suiteFile}.xml.. specify test suite XML file");
		out.println("  {suiteFile}.txt.. specify test suite .txt file (deprecated)");
	}

	/** print known aliases at the end of the syntax message */
	protected void printAliases(PrintStream out) {
		LangUtil.throwIaxIfNull(out, "out");
		Map props = getOptionAliases();
		if (null == props) {
			return;
		}
		int pdLength = SYNTAX_PAD.length();
		Set<Map.Entry<Object,Object>> entries = props.entrySet();
		for (Map.Entry<Object,Object> entry : entries) {
			String alias = "  " + (String) entry.getKey();
			int buf = pdLength - alias.length();
			if (0 < buf) {
				alias += SYNTAX_PAD.substring(0, buf);
			} else {
				alias += " ";
			}
			out.println(alias + entry.getValue());
		}
	}

	/** result struct for run(AjcTest.Spec) */
	public static class RunResult {
		public final IRunStatus status;
		public final int numIncomplete;
		public RunResult(IRunStatus status, int numIncomplete) {
			this.status = status;
			this.numIncomplete = numIncomplete;
		}
	}
	/** feature implemented as named IRunIterator/IRun association */
	public static class Feature {
		/** never null, always assignable to IRun */
		public final Class clazz;

		/** never null */
		public final IRunListener listener;

		/** never null or empty */
		public final String name;

		/** @throws IllegalArgumentException if any is null/empty or clazz is
		 *           not assignable to IRun
		 */
		public Feature(String name, Class clazz, IRunListener listener) {
			LangUtil.throwIaxIfNull(clazz, "class");
			if (!IRun.class.isAssignableFrom(clazz)
					&& !IRunIterator.class.isAssignableFrom(clazz)) {
				String s = clazz.getName() + "is not assignable to IRun or IRunIterator";
				LangUtil.throwIaxIfFalse(false, s);
			}
			LangUtil.throwIaxIfNull(listener, "listener");
			LangUtil.throwIaxIfNull(name, "name");
			LangUtil.throwIaxIfFalse(0 < name.length(), "empty name");
			this.clazz = clazz;
			this.listener = listener;
			this.name = name;
		}

		/** @return feature name */
		@Override
		public String toString() {
			return name;
		}
	}
}


/**
 * Harness with features for controlling output
 * (logging results and hiding streams).
 * Use -help to get a list of feature options.
 */
class FeatureHarness extends Harness {

	private static final String[] ALIASES = new String[]
			{ "-hideStreams",
					"-hideCompilerStreams"
							+ OPTION_DELIM + "-hideRunStreams",
							"-jim",
							"-logMinFail"
									+ OPTION_DELIM + "-hideStreams",
									"-loud",
									"-verboseHarness",
									"-baseline",
									"-verboseHarness"
											+ OPTION_DELIM + "-traceTestsMin"
											+ OPTION_DELIM + "-hideStreams",
											"-release",
											"-baseline"
													+ OPTION_DELIM + "-ajctestSkipKeywords=knownLimitation,purejava",
													"-junit",
													"-silentHarness" + OPTION_DELIM + "-logJUnit" + OPTION_DELIM +
													"-hideStreams",
													"-cruisecontrol",
													"-junit" + OPTION_DELIM + "-ajctestSkipKeywords=knownLimitation,purejava"
			};
	static {
		Map optionAliases = Harness.getOptionAliases();
		if (null != optionAliases) {
			for (int i = 1; i < ALIASES.length; i += 2) {
				optionAliases.put(ALIASES[i-1], ALIASES[i]);
			}
		}
	}

	/** controller for suppressing and sniffing error and output streams. */
	StreamsHandler streamsHandler;

	/** facility of hiding-streams may be applied in many features */
	IRunListener streamHider;

	/** facility of capture/log may be applied in many features */
	IRunListener captureLogger;

	/** when making tests, do not run them */
	TestMaker testMaker;

	public FeatureHarness() {
		super();
		streamsHandler = new StreamsHandler(false, true);
	}
	/** override to make tests or run as usual */
	@Override
	protected RunResult run(AjcTest.Suite.Spec spec) {
		if (null != testMaker) {
			System.out.println("generating rather than running tests...");
			return testMaker.run(spec);
		} else {
			return super.run(spec);
		}
	}

	/**
	 * Log via StreamsHandler-designated log stream.
	 * @see org.aspectj.testing.drivers.Harness#log(String)
	 */
	@Override
	protected void logln(String s) {
		if (!silentHarness)
			streamsHandler.lnlog(s);
	}

	/**
	 * @see org.aspectj.testing.drivers.Harness#getLogStream()
	 * @return StreamsHandler-designated log stream.
	 */
	@Override
	protected PrintStream getLogStream() {
		return streamsHandler.out;
	}


	/** print detail message for syntax of main(String[]) command-line */
	@Override
	protected void printSyntax(PrintStream out) {
		super.printSyntax(out);
		out.println("  -progressDots     log . or ! for each AjcTest pass or fail");
		out.println("  -logFail          log each failed AjcTest");
		out.println("  -logPass          log each passed AjcTest");
		out.println("  -logAll           log each AjcTest");
		out.println("  -logMinFail       log each AjcTest failure with minimal excess data");
		out.println("  -logMinPass       log each AjcTest success with minimal excess data");
		out.println("  -logMinAll        log all AjcTest with minimal excess data");
		out.println("  -logXMLFail       log XML definition for each failed AjcTest");
		out.println("  -logXMLPass       log XML definition for each passed AjcTest");
		out.println("  -logXMLAll        log XML definition for each AjcTest");
		out.println("  -logJUnit         log all tests in JUnit XML report style");
		out.println("  -hideRunStreams   hide err/out streams during java runs");
		out.println("  -hideCompilerStreams   hide err/out streams during compiler runs");
		out.println("  -traceTests       log pass|fail, /time/memory taken after each test");
		out.println("  -traceTestsMin    log pass|fail after each test");
		out.println("  -XmakeTests       create source files/dirs for initial compile run of each test");
		out.println("  -XlogPublicType   log test XML if \"public type\" in an error message");
		out.println("  -XlogSourceIn=Y,Z log test XML if Y or Z is in path of any sources");
		super.printAliases(out);
	}

	/** Accept a number of logging and output options */
	@Override
	protected boolean acceptOption(String option) {
		if (null == option) {
			return false;
		}

		final StreamsHandler streams = streamsHandler;
		final IRunValidator validator = RunValidator.NORMAL;
		final RunUtils.IRunStatusPrinter verbose
		= RunUtils.VERBOSE_PRINTER;
		final RunUtils.IRunStatusPrinter terse
		= RunUtils.TERSE_PRINTER;
		//        final boolean LOGPASS = true;
		//        final boolean LOGFAIL = true;
		//        final boolean SKIPPASS = false;
		//        final boolean SKIPFAIL = false;
		//        final boolean LOGSTREAMS = true;
		final boolean SKIPSTREAMS = false;

		Feature feature = null;
		if (super.acceptOption(option)) {
			// ok, result returned below

		} else if ("-XmakeTests".equals(option)) {
			testMaker = TestMaker.ME;
		} else if (option.startsWith("-traceTestsMin")) {
			feature = new Feature(option, AjcTest.class,new TestTraceLogger(streams, false));
		} else if (option.startsWith("-traceTests")) {
			feature = new Feature(option, AjcTest.class,new TestTraceLogger(streams, true));
		} else if (option.startsWith("-logMin")) {
			feature = new Feature(option, AjcTest.class,
					new RunLogger(option, SKIPSTREAMS, streams, validator, terse));
		} else if (option.startsWith("-logXML")) {
			feature = new Feature(option, AjcTest.class,
					new XmlLogger(option, streams, validator));
		} else if (option.startsWith("-logJUnit")) {
			feature = new Feature(option, AjcTest.class,
					new JUnitXMLLogger(option,streams,validator));
		} else if (option.startsWith("-log")) {
			feature = new Feature(option, AjcTest.class,
					new RunLogger(option, SKIPSTREAMS, streams, validator, verbose));
		} else if ("-hideRunStreams".equals(option)) {
			feature = new Feature(option, JavaRun.class, getStreamHider());
		} else if ("-hideCompilerStreams".equals(option)) {
			addFeature(new Feature(option, IncCompilerRun.class, getStreamHider()));   // hmmm
			feature = new Feature(option, CompilerRun.class, getStreamHider());
		} else if ("-progressDots".equals(option)) {
			IRunListener listener = new RunListener() {
				@Override
				public void runCompleted(IRunStatus run) {
					streamsHandler.log((validator.runPassed(run) ? "." : "!"));
				}
			};
			feature = new Feature(option, AjcTest.class, listener);
		} else if (option.startsWith("-XlogPublicType")) {
			String label = option + TestCompleteListener.PASS; // print when validator true
			feature = new Feature(option, AjcTest.class,
					new XmlLogger(label, streams, MessageRunValidator.PUBLIC_TYPE_ERROR));
		} else if (option.startsWith("-XlogSourceIn")) {
			String input = option.substring("-XlogSourceIn=".length());
			LangUtil.throwIaxIfFalse(0 < input.length(), option);
			String label = "-XlogSourceIn="  + TestCompleteListener.PASS; // print when validator true
			StringRunner sr = new SubstringRunner(input, false);
			feature = new Feature(option, AjcTest.class,
					new XmlLogger(label, streams, new SourcePathValidator(sr)));
		} else {
			return false;
		}
		addFeature(feature);
		return true;
	}

	/** lazy construction for shared hider */
	protected IRunListener getStreamHider() {
		if (null == streamHider) {
			streamHider =  new RunListener() {
				@Override
				public void runStarting(IRunStatus run) {
					streamsHandler.hide();
				}
				@Override
				public void runCompleted(IRunStatus run) {
					streamsHandler.show();
				}
				@Override
				public String toString() { return "Harness StreamHider"; }
			};
		}
		return streamHider;
	}
}

/** Generate any needed test case files for any test. */
class TestMaker  {

	static TestMaker ME = new TestMaker();

	/** @throws Error if unable to make dir */
	static void mkdirs(File dir) {
		if (null != dir && !dir.exists()) {
			if (!dir.mkdirs()) {
				throw new Error("unable to make dir: " + dir);
			}
		}
	}
	static String getFileContents(File baseDir, File file, String label) {
		String fileName = file.getName();
		if (fileName.endsWith(".java")) {
			fileName = fileName.substring(0, fileName.length() - 5);
		}
		StringBuilder sb = new StringBuilder();
		String filePath = file.getParentFile().getAbsolutePath();
		String dirPath = baseDir.getAbsolutePath();
		String pack = null;
		if (filePath.startsWith(dirPath)) {
			pack = filePath.substring(dirPath.length()).replace('/', '.');
		}
		if (!LangUtil.isEmpty(pack)) {
			sb.append("package " + pack + ";");
		}
		final String EOL = "\n"; // XXX find discovered EOL
		sb.append( EOL
				+ EOL + "import org.aspectj.testing.Tester;"
				+ EOL + ""
				+ EOL + "/** @testcase " + label + " */"
				+ EOL + "public class " + fileName + " {"
				+ EOL + "\tpublic static void main(String[] args) { "
				+ EOL + "\t\tTester.check(null != args, \"null args\"); "
				+ EOL + "\t}"
				+ EOL + "}"
				+ EOL
				);

		return sb.toString();
	}

	/** create a minimal source file for a test */
	static void createSrcFile(File baseDir, File file, String testName) {
		if (file.exists()) {
			return;
		}
		String contents = getFileContents(baseDir, file, testName);
		String error = FileUtil.writeAsString(file, contents);
		if (null != error) {
			throw new Error(error);
		}
	}

	/** create an empty arg file for a test */
	static void createArgFile(File baseDir, File file, String testName) {
		if (file.exists()) {
			return;
		}
		String contents = "// argfile " + file;
		String error = FileUtil.writeAsString(file, contents);
		if (null != error) {
			throw new Error(error);
		}
	}

	public Harness.RunResult run(AjcTest.Suite.Spec spec) {
		Iterable<IRunSpec> kids = spec.getChildren();
		for (IRunSpec iRunSpec : kids) {
			makeTest( (AjcTest.Spec) iRunSpec);
		}
		IRunStatus status = new RunStatus(new MessageHandler(), new Runner());
		status.start();
		status.finish(IRunStatus.PASS);
		return new Harness.RunResult(status, 0);
	}

	private void makeTest(AjcTest.Spec spec) {
		CompilerRun.Spec compileSpec = AjcTest.unwrapCompilerRunSpec(spec);
		if (null == spec) {
			throw new Error("null spec");
		}
		System.out.println("  generating test files for test: " + spec.getDescription());
		File dir = spec.getSuiteDir();
		if (null != dir) {
			TestMaker.mkdirs(dir);
		}
		String offset = spec.getTestDirOffset();
		if (!LangUtil.isEmpty(offset)) {
			if (null == dir) {
				dir = new File(offset);
			} else {
				dir = new File(dir.getAbsolutePath() + "/" + offset);
			}
		} else if (null == dir) {
			dir = new File(".");
		}
		StringBuilder testName = new StringBuilder();
		int pr = spec.getBugId();
		if (0 < pr) {
			testName.append("PR#" + pr + " ");
		}

		testName.append(spec.getDescription());
		final String label = testName.toString();
		final File[] srcFiles = FileUtil.getBaseDirFiles(dir, compileSpec.getPathsArray());
		if (!LangUtil.isEmpty(srcFiles)) {
			for (File srcFile : srcFiles) {
				TestMaker.createSrcFile(dir, srcFile, label);
			}
		}
		final File[] argFiles = FileUtil.getBaseDirFiles(dir, compileSpec.getArgfilesArray());
		if (!LangUtil.isEmpty(argFiles)) {
			for (File argFile : argFiles) {
				TestMaker.createArgFile(dir, argFile, label);
			}
		}

	}

	/** @return "Testmaker()" */
	@Override
	public String toString() {
		return "TestMaker()";
	}
}

interface StringRunner {
	boolean accept(String s);
}

/**
 * StringRunner than accepts input matching 0+ substrings,
 * optionally case-insensitive.
 */
class SubstringRunner implements StringRunner {
	private static String[] extractSubstrings(
			String substrings,
			boolean caseSensitive) {
		if (null == substrings) {
			return null;
		}
		StringTokenizer st = new StringTokenizer(substrings, ",");
		String[] result = new String[st.countTokens()];
		for (int i = 0; i < result.length; i++) {
			result[i] = st.nextToken().trim();
			LangUtil.throwIaxIfFalse(0 < result[i].length(), "empty entry");
			if (!caseSensitive) {
				result[i] = result[i].toLowerCase();
			}
		}
		return result;
	}

	private final String[] substrings;
	private final boolean caseSensitive;

	/**
	 * @param substrings the String containing comma-separated substrings
	 *                    to find in input - if null, any input accepted
	 * @param caseSensitive if true, do case-sensitive comparison
	 * @throws IllegalArgumentException if any substrings contains empty entry ", ,"
	 */
	SubstringRunner(String substrings, boolean caseSensitive) {
		this.caseSensitive = caseSensitive;
		this.substrings = extractSubstrings(substrings, caseSensitive);
	}

	@Override
	public boolean accept(String input) {
		if (null == substrings) {
			return true;
		}
		if (null == input) {
			return false;
		}

		if (!caseSensitive) {
			input = input.toLowerCase();
		}
		for (String substring : substrings) {
			if (input.contains(substring)) {
				return true;
			}
		}
		return false;
	}
}

/**
 * Signal whether run "passed" based on validating absolute source paths.
 * (Static evaluation - no run necessary)
 */
class SourcePathValidator implements IRunValidator { // static - no run needed
	private final StringRunner validator;
	// XXX hoist common
	SourcePathValidator(StringRunner validator) {
		LangUtil.throwIaxIfNull(validator, "validator");
		this.validator = validator;
	}
	/**
	 * @return true if any source files in compile spec are
	 *          accepted by the validator.
	 * @see org.aspectj.testing.run.IRunValidator#runPassed(IRunStatus)
	 */
	@Override
	public boolean runPassed(IRunStatus run) {
		AjcTest.Spec testSpec = AjcTest.unwrapSpec(run);
		if (null != testSpec) {
			CompilerRun.Spec compileSpec = AjcTest.unwrapCompilerRunSpec(testSpec);
			File basedir = new File(testSpec.getSuiteDir(), testSpec.getTestDirOffset());
			String[] paths = compileSpec.getPathsArray();
			File[] files = FileUtil.getBaseDirFiles(basedir, paths);
			for (File file : files) {
				if (validator.accept(file.getAbsolutePath())) {
					return true;
				}
			}
		}
		return false;
	}

}

/** Signal whether run "passed" based on message kind and content */
class MessageRunValidator implements IRunValidator {

	/** signals "passed" if any error contains "public type" */
	static final IRunValidator PUBLIC_TYPE_ERROR
	= new MessageRunValidator("public type", IMessage.ERROR, false);

	private final IMessage.Kind kind;
	private final String sought;
	private final boolean orGreater;

	/**
	 * @param sought the String to seek anywhere in any message of the right kind
	 *         if null, accept any message of the right kind.
	 * @param kind the IMessage.Kind of messages to search - all if null
	 */
	MessageRunValidator(String sought, IMessage.Kind kind, boolean orGreater) {
		this.sought = sought;
		this.kind = kind;
		this.orGreater = orGreater;
	}

	/** @return true if this run has messages of the right kind and text */
	@Override
	public boolean runPassed(IRunStatus run) {
		return gotMessage(new IRunStatus[] {run});
	}

	/**
	 * Search these children and their children recursively
	 * for messages of the right kind and content.
	 * @return true at first match of message of the right kind and content
	 */
	private boolean gotMessage(IRunStatus[] children) {
		if (LangUtil.isEmpty(children)) {
			return false;
		}
		for (IRunStatus run : children) {
			if (null == run) {
				continue; // hmm
			}
			IMessage[] messages = run.getMessages(kind, orGreater);
			if (!LangUtil.isEmpty(messages)) {
				if (LangUtil.isEmpty(sought)) {
					return true;
				} else {
					for (IMessage message : messages) {
						if (null == message) {
							continue; // hmm
						}
						String text = message.getMessage();
						if ((null != text) && (text.contains(sought))) {
							return true;
						}
					}
				}
			}
			if (gotMessage(run.getChildren())) {
				return true;
			}
		}
		return false;
	}
}

/**
 * Base class for listeners that run depending on pass/fail status of input.
 * Template method runCompleted handled whether to run.
 * Subclasses implement doRunCompleted(..).
 */
abstract class TestCompleteListener extends RunListener {
	/** label suffix indicating both pass and fail */
	public static final String ALL = "All";

	/** label suffix indicating fail */
	public static final String FAIL = "Fail";

	/** label suffix indicating pass */
	public static final String PASS = "Pass";


	/** runValidator determines if a given run passed */
	protected final IRunValidator runValidator;

	/** label for this listener */
	final String label;

	/** if trun and run passed, then run doRunCompleted(..) */
	final boolean logOnPass;

	/** if true and run did not pass, then run doRunCompleted(..) */
	final boolean logOnNotPass;

	/** may be null */
	protected final StreamsHandler streamsHandler;

	/** true if the last run evaluation was ok */
	boolean lastRunOk;

	/** last run evaluated */
	IRunStatus lastRun; // XXX small memory leak - cache hashcode instead?

	/** @param label endsWith PASS || FAIL || ALL */
	protected TestCompleteListener(
			String label,
			IRunValidator runValidator,
			StreamsHandler streamsHandler) {
		if (null == runValidator) {
			runValidator = RunValidator.NORMAL;
		}
		this.label = (null == label? "" : label);
		this.logOnPass = label.endsWith(PASS) || label.endsWith(ALL);
		this.logOnNotPass = label.endsWith(FAIL) || label.endsWith(ALL);
		this.runValidator = runValidator;
		this.streamsHandler = streamsHandler;
	}

	public void runStarted(IRunStatus run) {
		if (null != streamsHandler) {
			streamsHandler.startListening();
		}
	}

	/** subclasses implement this to do some per-test initialization */
	protected void doRunStarted(IRunStatus run) {
	}


	/** subclasses implement this to do some per-suite initialization */
	protected void doStartSuite(File suite) {
	}

	/** subclasses implement this to do end-of-suite processing */
	protected void doEndSuite(File suite, long duration) {
	}

	@Override
	public final void runCompleted(IRunStatus run) {
		boolean doit = lastRunOk(run);
		StreamsHandler.Result result = null;
		if (null != streamsHandler) {
			streamsHandler.endListening(doit);
		}
		if (doit) {
			doRunCompleted(run, result);
		}
	}

	/**
	 * @return true if run is ok per constructor specifications
	 */
	protected boolean lastRunOk(IRunStatus run) {
		if (lastRun != run) {
			boolean passed = runValidator.runPassed(run);
			lastRunOk = (passed ? logOnPass : logOnNotPass);
		}
		return lastRunOk;
	}

	/** @return "{classname}({pass}{,fail})" indicating when this runs */
	@Override
	public String toString() { // XXX add label?
		return LangUtil.unqualifiedClassName(this)
				+ "(" + (logOnPass ? (logOnNotPass ? "pass, fail)" : "pass)")
						: (logOnNotPass ? "fail)" : ")"));
	}
	/**
	 * Subclasses implement this to do some completion action
	 * @param run the IRunStatus for this completed run
	 * @param result the StreamsHandler.Result (if any - may be null)
	 */
	public abstract void doRunCompleted(IRunStatus run, StreamsHandler.Result result);
}

/**
 * Write XML for any test passed and/or failed.
 * Must register with Runner for RunSpecIterator.class,
 * most sensibly AjcTest.class.
 */
class XmlLogger extends TestCompleteListener {
	/**
	 * @param printer the component that prints any status - not null
	 * @param runValidator if null, use RunValidator.NORMAL
	 */
	public XmlLogger(
			String label,
			StreamsHandler streamsHandler,
			IRunValidator runValidator) {
		super(label, runValidator, streamsHandler);
	}

	@Override
	public void doRunCompleted(IRunStatus run, StreamsHandler.Result result) {
		PrintStream out = streamsHandler.getLogStream();
		out.println("");
		XMLWriter writer = new XMLWriter(new PrintWriter(out, true));
		Object id = run.getIdentifier();
		if (!(id instanceof Runner.IteratorWrapper)) {
			out.println(this + " not IteratorWrapper: "
					+ id.getClass().getName() + ": " + id);
			return;
		}
		IRunIterator iter = ((Runner.IteratorWrapper) id).iterator;
		if (!(iter instanceof RunSpecIterator)) {
			out.println(this + " not RunSpecIterator: " + iter.getClass().getName()
					+ ": " + iter);
			return;
		}
		((RunSpecIterator) iter).spec.writeXml(writer);
		out.flush();
	}

}

/**
 * Write junit style XML output (for incorporation into html test results and
 * cruise control reports
 * format is...
 * <?xml version="1.0" encoding="UTF-8" ?>
 * <testsuite errors="x" failures="x" name="suite-name" tests="xx" time="ss.ssss">
 * <properties/>
 * <testcase name="passingTest" time="s.hh"></testcase>
 * <testcase name="failingTest" time="s.hh">
 *   <failure message="failureMessage" type="ExceptionType">free text</failure>
 * </testcase>
 * </testsuite>
 */
class JUnitXMLLogger extends TestCompleteListener {

	//  private File suite;
	private StringBuffer junitOutput;
	private long startTimeMillis;
	private int numTests = 0;
	private int numFails = 0;
	private DecimalFormat timeFormatter = new DecimalFormat("#.##");

	public JUnitXMLLogger(
			String label,
			StreamsHandler streamsHandler,
			IRunValidator runValidator) {
		super(label + ALL, runValidator, streamsHandler);
		junitOutput = new StringBuffer();
	}

	/* (non-Javadoc)
	 * @see org.aspectj.testing.drivers.TestCompleteListener#doRunCompleted(org.aspectj.testing.run.IRunStatus, org.aspectj.testing.util.StreamsHandler.Result)
	 */
	@Override
	public void doRunCompleted(IRunStatus run, Result result) {
		long duration = System.currentTimeMillis()  - startTimeMillis;
		numTests++;
		junitOutput.append("<testcase name=\"" + run.getIdentifier() + "\" ");
		junitOutput.append("time=\"" + timeFormatter.format((duration)/1000.0f) + "\"");
		junitOutput.append(">");
		if (!run.runResult())  {
			numFails++;
			junitOutput.append("\n");
			junitOutput.append("<failure message=\"test failed\" type=\"unknown\">\n");
			//			junitOutput.println(result.junitOutput);
			//			junitOutput.println(result.err);
			junitOutput.append("</failure>\n");
		}
		junitOutput.append("</testcase>\n");
	}

	/* (non-Javadoc)
	 * @see org.aspectj.testing.drivers.TestCompleteListener#runStarted(org.aspectj.testing.run.IRunStatus)
	 */
	@Override
	public void runStarting(IRunStatus run) {
		super.runStarting(run);
		startTimeMillis = System.currentTimeMillis();
	}

	/* (non-Javadoc)
	 * @see org.aspectj.testing.drivers.TestCompleteListener#doEndSuite(java.io.File, long)
	 */
	@Override
	protected void doEndSuite(File suite, long duration) {
		super.doEndSuite(suite, duration);
		String suiteName = suite.getName();
		// junit reporter doesn't like ".xml" on the end
		suiteName = suiteName.substring(0,suiteName.indexOf('.'));
		PrintStream out = streamsHandler.getLogStream();
		out.println("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
		String timeStr = new DecimalFormat("#.##").format(duration/1000.0);
		out.print("<testsuite errors=\"" + numFails + "\" failures=\"0\" ");
		out.print("name=\"" + suite.getName() + "\" " );
		out.println("tests=\"" + numTests + "\" time=\"" + timeStr + "\">");
		out.print(junitOutput.toString());
		out.println("</testsuite>");
	}

	/* (non-Javadoc)
	 * @see org.aspectj.testing.drivers.TestCompleteListener#doStartSuite(java.io.File)
	 */
	@Override
	protected void doStartSuite(File suite) {
		super.doStartSuite(suite);
		//		this.suite = suite;
		numTests = 0;
		numFails = 0;
		junitOutput = new StringBuffer();
	}

}

/** log pass and/or failed runs */
class RunLogger extends TestCompleteListener {
	final boolean logStreams;
	final RunUtils.IRunStatusPrinter printer;

	/**
	 * @param printer the component that prints any status - not null
	 * @param runValidator if null, use RunValidator.NORMAL
	 */
	public RunLogger(
			String label,
			boolean logStreams,
			StreamsHandler streamsHandler,
			IRunValidator runValidator,
			RunUtils.IRunStatusPrinter printer) {
		super(label, runValidator, streamsHandler);
		LangUtil.throwIaxIfNull(streamsHandler, "streamsHandler");
		LangUtil.throwIaxIfNull(printer, "printer");
		this.logStreams = logStreams;
		this.printer = printer;
	}

	@Override
	public void doRunCompleted(IRunStatus run, StreamsHandler.Result result) {
		PrintStream out = streamsHandler.getLogStream();
		printer.printRunStatus(out, run);
		if (logStreams) {
			if (!LangUtil.isEmpty(result.err)) {
				out.println("--- error");
				out.println(result.err);
			}
			if (!LangUtil.isEmpty(result.out)) {
				out.println("--- ouput");
				out.println(result.out);
			}
		}
		out.println("");
	}
}

/** trace time and memory between runStaring and runCompleted */
class TestTraceLogger extends TestCompleteListener {
	private static final Runtime runtime = Runtime.getRuntime();
	private long startTime;
	private long startMemoryFree;
	private final boolean verbose;

	public TestTraceLogger(StreamsHandler handler) {
		this(handler, true);
	}
	public TestTraceLogger(StreamsHandler handler, boolean verbose) {
		super("-traceTestsAll", null, handler);
		this.verbose = verbose;
	}
	@Override
	public void runStarting(IRunStatus run) {
		super.runStarting(run);
		startTime = System.currentTimeMillis();
		startMemoryFree = runtime.freeMemory();
	}

	@Override
	public void doRunCompleted(IRunStatus run, StreamsHandler.Result result) {
		long elapsed = System.currentTimeMillis() - startTime;
		long free = runtime.freeMemory();
		long used = startMemoryFree - free;
		String label = run.runResult() ? "PASS " : "FAIL ";
		PrintStream out = streamsHandler.getLogStream();
		if (verbose) {
			label = label
					+ "elapsed: " + LangUtil.toSizedString(elapsed, 7)
					+ " free: " + LangUtil.toSizedString(free, 10)
					+ " used: " + LangUtil.toSizedString(used, 10)
					+ " id: ";
		}
		out.println(label + renderId(run));
	}

	/** @return true - always trace tests */
	protected boolean isFailLabel(String label) {
		return true;
	}

	/** @return true - always trace tests */
	protected boolean isPassLabel(String label) {
		return true;
	}

	/**
	 * This implementation returns run identifier toString().
	 * Subclasses override this to render id as message suffix.
	 */
	protected String renderId(IRunStatus run) {
		return "" + run.getIdentifier();
	}
}
// printing files
//        AjcTest.Spec testSpec = AjcTest.unwrapSpec(run);
//        if (null != testSpec) {
//            CompilerRun.Spec compileSpec = AjcTest.unwrapCompilerRunSpec(testSpec);
//            File dir = new File(testSpec.getSuiteDir(), testSpec.getTestDirOffset());
//            List files = compileSpec.getPathsAsFile(dir);
//            StringBuffer sb = new StringBuffer();
//            for (Iterator iter = files.iterator(); iter.hasNext();) {
//                File file = (File) iter.next();
//                sb.append(" " + file.getPath().replace('\\','/').substring(2));
//            }
//            out.println("files: " + sb);
//        }