/* *******************************************************************
* Copyright (c) 2002 Palo Alto Research Center, Incorporated (PARC),
* 2003 Contributors.
* All rights reserved.
* This program and the accompanying materials are made available
* under the terms of the Eclipse Public License v 2.0
* which accompanies this distribution and is available at
* https://www.eclipse.org/org/documents/epl-2.0/EPL-2.0.txt
*
* Contributors:
* Xerox/PARC initial implementation
* Wes Isberg 2003 changes.
* ******************************************************************/
package org.aspectj.testing.drivers;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.io.PrintStream;
import java.io.PrintWriter;
import java.text.DecimalFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.Set;
import java.util.StringTokenizer;
import org.aspectj.bridge.IMessage;
import org.aspectj.bridge.IMessageHolder;
import org.aspectj.bridge.MessageHandler;
import org.aspectj.bridge.MessageUtil;
import org.aspectj.testing.harness.bridge.AbstractRunSpec;
import org.aspectj.testing.harness.bridge.AjcTest;
import org.aspectj.testing.harness.bridge.CompilerRun;
import org.aspectj.testing.harness.bridge.FlatSuiteReader;
import org.aspectj.testing.harness.bridge.IRunSpec;
import org.aspectj.testing.harness.bridge.IncCompilerRun;
import org.aspectj.testing.harness.bridge.JavaRun;
import org.aspectj.testing.harness.bridge.RunSpecIterator;
import org.aspectj.testing.harness.bridge.Sandbox;
import org.aspectj.testing.harness.bridge.Validator;
import org.aspectj.testing.run.IRun;
import org.aspectj.testing.run.IRunIterator;
import org.aspectj.testing.run.IRunListener;
import org.aspectj.testing.run.IRunStatus;
import org.aspectj.testing.run.IRunValidator;
import org.aspectj.testing.run.RunListener;
import org.aspectj.testing.run.RunStatus;
import org.aspectj.testing.run.RunValidator;
import org.aspectj.testing.run.Runner;
import org.aspectj.testing.util.BridgeUtil;
import org.aspectj.testing.util.RunUtils;
import org.aspectj.testing.util.StreamsHandler;
import org.aspectj.testing.util.StreamsHandler.Result;
import org.aspectj.testing.xml.AjcSpecXmlReader;
import org.aspectj.testing.xml.XMLWriter;
import org.aspectj.util.FileUtil;
import org.aspectj.util.LangUtil;
/**
* Test harness for running AjcTest.Suite test suites.
* This can be easily extended by subclassing.
*
*
template algorithms for reading arguments, printing syntax,
* reading suites, and reporting results all
* delegate to methods that subclasses can override to support
* additional arguments or different reporting.
*
implement arbitrary features as IRunListeners
*
support single-option aliases to any number of single-options
*
* See {@link report(IRunStatus, int, int)} for an explanation of test result
* categories.
*/
public class Harness {
/**
* Spaces up to the width that an option should take in the syntax,
* including the two-space leader
*/
protected static final String SYNTAX_PAD = " ";
protected static final String OPTION_DELIM = ";";
private static final String JAVA_VERSION;
private static final String ASPECTJ_VERSION;
static {
String version = "UNKNOWN";
try { version = System.getProperty("java.version", "UNKNOWN"); }
catch (Throwable t) {}
JAVA_VERSION = version;
version = "UNKNOWN";
try {
Class c = Class.forName("org.aspectj.bridge.Version");
version = (String) c.getField("text").get(null);
} catch (Throwable t) {
// ignore
}
ASPECTJ_VERSION = version;
}
/** factory for the subclass currently anointed as default */
public static Harness makeHarness() {
return new FeatureHarness();
}
/** @param args String[] like runMain(String[]) args */
public static void main(String[] args) throws Exception {
if (LangUtil.isEmpty(args)) {
File argFile = new File("HarnessArgs.txt");
if (argFile.canRead()) {
args = readArgs(argFile);
} else {
args = new String[] { "-help" };
}
}
makeHarness().runMain(args, null);
}
/**
* Get known option aliases.
* Subclasses may add new aliases, where the key is the alias option,
* and the value is a comma-delimited String of target options.
* @return Properties with feature aliases or null
*/
protected static Properties getOptionAliases() {
if (null == optionAliases) {
optionAliases = new Properties();
// XXX load from **OptionAliases.properties
}
return optionAliases;
}
/**
* Read argFile contents into String[],
* delimiting at any whitespace
*/
private static String[] readArgs(File argFile) {
ArrayList args = new ArrayList<>();
// int lineNum = 0;
try {
BufferedReader stream =
new BufferedReader(new FileReader(argFile));
String line;
while (null != (line = stream.readLine())) {
StringTokenizer st = new StringTokenizer(line);
while (st.hasMoreTokens()) {
args.add(st.nextToken());
}
}
} catch (IOException e) {
e.printStackTrace(System.err);
}
return args.toArray(new String[0]);
}
/** aliases key="option" value="option{,option}" */
private static Properties optionAliases;
/** be extra noisy if true */
private boolean verboseHarness;
/** be extra quiet if true */
private boolean quietHarness;
/** just don't say anything! */
protected boolean silentHarness;
private Map features;
/** if true, do not delete temporary files. */
private boolean keepTemp;
/** if true, delete temporary files as each test completes. */
private boolean killTemp;
/** if true, then log results in report(..) when done */
private boolean logResults;
/** if true and there were failures, do System.exit({numFailures})*/
private boolean exitOnFailure;
protected Harness() {
features = new HashMap<>();
}
/**
* Entry point for a test.
* This reads in the arguments,
* creates the test suite(s) from the input file(s),
* and for each suite does setup, run, report, and cleanup.
* When arguments are read, any option ending with "-" causes
* option variants, a set of args with and another without the
* option. See {@link LangUtil.optionVariants(String[])} for
* more details.
* @param args the String[] for the test suite - use -help to get options,
* and use "-" suffixes for variants.
* @param resultList List for IRunStatus results - ignored if null
*/
public void runMain(String[] args, List resultList) {
LangUtil.throwIaxIfFalse(!LangUtil.isEmpty(args), "empty args");
// read arguments
final ArrayList globals = new ArrayList<>();
final List files = new ArrayList<>();
final LinkedList argList = new LinkedList<>(Arrays.asList(args));
for (int i = 0; i < argList.size(); i++) {
String arg = argList.get(i);
List aliases = aliasOptions(arg);
if (!LangUtil.isEmpty(aliases)) {
argList.remove(i);
argList.addAll(i, aliases);
i--;
continue;
}
if ("-help".equals(arg)) {
logln("java " + Harness.class.getName() + " {option|suiteFile}..");
printSyntax(getLogStream());
return;
} else if (isSuiteFile(arg)) {
files.add(arg);
} else if (!acceptOption(arg)) {
globals.add(arg);
} // else our options absorbed
}
if (0 == files.size()) {
logln("## Error reading arguments: at least 1 suite file required");
logln("java " + Harness.class.getName() + " {option|suiteFile}..");
printSyntax(getLogStream());
return;
}
String[] globalOptions = globals.toArray(new String[0]);
String[][] globalOptionVariants = optionVariants(globalOptions);
AbstractRunSpec.RT runtime = new AbstractRunSpec.RT();
if (verboseHarness) {
runtime.setVerbose(true);
}
// run suites read from each file
AjcTest.Suite.Spec spec;
for (String string : files) {
File suiteFile = new File(string);
if (!suiteFile.canRead()) {
logln("runMain(..) cannot read file: " + suiteFile);
continue;
}
if (null == (spec = readSuite(suiteFile))) {
logln("runMain(..) cannot read suite from file: " + suiteFile);
continue;
}
MessageHandler holder = new MessageHandler();
for (String[] globalOptionVariant : globalOptionVariants) {
runtime.setOptions(globalOptionVariant);
holder.init();
boolean skip = !spec.adoptParentValues(runtime, holder);
// awful/brittle assumption about number of skips == number of skip messages
final List skipList = MessageUtil.getMessages(holder, IMessage.INFO, false, "skip");
if ((verboseHarness || skip || (0 < skipList.size()))) {
final List curArgs = Arrays.asList(globalOptionVariant);
logln("runMain(" + suiteFile + ", " + curArgs + ")");
if (verboseHarness) {
String format = "yyyy.MM.dd G 'at' hh:mm:ss a zzz";
SimpleDateFormat formatter = new SimpleDateFormat (format);
String date = formatter.format(new Date());
logln("test date: " + date);
logln("harness features: " + listFeatureNames());
logln("Java version: " + JAVA_VERSION);
logln("AspectJ version: " + ASPECTJ_VERSION);
}
if (!(quietHarness || silentHarness) && holder.hasAnyMessage(null, true)) {
MessageUtil.print(getLogStream(), holder, "skip - ");
MessageUtil.printMessageCounts(getLogStream(), holder, "skip - ");
}
}
if (!skip) {
doStartSuite(suiteFile);
long elapsed = 0;
RunResult result = null;
try {
final long startTime = System.currentTimeMillis();
result = run(spec);
if (null != resultList) {
resultList.add(result);
}
elapsed = System.currentTimeMillis() - startTime;
report(result.status, skipList.size(), result.numIncomplete, elapsed);
} finally {
doEndSuite(suiteFile,elapsed);
}
if (exitOnFailure) {
int numFailures = RunUtils.numFailures(result.status, true);
if (0 < numFailures) {
System.exit(numFailures);
}
Object value = result.status.getResult();
if ((value instanceof Boolean)
&& !(Boolean) value) {
System.exit(-1);
}
}
}
}
}
}
/**
* Tell all IRunListeners that we are about to start a test suite
* @param suiteFile
* @param elapsed
*/
private void doEndSuite(File suiteFile, long elapsed) {
Collection c = features.values();
for (Object o : c) {
Feature element = (Feature) o;
if (element.listener instanceof TestCompleteListener) {
((TestCompleteListener) element.listener).doEndSuite(suiteFile, elapsed);
}
}
}
/**
* Generate variants of String[] options by creating an extra set for
* each option that ends with "-". If none end with "-", then an
* array equal to new String[][] { options } is returned;
* if one ends with "-", then two sets are returned,
* three causes eight sets, etc.
* @return String[][] with each option set.
* @throws IllegalArgumentException if any option is null or empty.
*/
public static String[][] optionVariants(String[] options) {
if ((null == options) || (0 == options.length)) {
return new String[][] { new String[0]};
}
// be nice, don't stomp input
String[] temp = new String[options.length];
System.arraycopy(options, 0, temp, 0, temp.length);
options = temp;
boolean[] dup = new boolean[options.length];
int numDups = 0;
for (int i = 0; i < options.length; i++) {
String option = options[i];
if (LangUtil.isEmpty(option)) {
throw new IllegalArgumentException("empty option at " + i);
}
if (option.endsWith("-")) {
options[i] = option.substring(0, option.length()-1);
dup[i] = true;
numDups++;
}
}
final String[] NONE = new String[0];
final int variants = exp(2, numDups);
final String[][] result = new String[variants][];
// variant is a bitmap wrt doing extra value when dup[k]=true
for (int variant = 0; variant < variants; variant++) {
ArrayList next = new ArrayList<>();
int nextOption = 0;
for (int k = 0; k < options.length; k++) {
if (!dup[k] || (0 != (variant & (1 << (nextOption++))))) {
next.add(options[k]);
}
}
result[variant] = next.toArray(NONE);
}
return result;
}
private static int exp(int base, int power) { // not in Math?
if (0 > power) {
throw new IllegalArgumentException("negative power: " + power);
}
int result = 1;
while (0 < power--) {
result *= base;
}
return result;
}
/**
* @param suiteFile
*/
private void doStartSuite(File suiteFile) {
Collection c = features.values();
for (Feature element : c) {
if (element.listener instanceof TestCompleteListener) {
((TestCompleteListener)element.listener).doStartSuite(suiteFile);
}
}
}
/** Run the test suite specified by the spec */
protected RunResult run(AjcTest.Suite.Spec spec) {
LangUtil.throwIaxIfNull(spec, "spec");
/*
* For each run, initialize the runner and validator,
* create a new set of IRun{Iterator} tests,
* and run them.
* Delete all temp files when done.
*/
Runner runner = new Runner();
if (0 != features.size()) {
for (Entry entry : features.entrySet()) {
Feature feature = entry.getValue();
runner.registerListener(feature.clazz, feature.listener);
}
}
IMessageHolder holder = new MessageHandler();
int numIncomplete = 0;
RunStatus status = new RunStatus(holder, runner);
status.setIdentifier(spec);
// validator is used for all setup in entire tree...
Validator validator = new Validator(status);
if (!killTemp) {
validator.lock(this);
}
Sandbox sandbox = null;
try {
sandbox = new Sandbox(spec.getSuiteDirFile(), validator);
IRunIterator tests = spec.makeRunIterator(sandbox, validator);
runner.runIterator(tests, status, null);
if (tests instanceof RunSpecIterator) {
numIncomplete = ((RunSpecIterator) tests).getNumIncomplete();
}
} finally {
if (!keepTemp) {
if (!killTemp) {
validator.unlock(this);
}
validator.deleteTempFiles(verboseHarness);
}
}
return new RunResult(status, numIncomplete);
}
/**
* Report the results of a test run after it is completed.
* Clients should be able to identify the number of:
*
*
tests run and passed
*
tests failed, i.e., run and not passed (fail, error, etc.)
*
tests incomplete, i.e., test definition read but test run setup failed
*
tests skipped, i.e., test definition read and found incompatible with
* the current configuration.
*
*
* @param status returned from the run
* @param numSkipped int tests that were skipped because of
* configuration incompatibilities
* @param numIncomplete int tests that failed during setup,
* usually indicating a test definition or configuration error.
* @param msElapsed elapsed time in milliseconds
* */
protected void report(IRunStatus status, int numSkipped, int numIncomplete,
long msElapsed ) {
if (logResults) {
RunUtils.AJCSUITE_PRINTER.printRunStatus(getLogStream(), status);
} else if (!(quietHarness || silentHarness) && (0 < status.numMessages(null, true))) {
if (!silentHarness) {
MessageUtil.print(getLogStream(), status, "");
}
}
logln(BridgeUtil.childString(status, numSkipped, numIncomplete)
+ " " + (msElapsed/1000) + " seconds");
}
// --------------- delegate methods
protected void logln(String s) {
if (!silentHarness) {
getLogStream().println(s);
}
}
protected PrintStream getLogStream() {
return System.out;
}
protected boolean isSuiteFile(String arg) {
return ((null != arg)
&& (arg.endsWith(".txt") || arg.endsWith(".xml"))
&& new File(arg).canRead());
}
/**
* Get the options that the input option is an alias for.
* Subclasses may add options directly to the getFeatureAliases result
* or override this.
* @return null if the input is not an alias for other options,
* or a non-empty List (String) of options that this option is an alias for
*/
protected List aliasOptions(String option) {
Properties aliases = Harness.getOptionAliases();
if (null != aliases) {
String args = aliases.getProperty(option);
if (!LangUtil.isEmpty(args)) {
return LangUtil.anySplit(args, OPTION_DELIM);
}
}
return null;
}
/**
* Read and implement any of our options.
* Options other than this and suite files will be
* passed down as parent options through the test spec hierarchy.
* Subclasses override this to implement new options.
*/
protected boolean acceptOption(String option) {
// boolean result = false;
if (LangUtil.isEmpty(option)) {
return true; // skip bad input
} else if ("-verboseHarness".equals(option)) {
verboseHarness = true;
} else if ("-quietHarness".equals(option)) {
quietHarness = true;
} else if ("-silentHarness".equals(option)) {
silentHarness = true;
} else if ("-keepTemp".equals(option)) {
keepTemp = true;
} else if ("-killTemp".equals(option)) {
killTemp = true;
} else if ("-logResults".equals(option)) {
logResults = true;
} else if ("-exitOnFailure".equals(option)) {
exitOnFailure = true;
} else {
return false;
}
return true;
}
/**
* Read a test suite file.
* This implementation knows how to read .txt and .xml files
* and logs any errors.
* Subclasses override this to read new kinds of suites.
* @return null if unable to read (logging errors) or AjcTest.Suite.Spec otherwise
*/
protected AjcTest.Suite.Spec readSuite(File suiteFile) {
if (null != suiteFile) {
String path = suiteFile.getPath();
try {
if (path.endsWith(".xml")) {
return AjcSpecXmlReader.getReader().readAjcSuite(suiteFile);
} else if (path.endsWith(".txt")) {
return FlatSuiteReader.ME.readSuite(suiteFile);
} else {
logln("unrecognized extension? " + path);
}
} catch (IOException e) {
e.printStackTrace(getLogStream());
}
}
return null;
}
/** Add feature to take effect during the next runMain(..) invocation.
* @param feature the Feature to add, using feature.name as key.
*/
protected void addFeature(Feature feature) {
if (null != feature) {
features.put(feature.name, feature);
}
}
/** remove feature by name (same as feature.name) */
protected void removeFeature(String name) {
if (!LangUtil.isEmpty(name)) {
features.remove(name);
}
}
/** @return unmodifiable Set of feature names */
protected Set listFeatureNames() {
return Collections.unmodifiableSet(features.keySet());
}
/** print detail message for syntax of main(String[]) command-line */
protected void printSyntax(PrintStream out) {
out.println(" {??} unrecognized options are used as test spec globals");
out.println(" -help print this help message");
out.println(" -verboseHarness harness components log verbosely");
out.println(" -quietHarness harness components suppress logging");
out.println(" -keepTemp do not delete temporary files");
out.println(" -logResults log results at end, verbosely if fail");
out.println(" -exitOnFailure do System.exit({num-failures}) if suite fails");
out.println(" {suiteFile}.xml.. specify test suite XML file");
out.println(" {suiteFile}.txt.. specify test suite .txt file (deprecated)");
}
/** print known aliases at the end of the syntax message */
protected void printAliases(PrintStream out) {
LangUtil.throwIaxIfNull(out, "out");
Map props = getOptionAliases();
if (null == props) {
return;
}
int pdLength = SYNTAX_PAD.length();
Set> entries = props.entrySet();
for (Map.Entry