K. Liddle: testing support

git-svn-id: https://svn.apache.org/repos/asf/xmlgraphics/fop/trunk@194237 13f79535-47bb-0310-9956-ffa450edef68

10 files changed, 461 insertions, 3 deletions

diff --git a/build.sh b/build.sh
index ad0acde7a..f23d25714 100755
--- a/build.sh
+++ b/build.sh
@@ -16,10 +16,16 @@ LIBDIR=lib
 LOCALCLASSPATH=$JAVA_HOME/lib/tools.jar:$JAVA_HOME/lib/classes.zip:$LIBDIR/ant.jar:$LIBDIR/w3c.jar:$LIBDIR/buildtools.jar:$LIBDIR/xerces-1.2.3.jar:$LIBDIR/xalan-2.0.0.jar:$LIBDIR/xalanj1compat.jar:$LIBDIR/bsf.jar:$LIBDIR/jimi-1.0.jar
 ANT_HOME=$LIBDIR
 
-echo Building with classpath $CLASSPATH:$LOCALCLASSPATH
 echo
+CP=$LOCALCLASSPATH
+if [ ! "$CLASSPATH" = "" ]
+then
+CP=$CP:$CLASSPATH
+fi
+echo Building with classpath $CP
+
 
 echo Starting Ant...
 echo
 
-$JAVA_HOME/bin/java -Dant.home=$ANT_HOME -classpath "$LOCALCLASSPATH:$CLASSPATH" org.apache.tools.ant.Main $*
+$JAVA_HOME/bin/java -Dant.home=$ANT_HOME -classpath "$CP" org.apache.tools.ant.Main $*
diff --git a/build.xml b/build.xml
index 38334334d..47f797953 100644
--- a/build.xml
+++ b/build.xml
@@ -267,6 +267,7 @@ Sometimes ant gives out this warnings, but the build is finished without any pro
     <taskdef name="serHyph" classname="org.apache.fop.tools.anttasks.SerializeHyphPattern"/> 
     <taskdef name="xslt" classname="org.apache.fop.tools.anttasks.Xslt"/>
     <taskdef name="manifest" classname="org.apache.fop.tools.anttasks.Manifest"/>
+    <taskdef name="runTest" classname="org.apache.fop.tools.anttasks.RunTest"/>
 
     <property name="main.class" value="org.apache.fop.apps.Fop"/>
     <property name="runtime.classpath" value="lib/xerces-1.2.3.jar lib/xalan-2.0.0.jar lib/xalanj1compat.jar lib/w3c.jar lib/jimi-1.0.jar"/>
@@ -557,6 +558,12 @@ Sometimes ant gives out this warnings, but the build is finished without any pro
          includes="org/**,conf/**,hyph/**"/>
   </target>
 
+  <target name="test" depends="package">
+    <echo message="Testing build in jar file ${build.dir}/${name}.jar against reference"/>
+    <runTest testSuite="basictests.xml" basedir="test/" reference="reference/fop.jar"
+         refVersion="FOP 0.19.0-CVS"/>
+  </target>
+
   <target name="all" depends="package"/> <!-- "all" target for us Makefile converts ;-) -->
 
   <target name="package-jdk11" depends="compile-jdk11,hyphenation">
diff --git a/buildtools.xml b/buildtools.xml
index e68747124..8b3838fb8 100644
--- a/buildtools.xml
+++ b/buildtools.xml
@@ -50,7 +50,7 @@
      <copy todir="${build.src}">
        <fileset dir="${src.dir}"
 	        includes="${tools}/**,${hyph}/**"
-	        excludes="${tools}/anttasks/Fop.java,${hyph}/Hyphenator.java,**/${xalan1},**/${trax}"/>
+	        excludes="${tools}/anttasks/Fop.java,${tools}/TestConverter.java,${hyph}/Hyphenator.java,**/${xalan1},**/${trax}"/>
      </copy>
    </target>
 
diff --git a/src/org/apache/fop/tools/anttasks/RunTest.java b/src/org/apache/fop/tools/anttasks/RunTest.java
new file mode 100644
index 000000000..e051b500f
--- /dev/null
+++ b/src/org/apache/fop/tools/anttasks/RunTest.java
@@ -0,0 +1,146 @@
+/*
+ * Copyright (C) 2001 The Apache Software Foundation. All rights reserved.
+ * For details on use and redistribution please refer to the
+ * LICENSE file included with these sources.
+ */
+
+package org.apache.fop.tools.anttasks;
+
+// Ant
+import org.apache.tools.ant.*;
+
+import java.io.*;
+import java.lang.reflect.*;
+import java.net.URLClassLoader;
+import java.net.URL;
+import java.net.MalformedURLException;
+
+import javax.xml.parsers.*;
+
+import org.w3c.dom.*;
+import org.xml.sax.XMLReader;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+import org.xml.sax.SAXParseException;
+
+/**
+ */
+public class RunTest extends Task {
+    String basedir;
+    String testsuite = "";
+    String referenceJar = "";
+    String refVersion = "";
+
+    public RunTest() {
+    }
+
+    public void setTestSuite(String str)
+    {
+        testsuite = str;
+    }
+
+    public void setBasedir(String str)
+    {
+        basedir = str;
+    }
+
+    public void setReference(String str) {
+        referenceJar = str;
+    }
+
+    public void setRefVersion(String str) {
+        refVersion = str;
+    }
+
+    /**
+     * Execute this ant task.
+     * This creates the reference output, if required, then tests
+     * the current build.
+     */
+    public void execute() throws BuildException {
+        runReference();
+        testNewBuild();
+    }
+
+    /**
+     * Test the current build.
+     * This uses the current jar file (in build/fop.jar) to run the
+     * tests with.
+     * The output is then compared with the reference output.
+     */
+    protected void testNewBuild() {
+        try {
+            ClassLoader loader = new URLClassLoader(new URL[] {new URL("file:build/fop.jar")});
+            runConverter(loader);
+        } catch(MalformedURLException mue) {
+            mue.printStackTrace();
+        }
+    }
+
+    /**
+     * Run the tests for the reference jar file.
+     * This checks that the reference output has not already been
+     * run and then checks the version of the reference jar against
+     * the version required.
+     * The reference output is then created.
+     */
+    protected void runReference() throws BuildException
+    {
+        // check not already done
+            File f = new File(basedir + "reference/output/");
+            if(f.exists()) {
+                return;
+            } else {
+                try {
+                    ClassLoader loader = new URLClassLoader(new URL[] {new URL("file:" + basedir + referenceJar)});
+
+                    try {
+                        Class cla = Class.forName("org.apache.fop.apps.Options", true, loader);
+                        Object opts = cla.newInstance();
+                        cla = Class.forName("org.apache.fop.apps.Version", true, loader);
+                        Method get = cla.getMethod("getVersion", new Class[] {});
+                        if(!get.invoke(null, new Object[] {}).equals(refVersion)) {
+                            throw new BuildException("Reference jar is not correct version");
+                        }
+                    } catch(IllegalAccessException iae) {
+                    } catch(IllegalArgumentException are) {
+                    } catch(InvocationTargetException are) {
+                    } catch(ClassNotFoundException are) {
+                    } catch(InstantiationException are) {
+                    } catch(NoSuchMethodException are) {
+                    }
+ 
+                    runConverter(loader);
+                } catch(MalformedURLException mue) {
+                    mue.printStackTrace();
+                }
+            }
+    }
+
+    /**
+     * Run the Converter.
+     * Runs the test converter using the specified class loader.
+     * This loads the TestConverter using the class loader and
+     * then runs the test suite for the current test suite
+     * file in the base directory.
+     * @param loader the class loader to use to run the tests with
+     */
+    protected void runConverter(ClassLoader loader)
+    {
+        String converter = "org.apache.fop.tools.TestConverter";
+
+        try {
+            Class cla = Class.forName(converter, true, loader);
+            Object tc = cla.newInstance();
+            Method meth;
+
+            meth = cla.getMethod("setBaseDir", new Class[] {String.class});
+            meth.invoke(tc, new Object[] {basedir});
+
+            meth = cla.getMethod("runTests", new Class[] {String.class});
+            meth.invoke(tc, new Object[] {testsuite});
+        } catch(Exception e) {
+            e.printStackTrace();
+        }
+    }
+}
diff --git a/test/Testing b/test/Testing
new file mode 100644
index 000000000..180e73d87
--- /dev/null
+++ b/test/Testing
@@ -0,0 +1,59 @@
+Testing procedure for FOP
+
+(to be written using appropriate xml document)
+
+1) Aim
+
+Need an infrastructure to develop and perform tests.
+Prevent regressions and make checking results easier.
+Quantify the features and conformance.
+
+This must be done in a uniform and simple way.
+
+Ideally the testing and verification should all be automatic.
+
+A new release must not break any previously working features.
+
+Every time a new feature is added then tests should be made that test the feature in all possible situations.
+
+2) Infrastructure
+
+Each test for a feature must be designed to test only that feature and contain no other possible interactions.
+
+Tests should be as simple as possible and demonstrate the feature working.
+
+2.1) Options
+1. For each possible parameter have a file which fully excercises the parameter over all reasonable bounds. This will mean that the fo must be in a format that will ensure each test is independant (which may not always be possible). A typical test will be large and have some overhead.
+
+2. For each individual test have a separate fo file or fo fragment.
+
+The tests are specified in an xml file which is processed using an xsl file (which may simply copy the xml).
+
+This will be done as a build target.
+
+There should be an indication about whether the test is correct for the reference FOP snapshot. This will mean that all changes after the snapshot must not break the test and may improve the result for tests that do not work.
+
+2.2) Details
+
+make a "test" dir at top level
+have a script which can update the code to the snapshot tag, build and create an FOP jar.
+this will then be used as the reference point for comparing results.
+the tests are run and compared with the results from the reference FOP.
+all differences are reported, especially ones that change a test that is marked as working (ie. regressions)
+
+the output will not be compressed so that the pdf markup can be compared using a diff.
+
+Each test must contain a unique id, catagory, a description, the test fo data and pass/fail status.
+
+There will be a number of catagories of test fo files (and possibly sub-catagories).
+These will include conformance, bugtests and system tests.
+
+
+3) Problems
+
+The reference build must be done on a clean cvs build for the appropriate tag.
+The information must be updated appropriately to ensure that the results are correct.
+
+
+
+
diff --git a/test/basictests.xml b/test/basictests.xml
new file mode 100644
index 000000000..9a710d885
--- /dev/null
+++ b/test/basictests.xml
@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE testsuite  SYSTEM "testsuite.dtd">
+<testsuite profile="Basic Conformance Tests">
+   <testcases profile="Block Objects">
+      <testcases profile="alignment" > 
+         <test id="bbo1a1" interaction="none" fo="block" xml="xml/1.xml" xsl="xsl/doc.xsl" level="basic">
+
+6-4-2-1.xml contains only one fo:root element.
+
+
+</test>
+      </testcases>
+      </testcases>
+</testsuite>
diff --git a/test/testsuite.dtd b/test/testsuite.dtd
new file mode 100644
index 000000000..bf32ea23b
--- /dev/null
+++ b/test/testsuite.dtd
@@ -0,0 +1,178 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+    @(#)testcases.dtd 1.0 12/18/00
+
+    Mary Brady, NIST
+
+    DTD describing a database of XSL tests.
+
+    Revised 2001 Jan 10 (Paul Grosso) - augmented by XSL FO SG:
+	- add to the TEST element an XSL attribute to point to
+	  the XSL stylesheet and rename the URI attribute to XML
+	- add to the TEST element a PROPERTY attribute to 
+	  indicate when a given property is being tested
+	- add to the TEST element a LEVEL attribute to indicate
+	  which conformance level is being tested
+	- add to the TEST element a SPECPTR attribute
+	  (of URI type) to allow referring to relevant sections 
+	  of the XSL spec.
+	- add to the TEST element an ERRATUM attribute
+	  (of CDATA type) to allow recording of an erratum
+	  name/number that this test is meant to test.
+	- add to INTERACTION attribute value list MULTIPLE
+	  and give it a default of NONE
+	- add to FO attribute value list MULTIPLE
+	- make the BASE attribute on TESTCASES #IMPLIED, 
+	  since it's not unreasonable for all the URI-type
+	  attributes to be absolute
+	- add to the TEST element a RESULTS attribute to point
+	  to, say, the PDF demonstrating the expected output
+   Revised 2001 Jan 10 (Paul Grosso) - added TESTRESULT structure
+   Revised 2001 Jan 24 (Paul Grosso) - lowercased names (though
+          not in comments, since that helps highlight them)
+
+		2001-02-19 (AntennaHouse) added repeatable-page-master-alternatives 
+		conditional-page-master-reference region-before
+		region-after region-start region-end static-content 
+		inline inline-container.
+
+-->
+
+<!ENTITY % URI "CDATA">
+<!ENTITY % text "(#PCDATA | em | b | a)*">
+
+<!--
+    The root element of the whole collection is TESTSUITE.  While not
+    very different from TESTCASES, it must be distinguished due to
+    improper processing by IE5.
+-->
+<!ELEMENT testsuite ( testcases+ )>
+<!ATTLIST testsuite 
+    profile	CDATA		#IMPLIED
+    >
+
+<!--
+    The root element of a collection should be "TESTCASES".  It serves to 
+    group a set of tests to be collectively identified as follows:
+	
+	* PROFILE - name of test profile 
+	* BASE - base directory in which tests for this collection reside;
+	  allows tests to be found in many application instances.
+ 
+-->
+
+<!ELEMENT testcases (test | testresult | testcases)*>
+<!ATTLIST testcases
+    profile	CDATA		#IMPLIED
+    base	CDATA		#IMPLIED
+    >
+
+<!--
+    The body of each TEST element is its description as well as a
+    discussion of the expected results.  The following attributes 
+    must be specified for each test as follows:
+
+	* ID - unique test identifier
+	* INTERACTION - used later for categorizing tests
+	* FO - formatting object that is tested
+	* XML - relative uri that points to the XML of the actual test
+	* XSL - relative uri that points to the XSL of the actual test
+
+    Other optional attributes are:
+
+	* PROPERTY - indicates the property being tested
+	* LEVEL - indicates which conformance level is being tested
+	* SPECPTR - pointer to relevant part of the XSL spec
+	* ERRATUM-LBL - label/name of the erratum being tested
+	* ERRATUM-PTR - pointer to the erratum being tested
+	* RESULTS - pointer to a display of the expected results
+
+-->
+
+<!ELEMENT test %text;>
+<!ATTLIST test
+    id           ID                             #REQUIRED
+    interaction  (none|area|writing|spacing|collapsing|multiple)	"none"
+    fo        	 (root|page-sequence|page-sequence-master|single-page-master-reference|
+		  repeatable-page-master-reference|repeatable-page-master-alternatives
+|layout-master-set|conditional-page-master-reference|simple-page-master|
+	          region-body|region-before|region-after|region-start|
+	          region-end|static-content|inline | inline-container |
+	          declarations|flow|block|block-container|bidi-override|
+	          character|initial-property-set|external-graphic|leader|page-number|
+		  table-and-caption|table|table-column|table-header|table-body|table-row|
+	          table-cell|list-block|list-item|list-item-body|list-item-label|multi-case|
+		  wrapper|multiple)		        #REQUIRED
+    xml         %URI;                           #REQUIRED
+    xsl         %URI;                           #REQUIRED
+    property	CDATA				#IMPLIED
+    level	(basic|extended|complete)	"basic"
+    specptr     %URI;                           #IMPLIED
+    erratum-lbl CDATA                           #IMPLIED
+    erratum-ptr %URI;                           #IMPLIED
+    results     %URI;                           #IMPLIED
+>
+
+<!--
+    Really basic HTML font tweaks, to support highlighting
+    some aspects of test descriptions ...
+    EM == emphasis (e.g. italics, fun colors)
+    B == bold
+-->
+<!ELEMENT em (#PCDATA | b)*>
+<!ELEMENT b (#PCDATA | em)*>
+
+<!-- 
+    We also allow for hyperlinks in text (e.g., to include
+    references to supporting evidence within SPECPROBLEM and 
+    TESTPROBLEM).
+-->
+<!ELEMENT a (#PCDATA | b | em)*>
+<!ATTLIST a
+    href	%URI;				#REQUIRED
+>
+
+<!--
+    The TESTRESULT element is used to record test results.
+	It has a required ID attribute which identifies the TEST.
+	It has a required AGREEMENT attribute that indicates whether
+	the results are in full agreement with the expected
+	results as described in the TEST case or not.
+
+    The textual contents of the TESTRESULT element should describe
+	the results of the test and any issues or further information.
+
+    The target of the optional RESULTS attribute could be either some
+	PDF showing the results or any other arbitrary resource
+	describing/discussing the results.
+
+    The optional FUTURESUPPORT attribute is used to indicate
+	expected future support of the feature tested by this
+	test case.  
+
+    The optional SPECPROBLEM attribute indicates if there is
+	any ambiguity or other problem found in the spec that relates
+	to this test.  Especially if the results weren't as expected
+	because of a misinterpretation of the spec, this should be
+	documented here.  Details/discussion should appear in the 
+	textual contents of the TESTRESULT element.
+
+    The optional TESTPROBLEM attribute indicates if there is
+	any issue with the TEST case and the expected results it
+	suggests.  Especially if the results of the test differ
+	from that given as "expected" by the test case but are,
+	in fact, believed to be the correct results, this should
+	be documented here.  Details/discussion should appear in the 
+	textual contents of the TESTRESULT element.
+
+-->
+
+<!ELEMENT testresult %text;>
+<!ATTLIST testresult
+    id		CDATA				#REQUIRED
+    agreement	(full|issues)			#REQUIRED
+    results		%URI;			#IMPLIED
+    futuresupport	(full|partial|none)	#IMPLIED
+    specproblem	(yes|no)		"no"
+    testproblem	(yes|no)		"no"
+>
diff --git a/test/xml/1.xml b/test/xml/1.xml
new file mode 100644
index 000000000..0b41cccc3
--- /dev/null
+++ b/test/xml/1.xml
@@ -0,0 +1,9 @@
+<test  xmlns:fo="http://www.w3.org/1999/XSL/Format" xmlns:svg="http://www.w3.org/2000/svg">
+<data>
+
+<fo:block>
+text text text text
+</fo:block>
+
+</data>
+</test>
diff --git a/test/xsl/copy.xsl b/test/xsl/copy.xsl
new file mode 100644
index 000000000..4e3e6109f
--- /dev/null
+++ b/test/xsl/copy.xsl
@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
+  <xsl:template match="/"><xsl:copy-of select="*"/></xsl:template>
+</xsl:stylesheet>
diff --git a/test/xsl/doc.xsl b/test/xsl/doc.xsl
new file mode 100644
index 000000000..71a2cc434
--- /dev/null
+++ b/test/xsl/doc.xsl
@@ -0,0 +1,35 @@
+<?xml version="1.0"?>
+<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:fo="http://www.w3.org/1999/XSL/Format" xmlns:svg="http://www.w3.org/2000/svg">
+	<xsl:output method="xml" indent="yes"/>
+	<xsl:template match="test">
+		<fo:root>
+			<fo:layout-master-set>
+				<fo:simple-page-master master-name="one" page-height="29.7cm" page-width="21cm" margin-top="0.5cm" margin-bottom="0.5cm" margin-left="1.5cm" margin-right="1.5cm">
+					<fo:region-before extent="1.5cm"/>
+					<fo:region-body margin-top="1.5cm" margin-bottom="2cm"/>
+					<fo:region-after extent="1.5cm"/>
+				</fo:simple-page-master>
+			</fo:layout-master-set>
+			<fo:page-sequence master-name="one">
+				<fo:flow font-size="10pt" line-height="10pt" flow-name="xsl-region-body">
+					<xsl:apply-templates select="data"/>
+				</fo:flow>
+			</fo:page-sequence>
+		</fo:root>
+	</xsl:template>
+
+	<xsl:template match="data">
+		<xsl:apply-templates/>
+	</xsl:template>
+
+<!-- note: this causes any node not otherwise defined to be copied -->
+	<xsl:template match="@*|node()">
+		<xsl:copy>
+			<xsl:apply-templates select="@*|node()"/>
+		</xsl:copy>
+	</xsl:template>
+
+	<xsl:template match="title">
+		<xsl:apply-templates/>
+	</xsl:template>
+</xsl:stylesheet>