Hi again.
This runs not bad, I want to commit it before playing with it further. I don't want to commit the reference results yet, because I'm going to change a little the way exceptions are handled.
Except what is included in the patch, I made svn rm for the following:
cleanup-output.pl testresults.tar.gz alltest.cs
Regards, Andrew.
Index: ignore.lst =================================================================== --- ignore.lst (revision 44608) +++ ignore.lst (working copy) @@ -113,3 +113,5 @@ XSLTFunctions_ElementAvailFunctionTrueTest XSLTFunctions_specialCharInPattern +Keys_PerfRepro2 +Keys_PerfRepro3 Index: Makefile =================================================================== --- Makefile (revision 44628) +++ Makefile (working copy) @@ -1,99 +1,77 @@ # -# "make" prepares automated download and build for both existing xslttest -# and MainSoft tests. +# "make" prepares automated download and build for all tests. # # "make run-test" executes both existing xslttest and MainSoft's new test # and reports the total diff numbers: # -# - For xslttest, it is "TestResult.xml" and -# - For MainSoft test, it is "diffreport.txt" +# - For xslttest, it is "TestResult.xml" and the list of the failed tests +# is in failed.lst # -RUNTIME = mono --debug +.SUFFIXES: .cs .exe + +RUNTIME = mono CSCOMPILE = mcs -TESTS = testsuite/TESTS/Xalan_Conformance_Tests/whitespace/whitespace35.xsl -TARGET_RESULTS = Results -MSTEST_RESULTS = testsuite/TESTS/ExpectedResults + TEST_ARCHIVE = xslt-testsuite-03.ZIP -XSLTTEST_RESULTS_ARCHIVE = xslttest-reference-output.zip -MSTEST_RESULTS_ARCHIVE = mono-xslttest-ExpectedResults-20050214.zip +CATALOG = testsuite/TESTS/catalog.xml +FIXED_CATALOG = testsuite/TESTS/catalog-fixed.xml -test : xslttest.exe alltest.exe prepare +TEST_EXE = xslttest.exe +TEST_FLAGS = --report:TestResult.xml --xml --details --outall $(TEST_DOM) -prepare : $(TESTS) prepare-xslt prepare-mstest +ifdef TEST_DOM +REFERENCE_RESULTS_NAME = domresults +else +REFERENCE_RESULTS_NAME = results +endif +REFERENCE_RESULTS_ARCHIVE = xslt-reference-$(REFERENCE_RESULTS_NAME).tar.gz +REFERENCE_RESULTS_URL = http://svn.myrealbox.com/source/trunk/release/$(REFERENCE_RESULTS_ARCHIVE) +REFERENCE_RESULTS = $(REFERENCE_RESULTS_NAME)/timestamp -prepare-xslt : xslttest.exe $(XSLTTEST_RESULTS) testsuite/TESTS/catalog-fixed.xml +test : $(TEST_EXE) $(FIXED_CATALOG) $(REFERENCE_RESULTS) -prepare-mstest : alltest.exe $(MSTEST_RESULTS) testsuite/TESTS/catalog-fixed.xml +run-test : $(TEST_EXE) $(FIXED_CATALOG) + # Redirect stdout to /dev/null because it has only xsl:message garbage + $(RUNTIME) $(RUNTIME_FLAGS) $(TEST_EXE) $(TEST_FLAGS) >/dev/null -xslttest.exe : xslttest.cs - $(CSCOMPILE) xslttest.cs +clean : + rm -f TestResult.xml failed.lst missing.lst + rm -f $(TEST_EXE) -xmlnorm.exe : XmlNormalizer.cs - $(CSCOMPILE) XmlNormalizer.cs -out:xmlnorm.exe +# Be careful to use it! +distclean : clean + rm -rf testsuite + rm -f $(TEST_ARCHIVE) $(REFERENCE_RESULTS_ARCHIVE) $(REFERENCE_LIST) -alltest.exe : alltest.cs - $(CSCOMPILE) alltest.cs +# Check that we are running on MS.NET - otherwise the reference output can be +# created on mono - and we will compare mono with mono +must-be-dotnet: + uname | grep CYGWIN || uname | grep Windows + +create-reference-output : must-be-dotnet $(TEST_EXE) + ./$(TEST_EXE) $(TEST_DOM) --generate + cd $(REFERENCE_RESULTS_NAME); echo "$(TEST_DOM)" > generate_options # Must cd to work with any path separator + tar -c $(REFERENCE_RESULTS_NAME) | gzip > $(REFERENCE_RESULTS_ARCHIVE) + echo "Now you can upload $(REFERENCE_RESULTS_ARCHIVE) to $(REFERENCE_RESULTS_URL)" -testsuite/TESTS/catalog-fixed.xml : testsuite/TESTS/catalog.xml catalog.sed - sed -f catalog.sed testsuite/TESTS/catalog.xml > testsuite/TESTS/catalog-fixed.xml +.cs.exe : + $(CSCOMPILE) $< -$(TESTS) : $(TEST_ARCHIVE) +$(FIXED_CATALOG) : $(CATALOG) catalog.sed + sed -f catalog.sed $(CATALOG) > $(FIXED_CATALOG) + +$(CATALOG) : $(TEST_ARCHIVE) unzip -un $(TEST_ARCHIVE) - touch $(TESTS) + touch $(CATALOG) -$(XSLTTEST_RESULTS) : $(XSLTTEST_RESULTS_ARCHIVE) - unzip -n -d $(XSLTTEST_RESULTS_ARCHIVE) - touch $(XSLTTEST_RESULTS) - -$(MSTEST_RESULTS) : $(MSTEST_RESULTS_ARCHIVE) - unzip -n -d testsuite/TESTS $(MSTEST_RESULTS_ARCHIVE) - touch $(MSTEST_RESULTS) - $(TEST_ARCHIVE) : wget http://www.oasis-open.org/committees/download.php/9584/$(TEST_ARCHIVE) -$(XSLTTEST_RESULTS_ARCHIVE) : - wget http://monkey.workarea.jp/mono/xml/$(XSLTTEST_RESULTS_ARCHIVE) +$(REFERENCE_RESULTS) : $(REFERENCE_RESULTS_ARCHIVE) + tar -xzf $< + touch $@ -$(MSTEST_RESULTS_ARCHIVE) : - wget http://monkey.workarea.jp/mono/xml/$(MSTEST_RESULTS_ARCHIVE) +$(REFERENCE_RESULTS_ARCHIVE) : + wget $(REFERENCE_RESULTS_URL) -run-test : run-test-xslt run-test-ms - -run-test-xslt : xslttest.exe testsuite/TESTS/catalog-fixed.xml - # Redirect stdout to /dev/null because it has only xsl:message garbage - $(RUNTIME) $(RUNTIME_FLAGS) xslttest.exe --report:TestResult.xml --xml --details --outall >/dev/null - -run-test-ms : alltest.exe xmlnorm.exe testsuite/TESTS/catalog-fixed.xml - $(RUNTIME) $(RUNTIME_FLAGS) ./alltest.exe $(TARGET_RESULTS) - mono ./xmlnorm.exe -s testsuite/TESTS/$(TARGET_RESULTS) testsuite/TESTS/norm-tmp - rm -rf testsuite/TESTS/$(TARGET_RESULTS) - mv testsuite/TESTS/norm-tmp testsuite/TESTS/$(TARGET_RESULTS) - diff -r -q testsuite/TESTS/ExpectedResults testsuite/TESTS/$(TARGET_RESULTS) > diffreport.txt - -clean : clean-xslttest clean-mstest - rm -f xslttest.exe - -clean-xslttest : xslttest.exe - if (test -e testsuite) then \ - $(RUNTIME) ./xslttest.exe --list > output.lst; \ - perl cleanup-output.pl; \ - fi - rm -f TestResult.xml output.lst - -clean-mstest : - rm -rf testsuite/TESTS/Reports - rm -f diffreport.txt alltest.exe xmlnorm.exe - -# Be careful to use it! -distclean : - make clean - rm -rf testsuite - rm -f $(TEST_ARCHIVE) $(XSLTTEST_RESULTS_ARCHIVE) $(MSTEST_RESULTS_ARCHIVE) - -create-reference-output : clean-xslttest xslttest.exe - ./xslttest.exe --generate - ./xslttest.exe --list > output.lst - zip xslttest-reference-output.zip -@ < output.lst - Index: xslttest.cs =================================================================== --- xslttest.cs (revision 44628) +++ xslttest.cs (working copy) @@ -17,7 +17,7 @@ static bool useDomStyle; static bool useDomInstance; static bool generateOutput; - static bool listOutput; + static string outputDir; static bool whitespaceStyle; static bool whitespaceInstance; static bool stopImmediately; @@ -31,6 +31,14 @@ static StreamWriter failedTests = new StreamWriter ("failed.lst"); #endregion + enum TestResult + { + Crash, //exception + Failure, //no exception but output is different + Unknown, //no exception but expected result is unknown + Success, //no exception and output is as expected + }; + static XsltTest () { skipTargets = new ArrayList (new string [] { @@ -50,7 +58,6 @@ --generate Generate output files specified in catalog. Use this feature only when you want to update reference output. - --list Print output list to console. --noExclude Don't exclude meaningless comparison testcases. --outall Output fine results as OK (omitted by default). --stoponerror Stops the test process and throw detailed @@ -101,9 +108,6 @@ case "--generate": generateOutput = true; break; - case "--list": - listOutput = true; - break; case "--noExclude": noExclude = true; break; @@ -145,6 +149,10 @@ break; } } + if (useDomStyle || useDomInstance) + outputDir = "domresults"; + else + outputDir = "results"; } static void RunMain (string [] args) @@ -172,17 +180,9 @@ XmlDocument whole = new XmlDocument (); whole.Load (@"testsuite/TESTS/catalog-fixed.xml"); - if (!listOutput) - Console.Error.WriteLine ("Started: " - + DateTime.Now.ToString ("yyyyMMdd-HHmmss.fff")); - foreach (XmlElement testCase in whole.SelectNodes ("test-suite/test-catalog/test-case")) ProcessTestCase (testCase); - if (!listOutput) - Console.Error.WriteLine ("Finished: " - + DateTime.Now.ToString ("yyyyMMdd-HHmmss.fff")); - if (reportAsXml) reportXmlWriter.WriteEndElement (); // test-results } @@ -205,37 +205,40 @@ else return; //unknown directory - string path = @"testsuite/TESTS/" + testAuthorDir + "/" + filePath + "/"; + string relPath = Path.Combine (testAuthorDir, filePath); + string path = Path.Combine ("testsuite/TESTS", relPath); + string outputPath = Path.Combine (outputDir, relPath); + if (!Directory.Exists (outputPath)) + Directory.CreateDirectory (outputPath); foreach (XmlElement scenario in testCase.SelectNodes ("[EMAIL PROTECTED]'standard']")) { - RunTest (testid, scenario, path, stylesheetBase); + RunTest (testid, scenario, path, outputPath, stylesheetBase); } } catch (Exception ex) { if (stopImmediately) throw; - Report (false, testid, "Exception: " + ex.Message); + Report (TestResult.Crash, testid, "Exception: " + ex.Message); } } - static void RunTest (string testid, XmlElement scenario, string path, string stylesheetBase) + static void RunTest (string testid, XmlElement scenario, string path, string outputPath, + string stylesheetBase) { - stylesheetBase = scenario.SelectSingleNode ("[EMAIL PROTECTED]'principal-stylesheet']").InnerText; - string stylesheet = path + stylesheetBase; + stylesheetBase = scenario.SelectSingleNode ("[EMAIL PROTECTED]'principal-stylesheet']") + .InnerText; + string stylesheet = Path.Combine (path, stylesheetBase); if (!File.Exists (stylesheet)) { missingFiles.WriteLine (stylesheet); missingFiles.Flush (); } - string srcxml = path + scenario.SelectSingleNode ("[EMAIL PROTECTED]'principal-data']").InnerText; + string srcxml = Path.Combine (path, + scenario.SelectSingleNode ("[EMAIL PROTECTED]'principal-data']").InnerText); XmlNode outputNode = scenario.SelectSingleNode ("[EMAIL PROTECTED]'principal']"); - string outfile = outputNode != null ? path + outputNode.InnerText : null; + string outfile = null; + if (outputNode != null) + outfile = Path.Combine (outputPath, outputNode.InnerText); - if (listOutput) { - if (outfile != null) - Console.Error.WriteLine (outfile); - return; - } - XslTransform trans = new XslTransform (); if (explicitTarget != null && testid.IndexOf (explicitTarget) < 0) @@ -276,55 +279,63 @@ if (generateOutput) { StreamWriter fw = new StreamWriter (outfile, false, Encoding.UTF8); - fw.Write (sw.ToString ()); fw.Close (); + Report (TestResult.Success, testid, "Created reference result"); // ... and don't run comparison return; } if (!File.Exists (outfile)) { // Reference output file does not exist. - Report (true, testid, "No reference file found"); + Report (TestResult.Unknown, testid, "No reference file found"); return; } StreamReader sr = new StreamReader (outfile); - string reference_out = sr.ReadToEnd (); - string actual_out = sw.ToString (); + string reference_out = sr.ReadToEnd ().Replace ("\r\n","\n"); + string actual_out = sw.ToString ().Replace ("\r\n","\n"); + if (reference_out != actual_out) - Report (false, testid, reference_out, actual_out); + Report (TestResult.Failure, testid, reference_out, actual_out); else if (outputAll) - Report (true, testid, "OK"); + Report (TestResult.Success, testid, "OK"); } - static void Report (bool passed, string testid, string message) + static void Report (TestResult res, string testid, string message) { - if (passed) { + if (TestResult.Success == res) { Console.Error.Write ("."); return; } - failedTests.WriteLine (testid + "\t" + message); + else if (TestResult.Unknown == res) { + Console.Error.Write ("?"); + return; + } + + failedTests.WriteLine (testid + "\t" + message); failedTests.Flush (); + if (reportAsXml) { reportXmlWriter.WriteStartElement ("testcase"); reportXmlWriter.WriteAttributeString ("id", testid); reportXmlWriter.WriteString (message); reportXmlWriter.WriteEndElement (); - if (knownFailures.Contains (testid)) - Console.Error.Write ("k"); - else - Console.Error.Write ("E"); } + + if (knownFailures.Contains (testid)) + Console.Error.Write ("k"); + else if (TestResult.Crash == res) + Console.Error.Write ("E"); else - reportOutput.WriteLine (message); + Console.Error.Write ("e"); } - static void Report (bool passed, string testid, string reference_out, string actual_out) + static void Report (TestResult res, string testid, string reference_out, string actual_out) { string baseMessage = reportAsXml ? "Different." : "Different: " + testid; if (!reportDetails) - Report (passed, testid, baseMessage); + Report (res, testid, baseMessage); else - Report (passed, testid, baseMessage + + Report (res, testid, baseMessage + "\n Actual*****\n" + actual_out + "\n-------------------\nReference*****\n" +
_______________________________________________ Mono-devel-list mailing list Mono-devel-list@lists.ximian.com http://lists.ximian.com/mailman/listinfo/mono-devel-list
