Diff
Modified: trunk/Tools/ChangeLog (232111 => 232112)
--- trunk/Tools/ChangeLog 2018-05-23 15:24:58 UTC (rev 232111)
+++ trunk/Tools/ChangeLog 2018-05-23 15:33:49 UTC (rev 232112)
@@ -1,3 +1,35 @@
+2018-05-23 Valerie R Young <[email protected]>
+
+ test262/Runner.pm: add unit tests
+ https://bugs.webkit.org/show_bug.cgi?id=185783
+
+ Reviewed by Aakash Jain.
+
+ Add unit tests to be run by test-webkitperl
+
+ * Scripts/test262/Runner.pm:
+ (main):
+ - If harness files cannot be found in a custom, supplies test262 path,
+ then sure the webkit test262 harness files.
+ - Lazily create the test262-results directory.
+ - If an expectation file is supplied via command line, save to the
+ supplied location instead of the detault webkit location.
+ (compileTest):
+ (processResult):
+ (getHarness):
+ * Scripts/webkitperl/test262_unittest/README: Added.
+ * Scripts/webkitperl/test262_unittest/fixtures/expectations-compare.yaml: Added.
+ * Scripts/webkitperl/test262_unittest/fixtures/expectations.yaml: Added.
+ * Scripts/webkitperl/test262_unittest/fixtures/test/expected-to-fail-now-failing-with-new-error.js: Added.
+ * Scripts/webkitperl/test262_unittest/fixtures/test/expected-to-fail-now-failing.js: Added.
+ * Scripts/webkitperl/test262_unittest/fixtures/test/expected-to-fail-now-passing.js: Added.
+ (f):
+ * Scripts/webkitperl/test262_unittest/fixtures/test/expected-to-pass-now-failing.js: Added.
+ * Scripts/webkitperl/test262_unittest/fixtures/test/fail.js: Added.
+ * Scripts/webkitperl/test262_unittest/fixtures/test/pass.js: Added.
+ (f):
+ * Scripts/webkitperl/test262_unittest/test262-runner-tests.pl: Added.
+
2018-05-22 Roy Reapor <[email protected]>
Test infrastructure websites should use jQuery and jQuery UI hosted on webkit.org
Modified: trunk/Tools/Scripts/test262/Runner.pm (232111 => 232112)
--- trunk/Tools/Scripts/test262/Runner.pm 2018-05-23 15:24:58 UTC (rev 232111)
+++ trunk/Tools/Scripts/test262/Runner.pm 2018-05-23 15:33:49 UTC (rev 232112)
@@ -78,7 +78,6 @@
my @cliTestDirs;
my $verbose;
my $JSC;
-my $test262Dir;
my $harnessDir;
my %filterFeatures;
my $ignoreConfig;
@@ -91,16 +90,16 @@
my $runningAllTests;
my $timeout;
+my $test262Dir;
+my $webkitTest262Dir = abs_path("$Bin/../../../JSTests/test262");
my $expectationsFile = abs_path("$Bin/../../../JSTests/test262/expectations.yaml");
my $configFile = abs_path("$Bin/../../../JSTests/test262/config.yaml");
my $resultsDir = $ENV{PWD} . "/test262-results";
-mkpath($resultsDir);
+my $resultsFile;
+my $summaryTxtFile;
+my $summaryFile;
-my $resultsFile = abs_path("$resultsDir/results.yaml");
-my $summaryTxtFile = abs_path("$resultsDir/summary.txt");
-my $summaryFile = abs_path("$resultsDir/summary.yaml");
-
my @results;
my @files;
@@ -107,8 +106,6 @@
my $tempdir = tempdir();
my ($deffh, $deffile) = getTempFile();
-my @default_harnesses;
-
my $startTime = time();
main();
@@ -166,15 +163,26 @@
}
}
- if ($stats) {
- if (! -e $resultsFile) {
- die "Error: cannot find results file to summarize," .
+ if ($stats || $failingOnly) {
+ # If not supplied, try to find the results file in expected directory
+ $resultsFile ||= abs_path("$resultsDir/results.yaml");
+
+ if ($failingOnly && ! -e $resultsFile) {
+ die "Error: cannot find results file to run failing tests," .
"please specify with --results.";
}
- summarizeResults();
- exit;
+
+ if ($stats) {
+ if (! -e $resultsFile) {
+ die "Error: cannot find results file to summarize," .
+ " please specify with --results.";
+ }
+ summarizeResults();
+ exit;
+ }
}
+
if ($JSC) {
$JSC = abs_path($JSC);
# Make sure the path and file jsc exist
@@ -195,12 +203,19 @@
}
if (! $test262Dir) {
- $test262Dir = abs_path("$Bin/../../../JSTests/test262");
+ $test262Dir = $webkitTest262Dir;
} else {
$test262Dir = abs_path($test262Dir);
}
+
$harnessDir = "$test262Dir/harness";
+ if (! -e $harnessDir) {
+ # if the harness directory does not exist in the custom test262 path,
+ # then use the webkits harness directory.
+ $harnessDir = "$webkitTest262Dir/harness";
+ }
+
if (! $ignoreConfig) {
if ($configFile && ! -e $configFile) {
die "Error: Config file $configFile does not exist!\n" .
@@ -212,11 +227,6 @@
}
}
- if ( $failingOnly && ! -e $resultsFile ) {
- die "Error: cannot find results file to run failing tests," .
- " please specify with --results.";
- }
-
if ($specifiedExpectationsFile) {
$expectationsFile = abs_path($specifiedExpectationsFile);
if (! -e $expectationsFile && ! $ignoreExpectations) {
@@ -256,17 +266,19 @@
print "--------------------------------------------------------\n\n";
}
+
sub main {
processCLI();
- @default_harnesses = (
+ my @defaultHarnessFiles = (
"$harnessDir/sta.js",
"$harnessDir/assert.js",
"$harnessDir/doneprintHandle.js",
"$Bin/agent.js"
);
- print $deffh getHarness(<@default_harnesses>);
+ print $deffh getHarness(\@defaultHarnessFiles);
+
# If not commandline test path supplied, use the root directory of all tests.
push(@cliTestDirs, 'test') if not @cliTestDirs;
@@ -389,6 +401,11 @@
}
if ($runningAllTests) {
+ if (! -e $resultsDir) {
+ mkpath($resultsDir);
+ }
+ $resultsFile = abs_path("$resultsDir/results.yaml");
+
DumpFile($resultsFile, \@results);
print "Saved all the results in $resultsFile\n";
summarizeResults();
@@ -592,7 +609,8 @@
my $includes = shift;
my ($tfh, $tfname) = getTempFile();
- my $includesContent = getHarness(map { "$harnessDir/$_" } @{ $includes });
+ my @includes = map { "$harnessDir/$_" } @{ $includes };
+ my $includesContent = getHarness(\@includes);
print $tfh $includesContent;
return ($tfh, $tfname);
@@ -664,10 +682,11 @@
if ($scenario ne 'skip' && $currentfailure) {
- # We have a new failure if we have loaded an expectation file
- # AND (there is no expected failure OR the failure has changed).
- my $isnewfailure = $expect
- && (!$expectedfailure || $expectedfailure ne $currentfailure);
+ # We have a new failure if we haven't loaded an expectation file
+ # (all fails are new) OR we have loaded an expectation fail and
+ # (there is no expected failure OR the failure has changed).
+ my $isnewfailure = ! $expect
+ || !$expectedfailure || $expectedfailure ne $currentfailure;
# Print the failure if we haven't loaded an expectation file
# or the failure is new.
@@ -743,9 +762,10 @@
}
sub getHarness {
- my @files = @_;
+ my ($filesref) = @_;
+
my $content;
- for (@files) {
+ for (@{$filesref}) {
my $file = $_;
open(my $harness_file, '<', $file)
@@ -756,7 +776,7 @@
close $harness_file;
};
- return $content;
+ return $content || '';
}
sub summarizeResults {
@@ -766,6 +786,14 @@
my @rawresults = LoadFile($resultsFile) or die $!;
@results = @{$rawresults[0]};
}
+
+ # Create test262-results folder if it does not exits
+ if (! -e $resultsDir) {
+ mkpath($resultsDir);
+ }
+ $summaryTxtFile = abs_path("$resultsDir/summary.txt");
+ $summaryFile = abs_path("$resultsDir/summary.yaml");
+
my %byfeature;
my %bypath;
Added: trunk/Tools/Scripts/webkitperl/test262_unittest/README (0 => 232112)
--- trunk/Tools/Scripts/webkitperl/test262_unittest/README (rev 0)
+++ trunk/Tools/Scripts/webkitperl/test262_unittest/README 2018-05-23 15:33:49 UTC (rev 232112)
@@ -0,0 +1,6 @@
+These unit tests test:
+ webkit/Tools/Scripts/test262-runner
+ webkit/Tools/Scripts/test262-import
+
+Helper perl modules for these scripts are located in:
+ webkit/Tools/Scripts/test262/
\ No newline at end of file
Added: trunk/Tools/Scripts/webkitperl/test262_unittest/fixtures/expectations-compare.yaml (0 => 232112)
--- trunk/Tools/Scripts/webkitperl/test262_unittest/fixtures/expectations-compare.yaml (rev 0)
+++ trunk/Tools/Scripts/webkitperl/test262_unittest/fixtures/expectations-compare.yaml 2018-05-23 15:33:49 UTC (rev 232112)
@@ -0,0 +1,4 @@
+---
+test/fail.js:
+ default: 'Test262: This test fails.'
+ strict mode: 'Test262: This test fails.'
Added: trunk/Tools/Scripts/webkitperl/test262_unittest/fixtures/expectations.yaml (0 => 232112)
--- trunk/Tools/Scripts/webkitperl/test262_unittest/fixtures/expectations.yaml (rev 0)
+++ trunk/Tools/Scripts/webkitperl/test262_unittest/fixtures/expectations.yaml 2018-05-23 15:33:49 UTC (rev 232112)
@@ -0,0 +1,10 @@
+---
+test/expected-to-fail-now-passing.js:
+ default: 'Test262: This test fails.'
+ strict mode: 'Test262: This test fails.'
+test/expected-to-fail-now-failing.js:
+ default: 'Test262: This test fails.'
+ strict mode: 'Test262: This test fails.'
+test/expected-to-fail-now-failing-with-new-error.js:
+ default: 'Test262: This test fails.'
+ strict mode: 'Test262: This test fails.'
Added: trunk/Tools/Scripts/webkitperl/test262_unittest/fixtures/test/expected-to-fail-now-failing-with-new-error.js (0 => 232112)
--- trunk/Tools/Scripts/webkitperl/test262_unittest/fixtures/test/expected-to-fail-now-failing-with-new-error.js (rev 0)
+++ trunk/Tools/Scripts/webkitperl/test262_unittest/fixtures/test/expected-to-fail-now-failing-with-new-error.js 2018-05-23 15:33:49 UTC (rev 232112)
@@ -0,0 +1,8 @@
+// Copyright (C) 2018 Valerie Young. All rights reserved.
+// This code is governed by the BSD license found in the LICENSE file.
+/*---
+esid: sec-function-definitions
+description: Minimal test
+---*/
+
+throw "Test262Unexpected: This test fails WITH AN UNEXPECTED ERROR.";
Added: trunk/Tools/Scripts/webkitperl/test262_unittest/fixtures/test/expected-to-fail-now-failing.js (0 => 232112)
--- trunk/Tools/Scripts/webkitperl/test262_unittest/fixtures/test/expected-to-fail-now-failing.js (rev 0)
+++ trunk/Tools/Scripts/webkitperl/test262_unittest/fixtures/test/expected-to-fail-now-failing.js 2018-05-23 15:33:49 UTC (rev 232112)
@@ -0,0 +1,8 @@
+// Copyright (C) 2018 Valerie Young. All rights reserved.
+// This code is governed by the BSD license found in the LICENSE file.
+/*---
+esid: sec-function-definitions
+description: Minimal test
+---*/
+
+throw "Test262: This test fails.";
Added: trunk/Tools/Scripts/webkitperl/test262_unittest/fixtures/test/expected-to-fail-now-passing.js (0 => 232112)
--- trunk/Tools/Scripts/webkitperl/test262_unittest/fixtures/test/expected-to-fail-now-passing.js (rev 0)
+++ trunk/Tools/Scripts/webkitperl/test262_unittest/fixtures/test/expected-to-fail-now-passing.js 2018-05-23 15:33:49 UTC (rev 232112)
@@ -0,0 +1,9 @@
+// Copyright (C) 2018 Valerie Young. All rights reserved.
+// This code is governed by the BSD license found in the LICENSE file.
+/*---
+esid: sec-function-definitions
+description: Minimal test
+---*/
+
+function f() {};
+
Added: trunk/Tools/Scripts/webkitperl/test262_unittest/fixtures/test/expected-to-pass-now-failing.js (0 => 232112)
--- trunk/Tools/Scripts/webkitperl/test262_unittest/fixtures/test/expected-to-pass-now-failing.js (rev 0)
+++ trunk/Tools/Scripts/webkitperl/test262_unittest/fixtures/test/expected-to-pass-now-failing.js 2018-05-23 15:33:49 UTC (rev 232112)
@@ -0,0 +1,8 @@
+// Copyright (C) 2018 Valerie Young. All rights reserved.
+// This code is governed by the BSD license found in the LICENSE file.
+/*---
+esid: sec-function-definitions
+description: Minimal test
+---*/
+
+throw "Test262: This test fails.";
Added: trunk/Tools/Scripts/webkitperl/test262_unittest/fixtures/test/fail.js (0 => 232112)
--- trunk/Tools/Scripts/webkitperl/test262_unittest/fixtures/test/fail.js (rev 0)
+++ trunk/Tools/Scripts/webkitperl/test262_unittest/fixtures/test/fail.js 2018-05-23 15:33:49 UTC (rev 232112)
@@ -0,0 +1,8 @@
+// Copyright (C) 2018 Valerie Young. All rights reserved.
+// This code is governed by the BSD license found in the LICENSE file.
+/*---
+esid: sec-function-definitions
+description: Minimal test
+---*/
+
+throw "Test262: This test fails.";
Added: trunk/Tools/Scripts/webkitperl/test262_unittest/fixtures/test/pass.js (0 => 232112)
--- trunk/Tools/Scripts/webkitperl/test262_unittest/fixtures/test/pass.js (rev 0)
+++ trunk/Tools/Scripts/webkitperl/test262_unittest/fixtures/test/pass.js 2018-05-23 15:33:49 UTC (rev 232112)
@@ -0,0 +1,9 @@
+// Copyright (C) 2018 Valerie Young. All rights reserved.
+// This code is governed by the BSD license found in the LICENSE file.
+/*---
+esid: sec-function-definitions
+description: Minimal test
+---*/
+
+function f() {};
+
Added: trunk/Tools/Scripts/webkitperl/test262_unittest/test262-runner-tests.pl (0 => 232112)
--- trunk/Tools/Scripts/webkitperl/test262_unittest/test262-runner-tests.pl (rev 0)
+++ trunk/Tools/Scripts/webkitperl/test262_unittest/test262-runner-tests.pl 2018-05-23 15:33:49 UTC (rev 232112)
@@ -0,0 +1,145 @@
+#!/usr/bin/env perl
+
+# Copyright (C) 2018 Bocoup LLC. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above
+# copyright notice, this list of conditions and the following
+# disclaimer.
+# 2. Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials
+# provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER "AS IS" AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+# OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
+# TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+
+use strict;
+use warnings;
+
+use Test::More;
+use File::Spec;
+use FindBin;
+use Cwd qw(abs_path);
+use File::Path qw(rmtree);
+use File::Temp qw(tempfile);
+use File::Compare qw(compare);
+
+# This test should not be run on Windows
+if ($^O eq 'MSWin32' || $^O eq 'cygwin' || $^O eq 'WinCairo') {
+ plan(tests => 1);
+ is(1, 1, 'do nothing for Windows builds.');
+ exit 0;
+}
+
+my $Bin = $FindBin::Bin;
+my $runner = abs_path("$Bin/../../test262-runner");
+my $mockTest262 = abs_path("$Bin/fixtures");
+
+my @testCases = (
+ {
+ "TESTNAME" => "test262 test failed, ignore expectations",
+ "EXITSTATUS" => 1,
+ "TEST262TESTS_ARG" => "--test-only test/fail.js",
+ "EXPECTATION_ARG" => "--ignore-expectations",
+ "EXPECTED_NEW_FAIL_COUNT" => 2,
+ },
+ {
+ "TESTNAME" => "test262 test passed, ignore expectations",
+ "EXITSTATUS" => 0,
+ "TEST262TESTS_ARG" => "--test-only test/pass.js",
+ "EXPECTATION_ARG" => "--ignore-expectations",
+ "EXPECTED_NEW_FAIL_COUNT" => 0,
+ },
+ {
+ "TESTNAME" => "test262 tests newly failed",
+ "EXITSTATUS" => 1,
+ "TEST262TESTS_ARG" => "--test-only test/expected-to-pass-now-failing.js",
+ "EXPECTATION_ARG" => "--expectations $Bin/fixtures/expectations.yaml",
+ "EXPECTED_NEW_FAIL_COUNT" => 2,
+ },
+ {
+ "TESTNAME" => "test262 tests newly passed",
+ "EXITSTATUS" => 0,
+ "TEST262TESTS_ARG" => "--test-only test/expected-to-fail-now-passing.js",
+ "EXPECTATION_ARG" => "--expectations $Bin/fixtures/expectations.yaml",
+ "EXPECTED_NEW_FAIL_COUNT" => 0,
+ },
+ {
+ "TESTNAME" => "test262 tests fails, expected failure",
+ "EXITSTATUS" => 0,
+ "TEST262TESTS_ARG" => "--test-only test/expected-to-fail-now-failing.js",
+ "EXPECTATION_ARG" => "--expectations $Bin/fixtures/expectations.yaml",
+ "EXPECTED_NEW_FAIL_COUNT" => 0,
+ },
+ {
+ "TESTNAME" => "test262 tests fails, with unexpected error string",
+ "EXITSTATUS" => 1,
+ "TEST262TESTS_ARG" => "--test-only test/expected-to-fail-now-failing-with-new-error.js",
+ "EXPECTATION_ARG" => "--expectations $Bin/fixtures/expectations.yaml",
+ "EXPECTED_NEW_FAIL_COUNT" => 2,
+ },
+);
+
+my $testCasesCount = (scalar(@testCases) * 2) + 1;
+plan(tests => $testCasesCount);
+
+## Test error codes and expected output messages of tests ##
+
+foreach my $testcase (@testCases) {
+
+ my $test = $testcase->{TEST262TESTS_ARG};
+ my $expectation = $testcase->{EXPECTATION_ARG};
+ my $test262loc = "--t262 $mockTest262";
+
+ my $cmd = qq($runner $test262loc $test $expectation);
+ my $output = qx($cmd);
+
+ # Test the resulting exit code
+ my $exitcode = $? >> 8;
+ my $expectedexitcode = $testcase->{EXITSTATUS};
+ my $testname = $testcase->{TESTNAME} . " (exit code: $expectedexitcode)";
+ is($exitcode, $testcase->{EXITSTATUS}, $testname);
+
+ # Test the number of occurences of string "! NEW FAIL"
+ my @newfailcount = $output =~ /! NEW FAIL/g;
+ my $expectednewfailures = $testcase->{EXPECTED_NEW_FAIL_COUNT};
+ $testname = $testcase->{TESTNAME} . " (new failures: $expectednewfailures)";
+ is(scalar(@newfailcount), $expectednewfailures, $testname);
+}
+
+## Test format of saved expectations file ##
+
+my $test262loc = "--t262 $mockTest262";
+my ($expectationsfh, $expectationsfile) = tempfile();
+my $expect = "--expectations $expectationsfile";
+my $test = "--test-only test/fail.js";
+my $cmd = qq($runner --save --ignore-expectations $test262loc $test $expect);
+qx($cmd);
+
+my $expectedexpectationsfile = "$Bin/fixtures/expectations-compare.yaml";
+my $filesmatch = compare($expectationsfile, $expectedexpectationsfile);
+ok($filesmatch == 0, "expectations yaml file format");
+
+close $expectationsfh;
+
+END {
+ # Clean up test262 results directory after running tests
+ my $resultsDir = $ENV{PWD} . "/test262-results";
+ if (-e $resultsDir) {
+ rmtree($resultsDir);
+ }
+}
Property changes on: trunk/Tools/Scripts/webkitperl/test262_unittest/test262-runner-tests.pl
___________________________________________________________________
Added: svn:executable
+*
\ No newline at end of property
