Modified: trunk/Tools/Scripts/run-_javascript_core-tests (252489 => 252490)
--- trunk/Tools/Scripts/run-_javascript_core-tests 2019-11-15 18:00:42 UTC (rev 252489)
+++ trunk/Tools/Scripts/run-_javascript_core-tests 2019-11-15 19:18:52 UTC (rev 252490)
@@ -57,6 +57,12 @@
my $testWriter;
my $memoryLimited;
+my $report;
+my $buildbotMaster;
+my $builderName;
+my $buildNumber;
+my $buildbotWorker;
+
my $buildJSC = 1;
my $copyJSC = 1;
@@ -91,8 +97,12 @@
my $createTarball = 0;
my $remoteHost = 0;
my $model = 0;
+my $version;
+my $versionName;
+my $sdk;
my $failFast = 1;
my %jsonData = ();
+my %reportData = ();
my @testResults = ();
my $isTestFailed = 0;
my $remoteConfigFile;
@@ -231,6 +241,9 @@
--remote= Run the JSC stress tests on the specified remote host. Implies --tarball.
--remote-config-file= Same as remote, but read config from JSON file.
--model= Specify remote hardware model, this info used for determine what jsc tests should run on remote
+ --version Specify the version number of the device running tests.
+ --version-name Specify the version name of the hardware running tests.
+ --sdk Specific SDK or OS version of the form ##*###
--extra-tests= Path to a file containing extra tests
--child-processes= Specify the number of child processes.
--shell-runner Uses the shell-based test runner instead of the default make-based runner.
@@ -252,6 +265,12 @@
--gmalloc: Run tests with Guard Malloc enabled (if no path is given: $gmallocDefaultPath is used)
--verbose: Verbose output (specify more than once to increase verbosity).
+ --report: Results database url to report results to.
+ --buildbot-master: The url of the buildbot master.
+ --builder-name: The name of the buildbot builder tests were run on.
+ --build-number: The buildbot build number tests are associated with.
+ --buildbot-worker: The buildbot worker tests were run on.
+
Environment Variables:
- set RUN_JAVASCRIPTCORE_TESTS_TESTMASM to "true" or "false" (no quotes) to determine if we run testmasm by default.
- set RUN_JAVASCRIPTCORE_TESTS_TESTAIR to "true" or "false" (no quotes) to determine if we run testair by default.
@@ -289,6 +308,9 @@
'tarball!' => \$createTarball,
'remote=s' => \$remoteHost,
'model=s' => \$model,
+ 'version=s' => \$version,
+ 'version-name=s' => \$versionName,
+ 'sdk=s' => \$sdk,
'remote-config-file=s' => \$remoteConfigFile,
'child-processes=s' => \$childProcesses,
'shell-runner' => \$shellRunner,
@@ -300,7 +322,12 @@
'help' => \$showHelp,
'env-vars=s' => \$envVars,
'gmalloc:s' => \$gmallocPath,
- 'verbose+' => \$verbose
+ 'verbose+' => \$verbose,
+ 'report=s' => \$report,
+ 'buildbot-master=s' => \$buildbotMaster,
+ 'builder-name=s' => \$builderName,
+ 'build-number=s' => \$buildNumber,
+ 'buildbot-worker=s' => \$buildbotWorker,
);
@@ -315,6 +342,10 @@
$specificTestsSpecified = 1;
}
+if ($version) {
+ $version = splitVersionString($version);
+}
+
sub enableTestOrNot {
my ($state) = @_;
if ($state == RUN_IF_NO_TESTS_SPECIFIED || $state == ENV_VAR_SAYS_DO_RUN) {
@@ -325,6 +356,86 @@
return $state;
}
+sub configurationForUpload()
+{
+ my $platform;
+ my $sdk;
+ my $simulator = 0;
+
+ if (index(xcodeSDKPlatformName(), "simulator") != -1) {
+ $simulator = 1;
+ }
+
+ if (index($model, 'iPhone') != -1 || index($model, 'iPad') != -1) {
+ $platform = 'ios';
+ if (!$version) {
+ $version = iosVersion();
+ }
+ } elsif (index($model, 'watch') != -1 || index(xcodeSDKPlatformName(), "watch") != -1) {
+ $platform = 'watchos';
+ die "No watchOS version specified" if !$version;
+ } elsif (index(xcodeSDKPlatformName(), "appletv") != -1) {
+ $platform = 'tvos';
+ die "No tvOS version specified" if !$version;
+ } elsif (isGtk()) {
+ $platform = 'GTK';
+ if (!$version) {
+ chomp($version = `/usr/bin/uname -r`);
+ $version = splitVersionString($version);
+ }
+ } elsif (isWPE()) {
+ $platform = 'WPE';
+ if (!$version) {
+ chomp($version = `/usr/bin/uname -r`);
+ $version = splitVersionString($version);
+ }
+ } elsif (isAnyWindows()) {
+ $platform = 'win';
+ if (!$version) {
+ $version = winVersion();
+ }
+ } elsif (isAppleMacWebKit()) {
+ $platform = 'mac';
+ chomp($model = `/usr/sbin/sysctl -n hw.model`);
+ if (!$version) {
+ $version = osXVersion();
+ }
+ if (!$sdk) {
+ chomp($sdk = `/usr/bin/sw_vers -buildVersion`);
+ }
+
+ if (!$versionName) {
+ if ($version->{minor} eq 15) {
+ $versionName = "Catalina";
+ } elsif ($version->{minor} eq 14) {
+ $versionName = "Mojave";
+ } elsif ($version->{minor} eq 13) {
+ $versionName = "High Sierra";
+ }
+ }
+ } else {
+ die "Cannot determine platform\n";
+ }
+
+ my $result = {
+ platform => $platform,
+ architecture => architecture(),
+ is_simulator => $simulator,
+ style => lc(configuration()),
+ version => "$version->{major}.$version->{minor}.$version->{subminor}",
+ };
+ if ($model) {
+ $result->{model} = $model;
+ }
+ if ($versionName) {
+ $result->{version_name} = $versionName;
+ }
+ if ($sdk) {
+ $result->{sdk} = $sdk;
+ }
+ return $result;
+}
+
$runTestMasm = enableTestOrNot($runTestMasm);
$runTestAir = enableTestOrNot($runTestAir);
$runTestB3 = enableTestOrNot($runTestB3);
@@ -342,6 +453,7 @@
}
setConfigurationProductDir(Cwd::abs_path($root)) if (defined($root));
+configurationForUpload() if (defined($report));
if (defined($jsonFileName)) {
$jsonFileName = File::Spec->rel2abs($jsonFileName);
@@ -374,6 +486,9 @@
$ENV{TZ}="US/Pacific"; # Some tests fail if the time zone is not set to US/Pacific (<https://webkit.org/b/136363>)
setPathForRunningWebKitApp(\%ENV) if isCygwin();
+my $startTime = time();
+my $endTime = $startTime;
+
sub testPath {
my ($productDir, $testName) = @_;
$testName .= "_debug" if configuration() eq "Debug_All";
@@ -397,9 +512,83 @@
}
}
- # Use an "indirect object" so that system() won't get confused if the path
- # contains spaces (see perldoc -f exec).
- my $testResult = system { $command[0] } @command;
+ my $testResult = 0;
+ my $failureFlag = 0;
+ my $lastOptimizeLevel;
+
+ open(TEST, "-|", "@command 2>&1") or die "Failed to run @command";
+ while ( my $line = <TEST> ) {
+ my $subTestName;
+ print $line;
+
+ # e.g.: 'O2: testRotRWithImmShift(uint64-max, uint32-min): OK!''
+ if ($line =~ /(O\d): (\S+)(\(.*\)): ([A-Za-z]+)!\n/) {
+ $lastOptimizeLevel = $1;
+ $subTestName = "$testName.$1.$2$3";
+ $failureFlag += 1 if ($4 ne "OK");
+ }
+ # Assertion in binary
+ # e.g.: ASSERTION FAILED: things[0] == 2
+ elsif ($line =~ /ASSERTION FAILED:/) {
+ $failureFlag += 1;
+ next;
+ }
+ # e.g.: file/path(397) : void (anonymous namespace)::testShuffleSimpleBroadcast()
+ elsif ($failureFlag && $line =~ /.*\(\d+\) : \S+ .*::(\S+)(\(.*\))/) {
+ if (defined $lastOptimizeLevel) {
+ $subTestName = "$testName.$lastOptimizeLevel.$1$2";
+ } else {
+ $subTestName = "$testName.$1$2";
+ }
+ }
+ # sub-test from the c-api, of the form:
+ # e.g.: 'testapi[15422:16904790] parallelPromiseResolveTest(): done.''
+ elsif ($line =~ /\[\d+:\d+\] (\S+)(): done\.\n/) {
+ $subTestName = "$testName.objective-c.$1";
+ }
+ # End of C-API tests (meaning we just finished the ungrouped main tests)
+ elsif ($line =~ /Starting C-API tests in C++/) {
+ $subTestName = "$testName.objective-c.main";
+ }
+ # End of testapi
+ # e.g.: FAIL: Some tests failed.
+ elsif ($line =~ /FAIL: Some tests failed\./) {
+ $failureFlag = 0;
+ next;
+ }
+ # Test cases inside API tests
+ # e.g.: 'testapi[15422:16904790] TEST: "2 + 2": PASSED'
+ elsif ($line =~ /\[\d+:\d+\] TEST: ".*": ([A-Z]+)\n/) {
+ $failureFlag += 1 if ($1 ne "PASSED" && $1 ne "PASS");
+ next;
+ }
+ # e.g.: 'testBitXorAndAndArgs(0, 0,uint32-max): OK!''
+ elsif ($line =~ /(\S+)(\(.*\)): ([A-Za-z]+)!\n/) {
+ $subTestName = "$testName.$1$2";
+ $failureFlag += 1 if ($3 ne "OK");
+ }
+ # General testcase statement, filtered to exclude some junk logging
+ # e.g.: 'PASS: DFG script timed out as expected when no callback is specified.''
+ elsif ($line =~ /([A-Z]+): (.*)\.*\n/ && !($line =~ /[A-Z]+: \d+\n/ || $line =~ /\[\d+:\d+\] [A-Z]+:/)) {
+ $subTestName = "$testName.$2";
+ $failureFlag += 1 if ($1 ne "PASS");
+ }
+ # e.g.: 'test description: PASSED'
+ elsif ($line =~ /.*: ([A-Z]+)\n/) {
+ $failureFlag += 1 if ($1 ne "PASSED" && $1 ne "PASS");
+ next;
+ } else {
+ next;
+ }
+
+ if (!defined $reportData{$subTestName}) {
+ print("$subTestName $failureFlag\n") if $failureFlag;
+ $reportData{$subTestName} = $failureFlag ? {actual => "FAIL"} : {actual => "PASS"};
+ }
+ $failureFlag = 0;
+ }
+ $testResult = close(TEST) ? 0 : $?;
+
my $exitStatus = exitStatus($testResult);
print "$testName completed with rc=$testResult ($exitStatus)\n\n";
@@ -415,7 +604,10 @@
if ($testResult && $failFast) {
reportTestFailures();
writeJsonDataIfApplicable();
- exit exitStatus($testResult);
+
+ $endTime = time();
+ uploadResults();
+ exit $exitStatus;
}
}
@@ -446,9 +638,13 @@
# Find _javascript_Core directory
chdirWebKit();
+# FIXME: report stress test failures https://bugs.webkit.org/show_bug.cgi?id=204096
runJSCStressTests();
reportTestFailures();
+$endTime = time();
+uploadResults();
+
if ($isTestFailed) {
exit(1);
}
@@ -666,3 +862,61 @@
close($fileHandler);
}
}
+
+sub uploadResults
+{
+ if (not defined $report) {
+ print "Skipping upload to results database since no report URL was specified\n";
+ return 0;
+ }
+
+ my %upload = (
+ version => 0,
+ suite => '_javascript_core-tests',
+ # FIXME: Integrate branches, https://bugs.webkit.org/show_bug.cgi?id=204094
+ # FIXME: Integrate the Safari repository, https://bugs.webkit.org/show_bug.cgi?id=204095
+ commits => [{
+ repository_id => 'webkit',
+ id => determineCurrentSVNRevision(),
+ }],
+ configuration => configurationForUpload(),
+ test_results => {
+ run_stats => {
+ start_time => $startTime,
+ end_time => $endTime,
+ tests_skipped => 0,
+ },
+ results => \%reportData,
+ },
+ timestamp => $startTime,
+ );
+ if (defined $buildbotMaster and defined $builderName and defined $buildNumber and defined $buildbotWorker) {
+ $upload{test_results}{details} = {
+ buildbot_master => $buildbotMaster,
+ builder_name => $builderName,
+ build_number => $buildNumber,
+ buildbot_worker => $buildbotWorker,
+ };
+ } else {
+ print " No buildbot details provided, test run will not be linked to CI system\n";
+ }
+ if (defined $ENV{'RESULTS_SERVER_API_KEY'}) {
+ $upload{'api_key'} = $ENV{'RESULTS_SERVER_API_KEY'};
+ }
+
+ print "Uploading results to $report\n";
+
+ open(HANDLE, "|-", "curl -X POST $report/api/upload -H 'Content-Type: application/json' -f -d '\@-'") or die "Failed to open curl";
+
+ # Json conforming to https://results.webkit.org/documentation#API-Uploads.
+ my $encodedUpload = encode_json(\%upload);
+ print HANDLE "$encodedUpload\n\0";
+ my $success = close HANDLE;
+
+ if ($success) {
+ print "Upload successful!\n";
+ return 0;
+ }
+ print "Upload to $report failed\n";
+ return 1;
+}
Modified: trunk/Tools/Scripts/webkitdirs.pm (252489 => 252490)
--- trunk/Tools/Scripts/webkitdirs.pm 2019-11-15 18:00:42 UTC (rev 252489)
+++ trunk/Tools/Scripts/webkitdirs.pm 2019-11-15 19:18:52 UTC (rev 252490)
@@ -1563,10 +1563,10 @@
my $versionString = shift;
my @splitVersion = split(/\./, $versionString);
@splitVersion >= 2 or die "Invalid version $versionString";
- $osXVersion = {
- "major" => $splitVersion[0],
- "minor" => $splitVersion[1],
- "subminor" => (defined($splitVersion[2]) ? $splitVersion[2] : 0),
+ return {
+ "major" => $splitVersion[0],
+ "minor" => $splitVersion[1],
+ "subminor" => (defined($splitVersion[2]) ? $splitVersion[2] : 0),
};
}
@@ -1579,7 +1579,7 @@
return;
}
- my $versionString = `sw_vers -productVersion`;
+ chomp(my $versionString = `sw_vers -productVersion`);
$osXVersion = splitVersionString($versionString);
}
