Diff
Modified: trunk/Tools/ChangeLog (275813 => 275814)
--- trunk/Tools/ChangeLog 2021-04-12 03:02:50 UTC (rev 275813)
+++ trunk/Tools/ChangeLog 2021-04-12 06:02:16 UTC (rev 275814)
@@ -1,3 +1,16 @@
+2021-04-11 Commit Queue <[email protected]>
+
+ Unreviewed, reverting r275801.
+ https://bugs.webkit.org/show_bug.cgi?id=224423
+
+ run-jsc-stress-tests output includes a lot of texts
+
+ Reverted changeset:
+
+ "[JSC] detect infrastructure failure for remote stress tests"
+ https://bugs.webkit.org/show_bug.cgi?id=222601
+ https://trac.webkit.org/changeset/275801
+
2021-04-11 Sam Weinig <[email protected]>
Simplify WebKitTestRunner preference reset to be more like DRT
Modified: trunk/Tools/Scripts/run-_javascript_core-tests (275813 => 275814)
--- trunk/Tools/Scripts/run-_javascript_core-tests 2021-04-12 03:02:50 UTC (rev 275813)
+++ trunk/Tools/Scripts/run-_javascript_core-tests 2021-04-12 06:02:16 UTC (rev 275814)
@@ -932,20 +932,9 @@
}
print "\n";
- my @jscStressNoResultList = readAllLines($jscStressResultsDir . "/noresult");
- my $numJSCStressNoResultTests = @jscStressNoResultList;
-
- if ($numJSCStressNoResultTests) {
- $isTestFailed = 1;
- }
- foreach my $testNoResult (@jscStressNoResultList) {
- $reportData{$testNoResult} = {actual => "ERROR"};
- }
-
print "Results for JSC stress tests:\n";
printThingsFound($numJSCStressFailures, "failure", "failures", "found");
- printThingsFound($numJSCStressNoResultTests, "test", "tests", "failed to complete");
- print " OK.\n" if $numJSCStressFailures == 0 and $numJSCStressNoResultTests == 0;
+ print " OK.\n" if $numJSCStressFailures == 0;
print "\n";
Modified: trunk/Tools/Scripts/run-jsc-stress-tests (275813 => 275814)
--- trunk/Tools/Scripts/run-jsc-stress-tests 2021-04-12 03:02:50 UTC (rev 275813)
+++ trunk/Tools/Scripts/run-jsc-stress-tests 2021-04-12 06:02:16 UTC (rev 275814)
@@ -57,9 +57,6 @@
raise unless SCRIPTS_PATH.dirname.basename.to_s == "Tools"
HELPERS_PATH = SCRIPTS_PATH + "jsc-stress-test-helpers"
-STATUS_FILE_PREFIX = "test_status_"
-STATUS_FILE_PASS = "P"
-STATUS_FILE_FAIL = "F"
begin
require 'shellwords'
@@ -137,7 +134,6 @@
$reportExecutionTime = false
$ldd = nil
$artifact_exec_wrapper = nil
-$runUniqueId = Random.new.bytes(16).unpack("H*")[0]
def usage
puts "run-jsc-stress-tests -j <shell path> <collections path> [<collections path> ...]"
@@ -539,6 +535,9 @@
end
end
+$numFailures = 0
+$numPasses = 0
+
# We force all tests to use a smaller (1.5M) stack so that stack overflow tests can run faster.
BASE_OPTIONS = ["--useFTLJIT=false", "--useFunctionDotArguments=true", "--validateExceptionChecks=true", "--useDollarVM=true", "--maxPerThreadStackUsage=1572864"]
EAGER_OPTIONS = ["--thresholdForJITAfterWarmUp=10", "--thresholdForJITSoon=10", "--thresholdForOptimizeAfterWarmUp=20", "--thresholdForOptimizeAfterLongWarmUp=20", "--thresholdForOptimizeSoon=20", "--thresholdForFTLOptimizeAfterWarmUp=20", "--thresholdForFTLOptimizeSoon=20", "--thresholdForOMGOptimizeAfterWarmUp=20", "--thresholdForOMGOptimizeSoon=20", "--maximumEvalCacheableSourceLength=150000", "--useEagerCodeBlockJettisonTiming=true", "--repatchBufferingCountdown=0"]
@@ -1822,6 +1821,7 @@
| outp |
outp.puts plan.name
}
+ $numFailures += 1
end
def appendPass(plan)
@@ -1829,15 +1829,9 @@
| outp |
outp.puts plan.name
}
+ $numPasses += 1
end
-def appendNoResult(plan)
- File.open($outputDir + "noresult", "a") {
- | outp |
- outp.puts plan.name
- }
-end
-
def appendResult(plan, didPass)
File.open($outputDir + "results", "a") {
| outp |
@@ -2043,7 +2037,7 @@
}
end
-def sshRead(cmd, remoteIndex=0, options={})
+def sshRead(cmd, remoteIndex=0)
raise unless $remote
remoteHost = $remoteHosts[remoteIndex]
@@ -2056,7 +2050,7 @@
result += line
}
}
- raise "#{$?}" unless $?.success? or options[:ignoreFailure]
+ raise "#{$?}" unless $?.success?
result
end
@@ -2211,110 +2205,52 @@
end
end
-def getStatusMap
- name_re = /^[.]\/#{STATUS_FILE_PREFIX}(\d+)$/
- map = {}
+def detectFailures
+ raise if $bundle
+ failures = []
if $remote
$remoteHosts.each_with_index {
| host, remoteIndex |
- output = sshRead("cd #{host.remoteDirectory}/#{$outputDir.basename}/.runner && find . -maxdepth 1 -name \"#{STATUS_FILE_PREFIX}*\" -exec sh -c \"printf \\\"%s \\\" {}; cat {}\" \\;", remoteIndex, :ignoreFailure => true)
+ output = sshRead("cd #{host.remoteDirectory}/#{$outputDir.basename}/.runner && find . -maxdepth 1 -name \"test_fail_*\"", remoteIndex)
output.split(/\n/).each {
| line |
- name, run_id, _, result = line.split(' ')
- md = name_re.match(name)
- if md.nil?
- $stderr.puts("Could not parse name in `#{line}`")
- exit(1)
- end
- if run_id != $runUniqueId
- # This may conceivably happen if a remote goes
- # away in the middle of a run and comes back
- # online in the middle of a different run.
- $stderr.puts("Ignoring stale status file for #{name} (ID #{run_id} but current ID is #{$runUniqueId})")
- next
- end
- index = md[1].to_i
- if map.has_key?(index)
- $stderr.puts("Duplicate state file for #{index}")
- # One scenario in which this could happen:
- # Test T runs on remote host A and
- # 1. the status file reaches A's disk
- # 2. somehow the gnu parallel runner is not made aware of the test's completion (packet loss?)
- # 3. A machine crashes
- # 4. gnu parallel re-schedules the test to run on remote host B, where it runs to completion
- # 5. B comes back online before the end of the run
- # 6. we collect the status files from all remotes and end up with two status files for T.
- prev = map[index]
- # map[index] holds
- # - a number, if all results codes we've observed for a test are the same
- # - an array, if they diverge.
- if prev.is_a?(Array)
- prev.push(result)
- elsif prev != result
- # If the two results differ, keep them
- # both. This is simply a way to make note of
- # the divergence (for later reporting).
- map[index] = [prev, result]
- else
- # Got the same result, no need to do anything.
- end
- else
- map[index] = result
- end
+ next unless line =~ /test_fail_/
+ failures << $~.post_match.to_i
}
}
else
Dir.foreach($runnerDir) {
| filename |
- md = name_re.match("./#{filename}")
- next unless md
- File.open("#{$runnerDir}/#{filename}", "r") { |f|
- runId, _, result = f.read.chomp.split(' ')
- if runId != $runUniqueId
- # We clean the dir before a starting a run.
- raise "Can't happen"
- end
- map[md[1].to_i] = result
- }
+ next unless filename =~ /test_fail_/
+ failures << $~.post_match.to_i
}
end
- map
-end
-def detectFailures
- raise if $bundle
- noresult = 0
- statusMap = getStatusMap
+ failureSet = {}
+
+ failures.each {
+ | failure |
+ appendFailure($runlist[failure])
+ failureSet[failure] = true
+ }
+
familyMap = {}
-
$runlist.each_with_index {
| plan, index |
unless familyMap[plan.family]
familyMap[plan.family] = []
end
- if not statusMap.has_key?(index) or statusMap[index].is_a?(Array)
- appendNoResult(plan)
- noresult += 1
+ if failureSet[index]
+ appendResult(plan, false)
+ familyMap[plan.family] << {:result => "FAIL", :plan => plan};
next
- end
- result = nil
- if statusMap[index] == STATUS_FILE_PASS
- appendPass(plan)
- result = "PASS"
else
- appendFailure(plan)
- result = "FAIL"
+ appendResult(plan, true)
+ familyMap[plan.family] << {:result => "PASS", :plan => plan};
end
- appendResult(plan, statusMap[index] == STATUS_FILE_PASS)
- familyMap[plan.family] << {:result => result, :plan => plan }
+ appendPass(plan)
}
- if noresult > 0
- $stderr.puts("Could not get the exit status for #{noresult} tests")
- # We can't change our exit code, as run-_javascript_core-tests
- # expects 0 even when there are failures.
- end
-
File.open($outputDir + "resultsByFamily", "w") {
| outp |
first = true
@@ -2325,7 +2261,7 @@
else
outp.puts
end
-
+
outp.print "#{familyName}:"
numPassed = 0
@@ -2363,7 +2299,6 @@
clean($outputDir + "failed")
clean($outputDir + "passed")
-clean($outputDir + "noresult")
clean($outputDir + "results")
clean($outputDir + "resultsByFamily")
clean($outputDir + ".vm")
Modified: trunk/Tools/Scripts/webkitruby/jsc-stress-test-writer-default.rb (275813 => 275814)
--- trunk/Tools/Scripts/webkitruby/jsc-stress-test-writer-default.rb 2021-04-12 03:02:50 UTC (rev 275813)
+++ trunk/Tools/Scripts/webkitruby/jsc-stress-test-writer-default.rb 2021-04-12 06:02:16 UTC (rev 275814)
@@ -37,7 +37,7 @@
def silentOutputHandler
Proc.new {
| name |
- pipeAndPrefixCommand((Pathname("..") + (name + ".out")).to_s, name)
+ " | " + pipeAndPrefixCommand((Pathname("..") + (name + ".out")).to_s, name)
}
end
@@ -45,24 +45,18 @@
def noisyOutputHandler
Proc.new {
| name |
- "cat > " + Shellwords.shellescape((Pathname("..") + (name + ".out")).to_s)
+ " | cat > " + Shellwords.shellescape((Pathname("..") + (name + ".out")).to_s)
}
end
-def getAndTestExitCode(plan, condition)
- <<-EOF
- if test "$exitCode" #{condition}
-EOF
-end
-
# Error handler for tests that fail exactly when they return non-zero exit status.
# This is useful when a test is expected to fail.
def simpleErrorHandler
Proc.new {
| outp, plan |
- outp.puts getAndTestExitCode(plan, "-ne 0")
+ outp.puts "if test -e #{plan.failFile}"
outp.puts "then"
- outp.puts " (echo ERROR: Unexpected exit code: $exitCode) | " + redirectAndPrefixCommand(plan.name)
+ outp.puts " (echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
outp.puts " " + plan.failCommand
outp.puts "else"
outp.puts " " + plan.successCommand
@@ -74,7 +68,7 @@
def expectedFailErrorHandler
Proc.new {
| outp, plan |
- outp.puts getAndTestExitCode(plan, "-ne 0")
+ outp.puts "if test -e #{plan.failFile}"
outp.puts "then"
outp.puts " " + plan.successCommand
outp.puts "else"
@@ -90,10 +84,10 @@
Proc.new {
| outp, plan |
outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
-
- outp.puts getAndTestExitCode(plan, "-ne 0")
+
+ outp.puts "if test -e #{plan.failFile}"
outp.puts "then"
- outp.puts " (cat #{outputFilename} && echo ERROR: Unexpected exit code: $exitCode) | " + redirectAndPrefixCommand(plan.name)
+ outp.puts " (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
outp.puts " " + plan.failCommand
outp.puts "else"
outp.puts " " + plan.successCommand
@@ -107,10 +101,10 @@
| outp, plan |
outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
diffFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".diff")).to_s)
-
- outp.puts getAndTestExitCode(plan, "-ne 0")
+
+ outp.puts "if test -e #{plan.failFile}"
outp.puts "then"
- outp.puts " (cat #{outputFilename} && echo ERROR: Unexpected exit code: $exitCode) | " + redirectAndPrefixCommand(plan.name)
+ outp.puts " (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
outp.puts " " + plan.failCommand
outp.puts "elif test -e ../#{Shellwords.shellescape(expectedFilename)}"
outp.puts "then"
@@ -136,9 +130,9 @@
| outp, plan |
outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
- outp.puts getAndTestExitCode(plan, "-ne 0")
+ outp.puts "if test -e #{plan.failFile}"
outp.puts "then"
- outp.puts " (cat #{outputFilename} && echo ERROR: Unexpected exit code: $exitCode) | " + redirectAndPrefixCommand(plan.name)
+ outp.puts " (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
outp.puts " " + plan.failCommand
outp.puts "elif grep -i -q failed! #{outputFilename}"
outp.puts "then"
@@ -157,7 +151,7 @@
| outp, plan |
outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
- outp.puts getAndTestExitCode(plan, "-ne 0")
+ outp.puts "if test -e #{plan.failFile}"
outp.puts "then"
outp.puts " " + plan.successCommand
outp.puts "elif grep -i -q failed! #{outputFilename}"
@@ -177,9 +171,9 @@
| outp, plan |
outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
- outp.puts getAndTestExitCode(plan, "-ne 0")
+ outp.puts "if test -e #{plan.failFile}"
outp.puts "then"
- outp.puts " if [ \"$exitCode\" -eq 3 ]"
+ outp.puts " if [ `cat #{plan.failFile}` -eq 3 ]"
outp.puts " then"
outp.puts " if grep -i -q failed! #{outputFilename}"
outp.puts " then"
@@ -189,7 +183,7 @@
outp.puts " " + plan.successCommand
outp.puts " fi"
outp.puts " else"
- outp.puts " (cat #{outputFilename} && echo ERROR: Unexpected exit code: $exitCode) | " + redirectAndPrefixCommand(plan.name)
+ outp.puts " (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
outp.puts " " + plan.failCommand
outp.puts " fi"
outp.puts "else"
@@ -206,9 +200,9 @@
| outp, plan |
outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
- outp.puts getAndTestExitCode(plan, "-ne 0")
+ outp.puts "if test -e #{plan.failFile}"
outp.puts "then"
- outp.puts " (cat #{outputFilename} && echo ERROR: Unexpected exit code: $exitCode) | " + redirectAndPrefixCommand(plan.name)
+ outp.puts " (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
outp.puts " " + plan.failCommand
outp.puts "elif grep -i -q FAILED #{outputFilename}"
outp.puts "then"
@@ -268,29 +262,25 @@
script += "#{shellCommand} || exit 1"
"echo #{Shellwords.shellescape(script)} > #{Shellwords.shellescape((Pathname.new("..") + @name).to_s)}"
end
-
- def statusCommand(status)
- "echo #{$runUniqueId} $exitCode #{status} > #{statusFile}"
- end
-
+
def failCommand
- "#{statusCommand(STATUS_FILE_FAIL)}; echo FAIL: #{Shellwords.shellescape(@name)}; " + reproScriptCommand
+ "echo FAIL: #{Shellwords.shellescape(@name)} ; touch #{failFile} ; " + reproScriptCommand
end
def successCommand
- command = ""
executionTimeMessage = ""
if $reportExecutionTime
executionTimeMessage = " $(($SECONDS - $START_TIME))s"
end
if $progressMeter or $reportExecutionTime or $verbosity >= 2
- command = "echo PASS: #{Shellwords.shellescape(@name)}#{executionTimeMessage}"
+ "rm -f #{failFile} ; echo PASS: #{Shellwords.shellescape(@name)}#{executionTimeMessage}"
+ else
+ "rm -f #{failFile}"
end
- "#{statusCommand(STATUS_FILE_PASS)}; #{command}"
end
- def statusFile
- "#{STATUS_FILE_PREFIX}#{@index}"
+ def failFile
+ "test_fail_#{@index}"
end
def writeRunScript(filename)
@@ -300,31 +290,8 @@
outp.puts "START_TIME=$SECONDS"
end
outp.puts "echo Running #{Shellwords.shellescape(@name)}"
- # From the top down (i.e. reading from the outer _expression_ inwards):
- #
- # - Redirect FD 4 to our stdout
- #
- # - Build a pipe of two command sequences. The
- # right-hand-side sequence reads a number from stdin and
- # exits with it. Since it's the last command in the
- # pipeline, this will be the value of $? after the
- # pipeline completes.
- #
- # - In the left-hand-side sequence, redirect FD 3 to FD 1.
- #
- # - Build a pipe of two commands
- # - run shellCommand, writing its exit code to FD 3.
- # - run the outputHandler, with its stdin coming from
- # the pipe, redirecting its output to FD 4. The
- # outputHandler needs to be in a command sequence
- # (i.e. in { cmd; ...}) as it may do its own
- # redirections.
- #
- # We do all this
- # - to avoid having to use a temporary file for the exit code
- # - to keep within the bounds of POSIX sh (i.e. can't use
- # PIPESTATUS)
- cmd = "{ { { { #{shellCommand}; echo $? >&3; } | { #{outputHandler.call(@name)} ;} >&4; } 3>&1; } | { read xs; exit $xs; } } 4>&1\nexitCode=$?\n"
+ cmd = "(" + shellCommand + " || (echo $? > #{failFile})) 2>&1 "
+ cmd += @outputHandler.call(@name)
if $verbosity >= 3
outp.puts "echo #{Shellwords.shellescape(cmd)}"
end
