Modified: trunk/Tools/ChangeLog (290636 => 290637)
--- trunk/Tools/ChangeLog 2022-03-01 09:36:58 UTC (rev 290636)
+++ trunk/Tools/ChangeLog 2022-03-01 10:11:01 UTC (rev 290637)
@@ -1,3 +1,23 @@
+2022-03-01 Angelos Oikonomopoulos <[email protected]>
+
+ [JSC] Implement chunking for the GNU parallel runner
+ https://bugs.webkit.org/show_bug.cgi?id=237028
+
+ Reviewed by Adrian Perez de Castro.
+
+ The GNU parallel runner can't keep up with the scheduling when using
+ multiple fast remotes. This results in poor CPU utilization on the
+ remotes and more waiting time.
+
+ As a quick fix, allow the user to specify a --gnu-parallel-chunk-size,
+ the value of which specifies the number of test jobs to schedule as a
+ unit (defaults to 1). Specifying a higher value means longer-running
+ jobs, so that GNU parallel can more efficiently schedule them.
+
+ * Scripts/run-_javascript_core-tests:
+ (runJSCStressTests):
+ * Scripts/run-jsc-stress-tests:
+
2022-03-01 Sihui Liu <[email protected]>
Migrate third-party IndexedDB data to GeneralStorageDirectory
Modified: trunk/Tools/Scripts/run-_javascript_core-tests (290636 => 290637)
--- trunk/Tools/Scripts/run-_javascript_core-tests 2022-03-01 09:36:58 UTC (rev 290636)
+++ trunk/Tools/Scripts/run-_javascript_core-tests 2022-03-01 10:11:01 UTC (rev 290637)
@@ -61,6 +61,7 @@
my $makeRunner;
my $rubyRunner;
my $gnuParallelRunner;
+my $gnuParallelChunkSize;
my $testWriter;
my $memoryLimited;
my $reportExecutionTime;
@@ -355,6 +356,7 @@
'make-runner' => \$makeRunner,
'ruby-runner' => \$rubyRunner,
'gnu-parallel-runner' => \$gnuParallelRunner,
+ 'gnu-parallel-chunk-size=s' => \$gnuParallelChunkSize,
'test-writer=s' => \$testWriter,
'memory-limited' => \$memoryLimited,
'report-execution-time' => \$reportExecutionTime,
@@ -887,6 +889,11 @@
push(@jscStressDriverCmd, "--gnu-parallel-runner");
}
+ if ($gnuParallelChunkSize) {
+ push(@jscStressDriverCmd, "--gnu-parallel-chunk-size");
+ push(@jscStressDriverCmd, $gnuParallelChunkSize);
+ }
+
if ($testWriter) {
push(@jscStressDriverCmd, "--test-writer");
push(@jscStressDriverCmd, $testWriter);
Modified: trunk/Tools/Scripts/run-jsc-stress-tests (290636 => 290637)
--- trunk/Tools/Scripts/run-jsc-stress-tests 2022-03-01 09:36:58 UTC (rev 290636)
+++ trunk/Tools/Scripts/run-jsc-stress-tests 2022-03-01 10:11:01 UTC (rev 290637)
@@ -171,6 +171,7 @@
$artifact_exec_wrapper = nil
$numChildProcessesSetByUser = false
$runUniqueId = Random.new.bytes(16).unpack("H*")[0]
+$gnuParallelChunkSize = 1
$testsDebugStream = nil # set to $stderr for debugging
def putd(s)
@@ -245,6 +246,7 @@
['--make-runner', GetoptLong::NO_ARGUMENT],
['--ruby-runner', GetoptLong::NO_ARGUMENT],
['--gnu-parallel-runner', GetoptLong::NO_ARGUMENT],
+ ['--gnu-parallel-chunk-size', GetoptLong::REQUIRED_ARGUMENT],
['--test-writer', GetoptLong::REQUIRED_ARGUMENT],
['--treat-failing-as-flaky', GetoptLong::REQUIRED_ARGUMENT],
['--remote', GetoptLong::REQUIRED_ARGUMENT],
@@ -293,6 +295,8 @@
$testRunnerType = :ruby
when '--gnu-parallel-runner'
$testRunnerType = :gnuparallel
+ when '--gnu-parallel-chunk-size'
+ $gnuParallelChunkSize = arg.to_i
when '--test-writer'
$testWriter = arg
when '--treat-failing-as-flaky'
@@ -2982,6 +2986,20 @@
liveRemotes
end
+def each_chunk(e, chunkSize)
+ accumulator = []
+ e.each { |element|
+ accumulator << element
+ if accumulator.size == chunkSize
+ yield accumulator
+ accumulator = []
+ end
+ }
+ if accumulator.size > 0
+ yield accumulator
+ end
+end
+
class TestRunnerGnuParallel < TestRunner
def prepareGnuParallelRunnerJobs(name, runlist, completedPlans)
path = @runnerDir + name
@@ -2989,13 +3007,15 @@
File.open(path, "w") {
| outp |
- runlist.each {
- | plan |
- if completedPlans.include?(plan)
- next
- end
- outp.puts("./test_script_#{plan.index}")
+ runlist = runlist.select { |plan|
+ not completedPlans.include?(plan)
}
+ each_chunk(runlist, $gnuParallelChunkSize) { |plans|
+ job = plans.collect { |plan|
+ "sh ./test_script_#{plan.index}"
+ }.join("; ")
+ outp.puts(job)
+ }
}
end
def prepareRunner(runlist, serialPlans, completedPlans, remoteHosts)
@@ -3098,7 +3118,7 @@
"if test -e #{$outputDir.basename}/.runner; then cd #{$outputDir.basename}/.runner; else echo #{PARALLEL_REMOTE_STATE_LOST_MARKER}; false; fi && " +
exportBaseEnvironmentVariables(false) +
$envVars.collect { |var | "export #{var} &&"}.join("") +
- "sh "
+ "sh -c "
]
runAndMonitorCommandOutput(cmd) {
| pid, line |
