Repository: kudu Updated Branches: refs/heads/master 2ff946030 -> 296b05ef4
Add support for running Java tests in dist-test Adds support for running Java tests in dist-test by: - Providing a way to gather the c++ âbaseâ dependencies for the Java test in dist_test.py - Adding a Java subparser to dist_test.py that can run or loop Java tests. - Adding a task to the Gradle build that generates the needed .isolate and .gen.json files. This task is called from dist_test.py. - Adjusting run_dist_test.py to find the Java binaries and run Java tests. Sample usage: $ python build-support/dist_test.py java run-all $ python build-support/dist_test.py java loop --num-instances 10 *AsyncKuduSession* Change-Id: I446a15192a45e296b323a4c7d305f236e22ab557 Reviewed-on: http://gerrit.cloudera.org:8080/10907 Reviewed-by: Adar Dembo <[email protected]> Tested-by: Grant Henke <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/f5117d29 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/f5117d29 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/f5117d29 Branch: refs/heads/master Commit: f5117d294a32ea9e4035bb4f8e9fb376dea68646 Parents: 2ff9460 Author: Grant Henke <[email protected]> Authored: Thu Jul 12 11:10:43 2018 -0500 Committer: Grant Henke <[email protected]> Committed: Wed Jul 18 14:57:07 2018 +0000 ---------------------------------------------------------------------- build-support/dist_test.py | 74 ++++- build-support/java-home-candidates.txt | 54 ++++ build-support/run_dist_test.py | 42 ++- cmake_modules/FindJavaHome.cmake | 38 +-- java/build.gradle | 26 ++ java/buildSrc/build.gradle | 1 + .../org/apache/kudu/gradle/DistTestTask.java | 285 +++++++++++++++++++ 7 files changed, 468 insertions(+), 52 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kudu/blob/f5117d29/build-support/dist_test.py ---------------------------------------------------------------------- diff --git a/build-support/dist_test.py b/build-support/dist_test.py index 1ca3d75..01ab478 100755 --- a/build-support/dist_test.py +++ b/build-support/dist_test.py @@ -78,6 +78,7 @@ DEPS_FOR_ALL = \ "build-support/run_dist_test.py", "build-support/tsan-suppressions.txt", "build-support/lsan-suppressions.txt", + "build-support/java-home-candidates.txt", # The LLVM symbolizer is necessary for suppressions to work "thirdparty/installed/uninstrumented/bin/llvm-symbolizer", @@ -213,6 +214,20 @@ def is_lib_blacklisted(lib): return False +def get_base_deps(): + deps = [] + for d in DEPS_FOR_ALL: + d = os.path.realpath(rel_to_abs(d)) + if os.path.isdir(d): + d += "/" + deps.append(d) + # DEPS_FOR_ALL may include binaries whose dependencies are not dependencies + # of the test executable. We must include those dependencies in the archive + # for the binaries to be usable. + deps.extend(ldd_deps(d)) + return deps + + def is_outside_of_tree(path): repo_dir = rel_to_abs("./") rel = os.path.relpath(path, repo_dir) @@ -299,15 +314,7 @@ def create_archive_input(staging, execution, files = [] files.append(rel_test_exe) deps = ldd_deps(abs_test_exe) - for d in DEPS_FOR_ALL: - d = os.path.realpath(rel_to_abs(d)) - if os.path.isdir(d): - d += "/" - deps.append(d) - # DEPS_FOR_ALL may include binaries whose dependencies are not dependencies - # of the test executable. We must include those dependencies in the archive - # for the binaries to be usable. - deps.extend(ldd_deps(d)) + deps.extend(get_base_deps()) # Deduplicate dependencies included via DEPS_FOR_ALL. for d in set(deps): @@ -522,6 +529,53 @@ def add_loop_test_subparser(subparsers): p.set_defaults(func=loop_test) +def run_java_tests(parser, options): + subprocess.check_call([rel_to_abs("java/gradlew"), "distTest"], + cwd=rel_to_abs("java")) + staging = StagingDir(rel_to_abs("java/build/dist-test")) + run_isolate(staging) + create_task_json(staging, 1) + submit_tasks(staging, options) + +def loop_java_test(parser, options): + """ + Runs many instances of a user-provided Java test class on the testing service. + """ + if options.num_instances < 1: + parser.error("--num-instances must be >= 1") + subprocess.check_call( + [rel_to_abs("java/gradlew"), "distTest", "--classes", "**/%s" % options.pattern], + cwd=rel_to_abs("java")) + staging = StagingDir(rel_to_abs("java/build/dist-test")) + run_isolate(staging) + create_task_json(staging, options.num_instances) + submit_tasks(staging, options) + + +def add_java_subparser(subparsers): + p = subparsers.add_parser('java', help='Run java tests via dist-test') + sp = p.add_subparsers() + run_all = sp.add_parser("run-all", + help="Run all of the Java tests via dist-test") + run_all.set_defaults(func=run_java_tests) + + loop = sp.add_parser("loop", help="Loop a single Java test") + loop.add_argument("--num-instances", "-n", dest="num_instances", type=int, + help="number of test instances to start", metavar="NUM", + default=100) + loop.add_argument("pattern", help="Pattern matching a Java test class to run") + loop.set_defaults(func=loop_java_test) + + +def dump_base_deps(parser, options): + print json.dumps(get_base_deps()) + + +def add_internal_commands(subparsers): + p = subparsers.add_parser('internal', help="[Internal commands not for users]") + p.add_subparsers().add_parser('dump_base_deps').set_defaults(func=dump_base_deps) + + def main(argv): p = argparse.ArgumentParser() p.add_argument("--collect-tmpdir", dest="collect_tmpdir", action="store_true", @@ -531,6 +585,8 @@ def main(argv): sp = p.add_subparsers() add_loop_test_subparser(sp) add_run_subparser(sp) + add_java_subparser(sp) + add_internal_commands(sp) args = p.parse_args(argv) args.func(p, args) http://git-wip-us.apache.org/repos/asf/kudu/blob/f5117d29/build-support/java-home-candidates.txt ---------------------------------------------------------------------- diff --git a/build-support/java-home-candidates.txt b/build-support/java-home-candidates.txt new file mode 100644 index 0000000..7a08118 --- /dev/null +++ b/build-support/java-home-candidates.txt @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This file list the locations to look for java candidates in priority order. +# Note: Trailing comments are not allowed. + +# Oracle JDK 8 Candidates +/usr/java/jdk1.8 +/usr/java/jre1.8 +/usr/lib/jvm/j2sdk1.8-oracle +/usr/lib/jvm/j2sdk1.8-oracle/jre +/usr/lib/jvm/java-8-oracle +/usr/lib/jdk8-latest + +# OpenJDK 8 Candidates +/usr/lib/jvm/java-1.8.0-openjdk-amd64 +/usr/lib/jvm/java-1.8.0-openjdk-ppc64el +/usr/lib/jvm/java-1.8.0-openjdk +/usr/lib64/jvm/java-1.8.0-openjdk-1.8.0 + +# Oracle JDK 7 Candidates +/usr/java/jdk1.7 +/usr/java/jre1.7 +/usr/lib/jvm/j2sdk1.7-oracle +/usr/lib/jvm/j2sdk1.7-oracle/jre +/usr/lib/jvm/java-7-oracle +/usr/lib/jdk7-latest + +# OpenJDK 7 Candidates +/usr/lib/jvm/java-1.7.0-openjdk +/usr/lib/jvm/java-7-openjdk-amd64 +/usr/lib/jvm/java-7-openjdk + +# Misc. Candidates +/usr/java/default +/usr/lib/jvm/java +/usr/lib/jvm/jre +/usr/lib/jvm/default-java +/usr/lib/jvm/java-openjdk +/usr/lib/jvm/jre-openjdk \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kudu/blob/f5117d29/build-support/run_dist_test.py ---------------------------------------------------------------------- diff --git a/build-support/run_dist_test.py b/build-support/run_dist_test.py index ac7075c..5c98299 100755 --- a/build-support/run_dist_test.py +++ b/build-support/run_dist_test.py @@ -28,6 +28,7 @@ # uploaded by the test slave back. import glob +import logging import optparse import os import re @@ -38,6 +39,12 @@ import sys ME = os.path.abspath(__file__) ROOT = os.path.abspath(os.path.join(os.path.dirname(ME), "..")) +with open(os.path.join(ROOT, "build-support", "java-home-candidates.txt"), 'r') as candidates: + JAVA_CANDIDATES = [x.strip() for x in candidates.readlines() if not x.startswith("#")] + # Ensure there aren't trailing comments in the path list. + for c in JAVA_CANDIDATES: + assert '#' not in c + def is_elf_binary(path): """ Determine if the given path is an ELF binary (executable or shared library) """ if not os.path.isfile(path) or os.path.islink(path): @@ -90,6 +97,12 @@ def fixup_rpaths(root): if is_elf_binary(p): fix_rpath(p) +def find_java(): + for x in JAVA_CANDIDATES: + if os.path.exists(x): + logging.info("found JAVA_HOME: ", x) + return os.path.join(x, "bin", "java") + def main(): p = optparse.OptionParser(usage="usage: %prog [options] <test-name>") p.add_option("-e", "--env", dest="env", type="string", action="append", @@ -98,13 +111,13 @@ def main(): p.add_option("--collect-tmpdir", dest="collect_tmpdir", action="store_true", help="whether to collect the test tmpdir as an artifact if the test fails", default=False) + p.add_option("--test-language", dest="test_language", action="store", + help="java or cpp", + default="cpp") options, args = p.parse_args() if len(args) < 1: p.print_help(sys.stderr) sys.exit(1) - test_exe = args[0] - test_name, _ = os.path.splitext(os.path.basename(test_exe)) - test_dir = os.path.dirname(test_exe) env = os.environ.copy() for env_pair in options.env: @@ -130,8 +143,8 @@ def main(): env['JAVA_HOME'] = glob.glob("/usr/lib/jvm/java-1.8.0-*")[0] env['LD_LIBRARY_PATH'] = ":".join( - [os.path.join(ROOT, "build/dist-test-system-libs/"), - os.path.abspath(os.path.join(test_dir, "..", "lib"))]) + [os.path.join(ROOT, "build/dist-test-system-libs/")] + + glob.glob(os.path.abspath((os.path.join(ROOT, "build/*/lib"))))) # Don't pollute /tmp in dist-test setting. If a test crashes, the dist-test slave # will clear up our working directory but won't be able to find and clean up things @@ -140,8 +153,23 @@ def main(): env['TEST_TMPDIR'] = test_tmpdir env['ASAN_SYMBOLIZER_PATH'] = os.path.join(ROOT, "thirdparty/installed/uninstrumented/bin/llvm-symbolizer") - rc = subprocess.call([os.path.join(ROOT, "build-support/run-test.sh")] + args, - env=env) + + stdout = None + stderr = None + if options.test_language == 'cpp': + cmd = [os.path.join(ROOT, "build-support/run-test.sh")] + args + elif options.test_language == 'java': + test_logdir = os.path.abspath(os.path.join(ROOT, "build/java/test-logs")) + if not os.path.exists(test_logdir): + os.makedirs(test_logdir) + cmd = [find_java()] + args + stdout = stderr = file(os.path.join(test_logdir, "test-output.txt"), "w") + else: + raise ValueError("invalid test language: " + options.test_language) + logging.info("Running command: ", cmd) + logging.info("in dir: ", os.getcwd()) + logging.info("Running with env: ", repr(env)) + rc = subprocess.call(cmd, env=env, stdout=stdout, stderr=stderr) if rc != 0 and options.collect_tmpdir: os.system("tar czf %s %s" % (os.path.join(test_dir, "..", "test-logs", "test_tmpdir.tgz"), test_tmpdir)) http://git-wip-us.apache.org/repos/asf/kudu/blob/f5117d29/cmake_modules/FindJavaHome.cmake ---------------------------------------------------------------------- diff --git a/cmake_modules/FindJavaHome.cmake b/cmake_modules/FindJavaHome.cmake index 0463cd8..c3b8cf4 100644 --- a/cmake_modules/FindJavaHome.cmake +++ b/cmake_modules/FindJavaHome.cmake @@ -23,42 +23,8 @@ # JAVA_HOME, directory containing a Java installation # JAVA_HOME_FOUND, whether JAVA_HOME has been found -set(JAVA_HOME_CANDIDATES - - # Oracle JDK 8 Candidates - /usr/java/jdk1.8 - /usr/java/jre1.8 - /usr/lib/jvm/j2sdk1.8-oracle - /usr/lib/jvm/j2sdk1.8-oracle/jre - /usr/lib/jvm/java-8-oracle - /usr/lib/jdk8-latest - - # OpenJDK 8 Candidates - /usr/lib/jvm/java-1.8.0-openjdk-amd64 - /usr/lib/jvm/java-1.8.0-openjdk-ppc64el - /usr/lib/jvm/java-1.8.0-openjdk - /usr/lib64/jvm/java-1.8.0-openjdk-1.8.0 - - # Oracle JDK 7 Candidates - /usr/java/jdk1.7 - /usr/java/jre1.7 - /usr/lib/jvm/j2sdk1.7-oracle - /usr/lib/jvm/j2sdk1.7-oracle/jre - /usr/lib/jvm/java-7-oracle - /usr/lib/jdk7-latest - - # OpenJDK 7 Candidates - /usr/lib/jvm/java-1.7.0-openjdk - /usr/lib/jvm/java-7-openjdk-amd64 - /usr/lib/jvm/java-7-openjdk - - # Misc. Candidates - /usr/java/default - /usr/lib/jvm/java - /usr/lib/jvm/jre - /usr/lib/jvm/default-java - /usr/lib/jvm/java-openjdk - /usr/lib/jvm/jre-openjdk) +file (STRINGS "${CMAKE_CURRENT_LIST_DIR}/../build-support/java-home-candidates.txt" + JAVA_HOME_CANDIDATES REGEX "^[^#].*") if (DEFINED ENV{JAVA_HOME}) set(JAVA_HOME $ENV{JAVA_HOME}) http://git-wip-us.apache.org/repos/asf/kudu/blob/f5117d29/java/build.gradle ---------------------------------------------------------------------- diff --git a/java/build.gradle b/java/build.gradle index a64e4a1..f5131e0 100755 --- a/java/build.gradle +++ b/java/build.gradle @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +import org.apache.kudu.gradle.DistTestTask + // This file is the entry-point for the gradle build and contains // common logic for the various subprojects in the build. // Plugins and scripts are applied in the natural "build order" @@ -66,4 +68,28 @@ task javadocAggregate(type: Javadoc, group: "Documentation") { source subprojects.collect { it.sourceSets.main.allJava } classpath = files(subprojects.collect { it.sourceSets.main.compileClasspath }) destinationDir = file("${buildDir}/docs/javadoc") +} + +// Copies all the dependency jars locally so that we can reference +// them inside the project structure while running the distributed +// tests instead of in the gradle cache which is in the users home +// directory by default. +task copyDistTestJars(type: Copy) { + into "$buildDir/jars/" + from subprojects.collect { + it.configurations.testRuntime + } + from subprojects.collect { + it.configurations.provided + } +} + +// Task called by dist_test.py to generate the needed .isolate and .gen.json +// files needed to run the distributed tests. +task distTest(type: DistTestTask, dependsOn: copyDistTestJars) { + subprojects.each { + it.tasks.withType(Test).each { + addTestTask it + } + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kudu/blob/f5117d29/java/buildSrc/build.gradle ---------------------------------------------------------------------- diff --git a/java/buildSrc/build.gradle b/java/buildSrc/build.gradle index 0729d53..0281f89 100644 --- a/java/buildSrc/build.gradle +++ b/java/buildSrc/build.gradle @@ -37,4 +37,5 @@ dependencies { compile "io.spring.gradle:propdeps-plugin:0.0.9.RELEASE" compile "net.ltgt.gradle:gradle-errorprone-plugin:0.0.14" compile "ru.vyarus:gradle-animalsniffer-plugin:1.4.3" + compile 'com.google.code.gson:gson:2.8.5' } http://git-wip-us.apache.org/repos/asf/kudu/blob/f5117d29/java/buildSrc/src/main/groovy/org/apache/kudu/gradle/DistTestTask.java ---------------------------------------------------------------------- diff --git a/java/buildSrc/src/main/groovy/org/apache/kudu/gradle/DistTestTask.java b/java/buildSrc/src/main/groovy/org/apache/kudu/gradle/DistTestTask.java new file mode 100644 index 0000000..bebd604 --- /dev/null +++ b/java/buildSrc/src/main/groovy/org/apache/kudu/gradle/DistTestTask.java @@ -0,0 +1,285 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.kudu.gradle; + +import org.gradle.api.DefaultTask; +import org.gradle.api.file.FileCollection; +import org.gradle.api.file.FileTree; +import org.gradle.api.internal.tasks.testing.TestClassProcessor; +import org.gradle.api.internal.tasks.testing.TestClassRunInfo; +import org.gradle.api.internal.tasks.testing.TestResultProcessor; +import org.gradle.api.internal.tasks.testing.detection.DefaultTestClassScanner; +import org.gradle.api.internal.tasks.testing.detection.TestFrameworkDetector; +import org.gradle.api.logging.Logger; +import org.gradle.api.logging.Logging; +import org.gradle.api.tasks.InputFiles; +import org.gradle.api.tasks.OutputDirectory; +import org.gradle.api.tasks.TaskAction; +import org.gradle.api.tasks.options.Option; +import org.gradle.api.tasks.testing.Test; + +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.common.io.Files; +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import com.google.gson.GsonBuilder; + +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.InputStream; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static java.nio.charset.StandardCharsets.UTF_8; + +/** + * This task is used in our top build.gradle file. It is called + * by dist_test.py to generate the needed .isolate and .gen.json + * files needed to run the distributed tests. + */ +public class DistTestTask extends DefaultTask { + private static final Logger LOGGER = Logging.getLogger(DistTestTask.class); + + private static final Gson GSON = new GsonBuilder() + .setPrettyPrinting() + .create(); + + String distTestBin = getProject().getRootDir() + "/../build-support/dist_test.py"; + + @OutputDirectory + File outputDir = new File(getProject().getBuildDir(), "dist-test"); + + private List<Test> testTasks = Lists.newArrayList(); + + /** + * Called by the build file to add test tasks to be considered for + * dist-tests. + */ + public void addTestTask(Test t) { + testTasks.add(t); + } + + @Option(option = "classes", + description = "Sets test class to be included, '*' is supported.") + public DistTestTask setClassPattern(List<String> classPattern) { + for (Test t : testTasks) { + // TODO: this is currently requiring a glob like **/*Foo* instead of just *Foo* + t.setIncludes(classPattern); + } + return this; + } + + @InputFiles + public FileCollection getInputClasses() { + FileCollection fc = getProject().files(); // Create and empty FileCollection. + for (Test t : testTasks) { + fc = fc.plus(t.getCandidateClassFiles()); + } + return fc; + } + + @TaskAction + public void doStuff() throws IOException { + getProject().delete(outputDir); + getProject().mkdir(outputDir); + List<String> baseDeps = getBaseDeps(); + for (Test t : testTasks) { + List<String> testClassNames = collectTestNames(t); + for (String c : testClassNames) { + File isoFile = new File(outputDir, c + ".isolate"); + File genJsonFile = new File(outputDir, c + ".gen.json"); + + Files.write(genIsolate(outputDir.toPath(), t, c, baseDeps), isoFile, UTF_8); + + // Write the gen.json + GenJson gen = new GenJson(); + gen.args = ImmutableList.of( + "-i", isoFile.toString(), + "-s", isoFile.toString() + ".isolated"); + gen.dir = outputDir.toString(); + gen.name = c; + Files.write(GSON.toJson(gen), genJsonFile, UTF_8); + } + } + } + + /** + * Calls dist_test.py to get the c++ "base" dependencies so that we can + * include them in the .isolate files. + * + * Note: This currently fails OSX because dump_base_deps use ldd. + */ + List<String> getBaseDeps() throws IOException { + Process proc = new ProcessBuilder(distTestBin, + "internal", + "dump_base_deps") + .redirectError(ProcessBuilder.Redirect.INHERIT) + .start(); + + try (InputStream is = proc.getInputStream()) { + return new Gson().fromJson(new InputStreamReader(is, UTF_8), + new TypeToken<List<String>>(){}.getType()); + } + } + + private String genIsolate(Path isolateFileDir, Test test, String testClass, + List<String> baseDeps) throws IOException { + Path rootDir = test.getProject().getRootDir().toPath(); + Path binDir = rootDir.resolve("../build/latest/bin").toRealPath(); + Path buildSupportDir = rootDir.resolve("../build-support").toRealPath(); + Path buildDir = rootDir.resolve("build"); + File jarDir = buildDir.resolve("jars").toFile(); + + // Build classpath with relative paths. + List<String> classpath = Lists.newArrayList(); + for (File f : test.getClasspath().getFiles()) { + File projectFile = f; + // This hack changes the path to dependent jars from the gradle cache + // in ~/.gradle/caches/... to a path to the jars copied under the project + // build directory. See the copyDistTestJars task in build.gradle to see + // the copy details. + if (projectFile.getAbsolutePath().contains(".gradle/caches/")) { + projectFile = new File(jarDir, projectFile.getName()); + } + + String s = isolateFileDir.relativize(projectFile.toPath().toAbsolutePath()).toString(); + // Isolate requires that directories be listed with a trailing '/'. + if (projectFile.isDirectory()) { + s += "/"; + } + // Gradle puts resources directories into the classpath even if they don't exist. + // isolate is unhappy with non-existent paths, though. + if (projectFile.exists()) { + classpath.add(s); + } + } + + // Build up the actual Java command line to run the test. + ImmutableList.Builder<String> cmd = new ImmutableList.Builder<>(); + cmd.add(isolateFileDir.relativize(buildSupportDir.resolve("run_dist_test.py")).toString(), + "--test-language=java", + "--", + "-ea", + "-cp", + Joiner.on(":").join(classpath)); + for (Map.Entry<String, Object> e : test.getSystemProperties().entrySet()) { + cmd.add("-D" + e.getKey() + "=" + e.getValue()); + } + cmd.add("-DbinDir=" + isolateFileDir.relativize(binDir), + "org.junit.runner.JUnitCore", + testClass); + + // Output the actual JSON. + IsolateFileJson isolate = new IsolateFileJson(); + isolate.variables.command = cmd.build(); + isolate.variables.files.addAll(classpath); + for (String s : baseDeps) { + File f = new File(s); + String path = isolateFileDir.relativize(f.toPath().toAbsolutePath()).toString(); + if (f.isDirectory()) { + path += "/"; + } + isolate.variables.files.add(path); + } + + String json = isolate.toJson(); + + // '.isolate' files are actually Python syntax, rather than true JSON. + // However, the two are close enough that just doing this replacement + // tends to work (we're assuming that no one has a quote character in a + // file path or system property. + return json.replace('"', '\''); + } + + // This is internal API but required to get the filtered list of test classes and process them. + // See the gradle code here which was used for reference: + // https://github.com/gradle/gradle/blob/c2067eaa129af4c9c29ad08da39d1c853eec4c59/subprojects/testing-jvm/src/main/java/org/gradle/api/internal/tasks/testing/detection/DefaultTestExecuter.java#L104-L112 + private List<String> collectTestNames(Test testTask) { + ClassNameCollectingProcessor processor = new ClassNameCollectingProcessor(); + Runnable detector; + final FileTree testClassFiles = testTask.getCandidateClassFiles(); + if (testTask.isScanForTestClasses()) { + TestFrameworkDetector testFrameworkDetector = testTask.getTestFramework().getDetector(); + testFrameworkDetector.setTestClasses(testTask.getTestClassesDirs().getFiles()); + testFrameworkDetector.setTestClasspath(testTask.getClasspath().getFiles()); + detector = new DefaultTestClassScanner(testClassFiles, testFrameworkDetector, processor); + } else { + detector = new DefaultTestClassScanner(testClassFiles, null, processor); + } + detector.run(); + LOGGER.debug("collected test class names: {}", processor.classNames); + return processor.classNames; + } + + private static class ClassNameCollectingProcessor implements TestClassProcessor { + public List<String> classNames = new ArrayList<String>(); + + @Override + public void startProcessing(TestResultProcessor testResultProcessor) { + // no-op + } + + @Override + public void processTestClass(TestClassRunInfo testClassRunInfo) { + classNames.add(testClassRunInfo.getTestClassName()); + } + + @Override + public void stop() { + // no-op + } + + @Override + public void stopNow() { + // no-op + } + } + + /** + * Structured to generate Json that matches the expected .isolate format. + * See here for a description of the .isolate format: + * https://github.com/cloudera/dist_test/blob/master/grind/python/disttest/isolate.py + */ + private static class IsolateFileJson { + private static class Variables { + public List<String> files = new ArrayList<>(); + public List<String> command; + }; + Variables variables = new Variables(); + + public String toJson() { + return GSON.toJson(this); + } + } + + /** + * Structured to generate Json that matches the expected .gen.json contents. + * See here for a description of the .gen.json contents: + * https://github.com/cloudera/dist_test/blob/master/grind/python/disttest/isolate.py + */ + private static class GenJson { + int version = 1; + String dir; + List<String> args; + String name; + } +}
