robertwb commented on a change in pull request #11792:
URL: https://github.com/apache/beam/pull/11792#discussion_r453099881
##########
File path: runners/portability/java/build.gradle
##########
@@ -31,9 +38,161 @@ dependencies {
compile project(path: ":sdks:java:harness", configuration: "shadow")
compile library.java.vendored_grpc_1_26_0
compile library.java.slf4j_api
+
testCompile project(path: ":runners:core-construction-java", configuration:
"testRuntime")
testCompile library.java.hamcrest_core
testCompile library.java.junit
testCompile library.java.mockito_core
testCompile library.java.slf4j_jdk14
+
+ validatesRunner project(path: ":sdks:java:core", configuration: "shadowTest")
+ validatesRunner project(path: ":runners:core-java", configuration:
"testRuntime")
+ validatesRunner project(path: project.path, configuration: "testRuntime")
+}
+
+
+project.evaluationDependsOn(":sdks:java:core")
+project.evaluationDependsOn(":runners:core-java")
+
+ext.localJobServicePidFile = "${project.buildDir}/local_job_service_pid"
+ext.localJobServicePortFile = project.hasProperty("localJobServicePortFile") ?
project.property("localJobServicePortFile") :
"${project.buildDir}/local_job_service_port"
+ext.localJobServiceStdoutFile = "${project.buildDir}/local_job_service_stdout"
+
+void execInVirtualenv(String... args) {
+ String shellCommand = ". ${project.ext.envdir}/bin/activate && " +
args.collect { arg -> "'" + arg.replaceAll("'", "\\'") + "'" }.join(" ")
+ exec {
+ workingDir pythonSdkDir
+ commandLine "sh", "-c", shellCommand
+ }
+}
+
+// Does not background the process, but allows the process to daemonize itself
+void execBackgroundInVirtualenv(String... args) {
+ String shellCommand = ". ${project.ext.envdir}/bin/activate && " +
args.collect { arg -> "'" + arg.replaceAll("'", "\\'") + "'" }.join(" ")
+ println "execBackgroundInVirtualEnv: ${shellCommand}"
+ ProcessBuilder pb = new
ProcessBuilder().redirectErrorStream(true).directory(new
File(pythonSdkDir)).command(["sh", "-c", shellCommand])
+ Process proc = pb.start();
+
+ // redirectIO does not work for connecting to groovy/gradle stdout
+ BufferedReader reader = new BufferedReader(new
InputStreamReader(proc.getInputStream()));
+ String line
+ while ((line = reader.readLine()) != null) {
+ println line
+ }
+ proc.waitFor();
+}
+
+task startLocalJobService {
+ dependsOn setupVirtualenv
+
+ doLast {
+ execBackgroundInVirtualenv "python",
+ "-m", "apache_beam.runners.portability.local_job_service_main",
+ "--background",
+ "--stdout_file=${localJobServiceStdoutFile}",
+ "--pid_file=${localJobServicePidFile}",
+ "--port_file=${localJobServicePortFile}"
+//
+// File pidFile = new File(localJobServicePidFile)
+// int totalSleep = 0
+// while (!pidFile.exists()) {
+// sleep(500)
+// totalSleep += 500
+// if (totalSleep > 5000) {
+// throw new RuntimeException("Local job service pid file never showed
up");
+// }
+// }
+ }
}
+
+task stopLocalJobService {
+ doLast {
+ execInVirtualenv "python",
+ "-m", "apache_beam.runners.portability.local_job_service_main",
+ "--stop",
+ "--pid_file=${localJobServicePidFile}"
+ }
+}
+
+startLocalJobService.finalizedBy stopLocalJobService
+
+/**
+ * Runs Java ValidatesRunner tests against the Universal Local Runner (ULR)
aka local_job_service_main
+ * with subprocess SDK harness environments.
+ */
+task ulrValidatesRunnerTests(type: Test) {
+ dependsOn ":sdks:java:container:docker"
+
+ if (!project.hasProperty("localJobServicePortFile")) {
+ dependsOn startLocalJobService
+ }
+
+ group = "Verification"
+ description "PortableRunner Java subprocess ValidatesRunner suite"
+ classpath = configurations.validatesRunner
+ systemProperty "beamTestPipelineOptions", JsonOutput.toJson([
+ "--runner=TestUniversalRunner",
+ "--experiments=beam_fn_api",
+ "--localJobServicePortFile=${localJobServicePortFile}"
+ ])
Review comment:
Ack. I don't think it makes sense to run every test to verify the
container/harness setup works (this is reminiscent of other threads) but
definitely agree these choices can be postponed while we get this PR in.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
