[GitHub] [beam] kennknowles commented on a change in pull request #11792: WIP: Add ValidatesRunner task for local_job_service and Java SDK harness
kennknowles commented on a change in pull request #11792:
URL: https://github.com/apache/beam/pull/11792#discussion_r451779380
##
File path: runners/portability/java/build.gradle
##
@@ -31,9 +45,123 @@ dependencies {
compile project(path: ":sdks:java:harness", configuration: "shadow")
compile library.java.vendored_grpc_1_26_0
compile library.java.slf4j_api
+
testCompile project(path: ":runners:core-construction-java", configuration:
"testRuntime")
testCompile library.java.hamcrest_core
testCompile library.java.junit
testCompile library.java.mockito_core
testCompile library.java.slf4j_jdk14
+
+ validatesRunner project(path: ":sdks:java:core", configuration: "shadowTest")
+ validatesRunner project(path: ":runners:core-java", configuration:
"testRuntime")
+ validatesRunner project(path: project.path, configuration: "testRuntime")
+}
+
+
+project.evaluationDependsOn(":sdks:java:core")
+project.evaluationDependsOn(":runners:core-java")
+
+ext.virtualenvDir = "${project.buildDir}/virtualenv"
+ext.localJobServicePidFile = "${project.buildDir}/local_job_service_pid"
+ext.localJobServicePortFile = project.hasProperty("localJobServicePortFile") ?
project.property("localJobServicePortFile") :
"${project.buildDir}/local_job_service_port"
+ext.localJobServiceStdoutFile = "${project.buildDir}/local_job_service_stdout"
+
+ext.pythonSdkDir = "${project.rootDir}/sdks/python"
+
+void execInVirtualenv(String... args) {
+ String shellCommand = ". ${virtualenvDir}/bin/activate && " + args.collect {
arg -> "'" + arg.replaceAll("'", "\\'") + "'" }.join(" ")
+ exec {
+workingDir pythonSdkDir
+commandLine "sh", "-c", shellCommand
+ }
}
+
+void execBackgroundInVirtualenv(String... args) {
+ String shellCommand = ". ${virtualenvDir}/bin/activate && " + args.collect {
arg -> "'" + arg.replaceAll("'", "\\'") + "'" }.join(" ")
+ println "execBackgroundInVirtualEnv: ${shellCommand}"
+ ProcessBuilder pb = new
ProcessBuilder().redirectErrorStream(true).directory(new
File(pythonSdkDir)).command(["sh", "-c", shellCommand])
+ Process proc = pb.start();
+
+ // redirectIO does not work for connecting to groovy/gradle stdout
+ BufferedReader reader = new BufferedReader(new
InputStreamReader(proc.getInputStream()));
+ String line
+ while ((line = reader.readLine()) != null) {
+println line
+ }
+ proc.waitFor();
+}
+
+task virtualenv {
Review comment:
Adding a dependency on something that _is_ ready and _is_ meant for it,
but is not polished and named to make clear that it is a _logical necessity_:
tech debt)
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [beam] kennknowles commented on a change in pull request #11792: WIP: Add ValidatesRunner task for local_job_service and Java SDK harness
kennknowles commented on a change in pull request #11792:
URL: https://github.com/apache/beam/pull/11792#discussion_r451772554
##
File path: runners/portability/java/build.gradle
##
@@ -31,9 +45,123 @@ dependencies {
compile project(path: ":sdks:java:harness", configuration: "shadow")
compile library.java.vendored_grpc_1_26_0
compile library.java.slf4j_api
+
testCompile project(path: ":runners:core-construction-java", configuration:
"testRuntime")
testCompile library.java.hamcrest_core
testCompile library.java.junit
testCompile library.java.mockito_core
testCompile library.java.slf4j_jdk14
+
+ validatesRunner project(path: ":sdks:java:core", configuration: "shadowTest")
+ validatesRunner project(path: ":runners:core-java", configuration:
"testRuntime")
+ validatesRunner project(path: project.path, configuration: "testRuntime")
+}
+
+
+project.evaluationDependsOn(":sdks:java:core")
+project.evaluationDependsOn(":runners:core-java")
+
+ext.virtualenvDir = "${project.buildDir}/virtualenv"
+ext.localJobServicePidFile = "${project.buildDir}/local_job_service_pid"
+ext.localJobServicePortFile = project.hasProperty("localJobServicePortFile") ?
project.property("localJobServicePortFile") :
"${project.buildDir}/local_job_service_port"
+ext.localJobServiceStdoutFile = "${project.buildDir}/local_job_service_stdout"
+
+ext.pythonSdkDir = "${project.rootDir}/sdks/python"
+
+void execInVirtualenv(String... args) {
+ String shellCommand = ". ${virtualenvDir}/bin/activate && " + args.collect {
arg -> "'" + arg.replaceAll("'", "\\'") + "'" }.join(" ")
+ exec {
+workingDir pythonSdkDir
+commandLine "sh", "-c", shellCommand
+ }
}
+
+void execBackgroundInVirtualenv(String... args) {
+ String shellCommand = ". ${virtualenvDir}/bin/activate && " + args.collect {
arg -> "'" + arg.replaceAll("'", "\\'") + "'" }.join(" ")
+ println "execBackgroundInVirtualEnv: ${shellCommand}"
+ ProcessBuilder pb = new
ProcessBuilder().redirectErrorStream(true).directory(new
File(pythonSdkDir)).command(["sh", "-c", shellCommand])
+ Process proc = pb.start();
+
+ // redirectIO does not work for connecting to groovy/gradle stdout
+ BufferedReader reader = new BufferedReader(new
InputStreamReader(proc.getInputStream()));
+ String line
+ while ((line = reader.readLine()) != null) {
+println line
+ }
+ proc.waitFor();
+}
+
+task virtualenv {
Review comment:
(I will explore this invitation once the tests are running properly)
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [beam] kennknowles commented on a change in pull request #11792: WIP: Add ValidatesRunner task for local_job_service and Java SDK harness
kennknowles commented on a change in pull request #11792:
URL: https://github.com/apache/beam/pull/11792#discussion_r451761061
##
File path: runners/portability/java/build.gradle
##
@@ -31,9 +45,123 @@ dependencies {
compile project(path: ":sdks:java:harness", configuration: "shadow")
compile library.java.vendored_grpc_1_26_0
compile library.java.slf4j_api
+
testCompile project(path: ":runners:core-construction-java", configuration:
"testRuntime")
testCompile library.java.hamcrest_core
testCompile library.java.junit
testCompile library.java.mockito_core
testCompile library.java.slf4j_jdk14
+
+ validatesRunner project(path: ":sdks:java:core", configuration: "shadowTest")
+ validatesRunner project(path: ":runners:core-java", configuration:
"testRuntime")
+ validatesRunner project(path: project.path, configuration: "testRuntime")
+}
+
+
+project.evaluationDependsOn(":sdks:java:core")
+project.evaluationDependsOn(":runners:core-java")
+
+ext.virtualenvDir = "${project.buildDir}/virtualenv"
+ext.localJobServicePidFile = "${project.buildDir}/local_job_service_pid"
+ext.localJobServicePortFile = project.hasProperty("localJobServicePortFile") ?
project.property("localJobServicePortFile") :
"${project.buildDir}/local_job_service_port"
+ext.localJobServiceStdoutFile = "${project.buildDir}/local_job_service_stdout"
+
+ext.pythonSdkDir = "${project.rootDir}/sdks/python"
+
+void execInVirtualenv(String... args) {
+ String shellCommand = ". ${virtualenvDir}/bin/activate && " + args.collect {
arg -> "'" + arg.replaceAll("'", "\\'") + "'" }.join(" ")
+ exec {
+workingDir pythonSdkDir
+commandLine "sh", "-c", shellCommand
+ }
}
+
+void execBackgroundInVirtualenv(String... args) {
+ String shellCommand = ". ${virtualenvDir}/bin/activate && " + args.collect {
arg -> "'" + arg.replaceAll("'", "\\'") + "'" }.join(" ")
+ println "execBackgroundInVirtualEnv: ${shellCommand}"
+ ProcessBuilder pb = new
ProcessBuilder().redirectErrorStream(true).directory(new
File(pythonSdkDir)).command(["sh", "-c", shellCommand])
+ Process proc = pb.start();
+
+ // redirectIO does not work for connecting to groovy/gradle stdout
+ BufferedReader reader = new BufferedReader(new
InputStreamReader(proc.getInputStream()));
+ String line
+ while ((line = reader.readLine()) != null) {
+println line
+ }
+ proc.waitFor();
+}
+
+task virtualenv {
+ doLast {
+exec {
+ commandLine "virtualenv", virtualenvDir, "--python=python3"
+}
+execInVirtualenv "pip", "install", "--retries", "10", "--upgrade",
"tox==3.11.1", "--requirement",
"${project.rootDir}/sdks/python/build-requirements.txt"
+execInVirtualenv "python", "setup.py", "build", "--build-base=${buildDir}"
+execInVirtualenv "pip", "install", "-e", "."
+ }
+}
+
+task startLocalJobService {
+ dependsOn virtualenv
+
+ doLast {
+execBackgroundInVirtualenv "python",
+"-m", "apache_beam.runners.portability.local_job_service_main",
+"--background",
+"--stdout_file=${localJobServiceStdoutFile}",
+"--pid_file=${localJobServicePidFile}",
+"--port_file=${localJobServicePortFile}"
+
+File pidFile = new File(localJobServicePidFile)
+int totalSleep = 0
+while (!pidFile.exists()) {
+ sleep(500)
Review comment:
Removed. This code was left over from when I was struggling to get
gradle to allow the thing to daemonize itself.
##
File path: sdks/python/apache_beam/runners/portability/local_job_service_main.py
##
@@ -99,11 +105,23 @@ def run(argv):
options.port_file = os.path.splitext(options.pid_file)[0] + '.port'
argv.append('--port_file')
argv.append(options.port_file)
+
+if not options.stdout_file:
+ raise RuntimeError('--stdout_file must be specified with --background')
+stdout_dest = open(options.stdout_file, mode='w')
+
+if options.stderr_file:
+ stderr_dest=open(options.stderr_file, mode='w')
+else:
+ stderr_dest=subprocess.STDOUT
+
subprocess.Popen([
sys.executable,
'-m',
'apache_beam.runners.portability.local_job_service_main'
-] + argv)
+] + argv,
+stderr=stderr_dest,
Review comment:
I didn't read the `subprocess` code, and the docs are vague. The special
`subprocess.STDOUT` token indicates that the output should be "captured" into
the same file handle. Comments at
https://stackoverflow.com/questions/31980411/closing-files-from-subprocess-stdout
imply that closing the file is the responsibility of this process. I did not
run the experiments suggested there. I also did not try to refactor this code
to allow a `with` statement.
##
File path: runners/portability/java/build.gradle
##
@@ -31,9 +45,123 @@ dependencies {
compile project(path: ":sdks:java:harness", configuration: "shadow")
compile library.java.vendored_grpc_1_26_0
compile library.java.slf4j_api
+
testCompile
[GitHub] [beam] kennknowles commented on a change in pull request #11792: WIP: Add ValidatesRunner task for local_job_service and Java SDK harness
kennknowles commented on a change in pull request #11792:
URL: https://github.com/apache/beam/pull/11792#discussion_r431350972
##
File path: runners/portability/java/build.gradle
##
@@ -31,9 +45,123 @@ dependencies {
compile project(path: ":sdks:java:harness", configuration: "shadow")
compile library.java.vendored_grpc_1_26_0
compile library.java.slf4j_api
+
testCompile project(path: ":runners:core-construction-java", configuration:
"testRuntime")
testCompile library.java.hamcrest_core
testCompile library.java.junit
testCompile library.java.mockito_core
testCompile library.java.slf4j_jdk14
+
+ validatesRunner project(path: ":sdks:java:core", configuration: "shadowTest")
+ validatesRunner project(path: ":runners:core-java", configuration:
"testRuntime")
+ validatesRunner project(path: project.path, configuration: "testRuntime")
+}
+
+
+project.evaluationDependsOn(":sdks:java:core")
+project.evaluationDependsOn(":runners:core-java")
+
+ext.virtualenvDir = "${project.buildDir}/virtualenv"
+ext.localJobServicePidFile = "${project.buildDir}/local_job_service_pid"
+ext.localJobServicePortFile = project.hasProperty("localJobServicePortFile") ?
project.property("localJobServicePortFile") :
"${project.buildDir}/local_job_service_port"
+ext.localJobServiceStdoutFile = "${project.buildDir}/local_job_service_stdout"
+
+ext.pythonSdkDir = "${project.rootDir}/sdks/python"
+
+void execInVirtualenv(String... args) {
+ String shellCommand = ". ${virtualenvDir}/bin/activate && " + args.collect {
arg -> "'" + arg.replaceAll("'", "\\'") + "'" }.join(" ")
+ exec {
+workingDir pythonSdkDir
+commandLine "sh", "-c", shellCommand
+ }
}
+
+void execBackgroundInVirtualenv(String... args) {
+ String shellCommand = ". ${virtualenvDir}/bin/activate && " + args.collect {
arg -> "'" + arg.replaceAll("'", "\\'") + "'" }.join(" ")
+ println "execBackgroundInVirtualEnv: ${shellCommand}"
+ ProcessBuilder pb = new
ProcessBuilder().redirectErrorStream(true).directory(new
File(pythonSdkDir)).command(["sh", "-c", shellCommand])
+ Process proc = pb.start();
+
+ // redirectIO does not work for connecting to groovy/gradle stdout
+ BufferedReader reader = new BufferedReader(new
InputStreamReader(proc.getInputStream()));
+ String line
+ while ((line = reader.readLine()) != null) {
+println line
+ }
+ proc.waitFor();
+}
+
+task virtualenv {
Review comment:
I hey now I recall - `applyPythonNature` causes conflicting
configurations. Really would like to move away from these nature things.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [beam] kennknowles commented on a change in pull request #11792: WIP: Add ValidatesRunner task for local_job_service and Java SDK harness
kennknowles commented on a change in pull request #11792:
URL: https://github.com/apache/beam/pull/11792#discussion_r431346522
##
File path: runners/portability/java/build.gradle
##
@@ -31,9 +45,123 @@ dependencies {
compile project(path: ":sdks:java:harness", configuration: "shadow")
compile library.java.vendored_grpc_1_26_0
compile library.java.slf4j_api
+
testCompile project(path: ":runners:core-construction-java", configuration:
"testRuntime")
testCompile library.java.hamcrest_core
testCompile library.java.junit
testCompile library.java.mockito_core
testCompile library.java.slf4j_jdk14
+
+ validatesRunner project(path: ":sdks:java:core", configuration: "shadowTest")
+ validatesRunner project(path: ":runners:core-java", configuration:
"testRuntime")
+ validatesRunner project(path: project.path, configuration: "testRuntime")
+}
+
+
+project.evaluationDependsOn(":sdks:java:core")
+project.evaluationDependsOn(":runners:core-java")
+
+ext.virtualenvDir = "${project.buildDir}/virtualenv"
+ext.localJobServicePidFile = "${project.buildDir}/local_job_service_pid"
+ext.localJobServicePortFile = project.hasProperty("localJobServicePortFile") ?
project.property("localJobServicePortFile") :
"${project.buildDir}/local_job_service_port"
+ext.localJobServiceStdoutFile = "${project.buildDir}/local_job_service_stdout"
+
+ext.pythonSdkDir = "${project.rootDir}/sdks/python"
+
+void execInVirtualenv(String... args) {
+ String shellCommand = ". ${virtualenvDir}/bin/activate && " + args.collect {
arg -> "'" + arg.replaceAll("'", "\\'") + "'" }.join(" ")
+ exec {
+workingDir pythonSdkDir
+commandLine "sh", "-c", shellCommand
+ }
}
+
+void execBackgroundInVirtualenv(String... args) {
+ String shellCommand = ". ${virtualenvDir}/bin/activate && " + args.collect {
arg -> "'" + arg.replaceAll("'", "\\'") + "'" }.join(" ")
+ println "execBackgroundInVirtualEnv: ${shellCommand}"
+ ProcessBuilder pb = new
ProcessBuilder().redirectErrorStream(true).directory(new
File(pythonSdkDir)).command(["sh", "-c", shellCommand])
+ Process proc = pb.start();
+
+ // redirectIO does not work for connecting to groovy/gradle stdout
+ BufferedReader reader = new BufferedReader(new
InputStreamReader(proc.getInputStream()));
+ String line
+ while ((line = reader.readLine()) != null) {
+println line
+ }
+ proc.waitFor();
+}
+
+task virtualenv {
Review comment:
I didn't use them because I was attempting to do more proper
idempotent/cached gradle tasks.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [beam] kennknowles commented on a change in pull request #11792: WIP: Add ValidatesRunner task for local_job_service and Java SDK harness
kennknowles commented on a change in pull request #11792: URL: https://github.com/apache/beam/pull/11792#discussion_r431343272 ## File path: runners/portability/java/build.gradle ## @@ -1,3 +1,13 @@ +import groovy.json.JsonOutput + +import java.nio.file.FileSystems Review comment: Great, I'll remove that. I think in an early draft I made the gradle task not wait for the process to terminate (because the daemonized process was causing the hang, but that turned out to be a different configuration) This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [beam] kennknowles commented on a change in pull request #11792: WIP: Add ValidatesRunner task for local_job_service and Java SDK harness
kennknowles commented on a change in pull request #11792:
URL: https://github.com/apache/beam/pull/11792#discussion_r429031586
##
File path: runners/portability/java/build.gradle
##
@@ -31,9 +45,123 @@ dependencies {
compile project(path: ":sdks:java:harness", configuration: "shadow")
compile library.java.vendored_grpc_1_26_0
compile library.java.slf4j_api
+
testCompile project(path: ":runners:core-construction-java", configuration:
"testRuntime")
testCompile library.java.hamcrest_core
testCompile library.java.junit
testCompile library.java.mockito_core
testCompile library.java.slf4j_jdk14
+
+ validatesRunner project(path: ":sdks:java:core", configuration: "shadowTest")
+ validatesRunner project(path: ":runners:core-java", configuration:
"testRuntime")
+ validatesRunner project(path: project.path, configuration: "testRuntime")
+}
+
+
+project.evaluationDependsOn(":sdks:java:core")
+project.evaluationDependsOn(":runners:core-java")
+
+ext.virtualenvDir = "${project.buildDir}/virtualenv"
+ext.localJobServicePidFile = "${project.buildDir}/local_job_service_pid"
+ext.localJobServicePortFile = project.hasProperty("localJobServicePortFile") ?
project.property("localJobServicePortFile") :
"${project.buildDir}/local_job_service_port"
+ext.localJobServiceStdoutFile = "${project.buildDir}/local_job_service_stdout"
+
+ext.pythonSdkDir = "${project.rootDir}/sdks/python"
+
+void execInVirtualenv(String... args) {
+ String shellCommand = ". ${virtualenvDir}/bin/activate && " + args.collect {
arg -> "'" + arg.replaceAll("'", "\\'") + "'" }.join(" ")
+ exec {
+workingDir pythonSdkDir
+commandLine "sh", "-c", shellCommand
+ }
}
+
+void execBackgroundInVirtualenv(String... args) {
+ String shellCommand = ". ${virtualenvDir}/bin/activate && " + args.collect {
arg -> "'" + arg.replaceAll("'", "\\'") + "'" }.join(" ")
+ println "execBackgroundInVirtualEnv: ${shellCommand}"
+ ProcessBuilder pb = new
ProcessBuilder().redirectErrorStream(true).directory(new
File(pythonSdkDir)).command(["sh", "-c", shellCommand])
+ Process proc = pb.start();
+
+ // redirectIO does not work for connecting to groovy/gradle stdout
+ BufferedReader reader = new BufferedReader(new
InputStreamReader(proc.getInputStream()));
+ String line
+ while ((line = reader.readLine()) != null) {
+println line
+ }
+ proc.waitFor();
+}
+
+task virtualenv {
Review comment:
I tried registering `outputs.dir virtualenvDir` and `inputs.dir
pythonSdkDir` but it did not result in incremental build.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [beam] kennknowles commented on a change in pull request #11792: WIP: Add ValidatesRunner task for local_job_service and Java SDK harness
kennknowles commented on a change in pull request #11792:
URL: https://github.com/apache/beam/pull/11792#discussion_r429031469
##
File path: runners/portability/java/build.gradle
##
@@ -31,9 +45,123 @@ dependencies {
compile project(path: ":sdks:java:harness", configuration: "shadow")
compile library.java.vendored_grpc_1_26_0
compile library.java.slf4j_api
+
testCompile project(path: ":runners:core-construction-java", configuration:
"testRuntime")
testCompile library.java.hamcrest_core
testCompile library.java.junit
testCompile library.java.mockito_core
testCompile library.java.slf4j_jdk14
+
+ validatesRunner project(path: ":sdks:java:core", configuration: "shadowTest")
+ validatesRunner project(path: ":runners:core-java", configuration:
"testRuntime")
+ validatesRunner project(path: project.path, configuration: "testRuntime")
+}
+
+
+project.evaluationDependsOn(":sdks:java:core")
+project.evaluationDependsOn(":runners:core-java")
+
+ext.virtualenvDir = "${project.buildDir}/virtualenv"
+ext.localJobServicePidFile = "${project.buildDir}/local_job_service_pid"
+ext.localJobServicePortFile = project.hasProperty("localJobServicePortFile") ?
project.property("localJobServicePortFile") :
"${project.buildDir}/local_job_service_port"
+ext.localJobServiceStdoutFile = "${project.buildDir}/local_job_service_stdout"
+
+ext.pythonSdkDir = "${project.rootDir}/sdks/python"
+
+void execInVirtualenv(String... args) {
+ String shellCommand = ". ${virtualenvDir}/bin/activate && " + args.collect {
arg -> "'" + arg.replaceAll("'", "\\'") + "'" }.join(" ")
+ exec {
+workingDir pythonSdkDir
+commandLine "sh", "-c", shellCommand
+ }
}
+
+void execBackgroundInVirtualenv(String... args) {
+ String shellCommand = ". ${virtualenvDir}/bin/activate && " + args.collect {
arg -> "'" + arg.replaceAll("'", "\\'") + "'" }.join(" ")
+ println "execBackgroundInVirtualEnv: ${shellCommand}"
+ ProcessBuilder pb = new
ProcessBuilder().redirectErrorStream(true).directory(new
File(pythonSdkDir)).command(["sh", "-c", shellCommand])
+ Process proc = pb.start();
+
+ // redirectIO does not work for connecting to groovy/gradle stdout
+ BufferedReader reader = new BufferedReader(new
InputStreamReader(proc.getInputStream()));
Review comment:
This verbosity is because ProcessBuilder.inheritIO does not work in this
context (don't know if it is Gradle or Groovy breaking it)
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [beam] kennknowles commented on a change in pull request #11792: WIP: Add ValidatesRunner task for local_job_service and Java SDK harness
kennknowles commented on a change in pull request #11792:
URL: https://github.com/apache/beam/pull/11792#discussion_r429031354
##
File path: runners/portability/java/build.gradle
##
@@ -31,9 +45,123 @@ dependencies {
compile project(path: ":sdks:java:harness", configuration: "shadow")
compile library.java.vendored_grpc_1_26_0
compile library.java.slf4j_api
+
testCompile project(path: ":runners:core-construction-java", configuration:
"testRuntime")
testCompile library.java.hamcrest_core
testCompile library.java.junit
testCompile library.java.mockito_core
testCompile library.java.slf4j_jdk14
+
+ validatesRunner project(path: ":sdks:java:core", configuration: "shadowTest")
+ validatesRunner project(path: ":runners:core-java", configuration:
"testRuntime")
+ validatesRunner project(path: project.path, configuration: "testRuntime")
+}
+
+
+project.evaluationDependsOn(":sdks:java:core")
+project.evaluationDependsOn(":runners:core-java")
+
+ext.virtualenvDir = "${project.buildDir}/virtualenv"
+ext.localJobServicePidFile = "${project.buildDir}/local_job_service_pid"
+ext.localJobServicePortFile = project.hasProperty("localJobServicePortFile") ?
project.property("localJobServicePortFile") :
"${project.buildDir}/local_job_service_port"
+ext.localJobServiceStdoutFile = "${project.buildDir}/local_job_service_stdout"
+
+ext.pythonSdkDir = "${project.rootDir}/sdks/python"
+
+void execInVirtualenv(String... args) {
+ String shellCommand = ". ${virtualenvDir}/bin/activate && " + args.collect {
arg -> "'" + arg.replaceAll("'", "\\'") + "'" }.join(" ")
+ exec {
+workingDir pythonSdkDir
+commandLine "sh", "-c", shellCommand
+ }
}
+
+void execBackgroundInVirtualenv(String... args) {
Review comment:
This verbosity is because Gradle exec tasks do not terminate when there
are open subprocesses.
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [beam] kennknowles commented on a change in pull request #11792: WIP: Add ValidatesRunner task for local_job_service and Java SDK harness
kennknowles commented on a change in pull request #11792:
URL: https://github.com/apache/beam/pull/11792#discussion_r429029786
##
File path: runners/portability/java/build.gradle
##
@@ -1,3 +1,13 @@
+import groovy.json.JsonOutput
+
+import java.nio.file.FileSystems
Review comment:
TODO(me): remove these imports
I first went through the "normal" route of using all this stuff to watch for
the pid file but it was verbose and had race conditions. No point. Just check
and sleep, now.
##
File path:
runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/artifact/ArtifactRetrievalService.java
##
@@ -39,7 +39,7 @@
public class ArtifactRetrievalService
extends ArtifactRetrievalServiceGrpc.ArtifactRetrievalServiceImplBase
implements FnService {
- public static final int DEFAULT_BUFFER_SIZE = 4 << 20; // 4 MB
+ public static final int DEFAULT_BUFFER_SIZE = 2 << 20; // 2 MB
Review comment:
@robertwb @lukecwik first thing I hit putting this together was
exceeding message size limit
##
File path: runners/portability/java/build.gradle
##
@@ -31,9 +45,123 @@ dependencies {
compile project(path: ":sdks:java:harness", configuration: "shadow")
compile library.java.vendored_grpc_1_26_0
compile library.java.slf4j_api
+
testCompile project(path: ":runners:core-construction-java", configuration:
"testRuntime")
testCompile library.java.hamcrest_core
testCompile library.java.junit
testCompile library.java.mockito_core
testCompile library.java.slf4j_jdk14
+
+ validatesRunner project(path: ":sdks:java:core", configuration: "shadowTest")
+ validatesRunner project(path: ":runners:core-java", configuration:
"testRuntime")
+ validatesRunner project(path: project.path, configuration: "testRuntime")
+}
+
+
+project.evaluationDependsOn(":sdks:java:core")
+project.evaluationDependsOn(":runners:core-java")
+
+ext.virtualenvDir = "${project.buildDir}/virtualenv"
+ext.localJobServicePidFile = "${project.buildDir}/local_job_service_pid"
+ext.localJobServicePortFile = project.hasProperty("localJobServicePortFile") ?
project.property("localJobServicePortFile") :
"${project.buildDir}/local_job_service_port"
+ext.localJobServiceStdoutFile = "${project.buildDir}/local_job_service_stdout"
+
+ext.pythonSdkDir = "${project.rootDir}/sdks/python"
+
+void execInVirtualenv(String... args) {
+ String shellCommand = ". ${virtualenvDir}/bin/activate && " + args.collect {
arg -> "'" + arg.replaceAll("'", "\\'") + "'" }.join(" ")
+ exec {
+workingDir pythonSdkDir
+commandLine "sh", "-c", shellCommand
+ }
}
+
+void execBackgroundInVirtualenv(String... args) {
+ String shellCommand = ". ${virtualenvDir}/bin/activate && " + args.collect {
arg -> "'" + arg.replaceAll("'", "\\'") + "'" }.join(" ")
+ println "execBackgroundInVirtualEnv: ${shellCommand}"
+ ProcessBuilder pb = new
ProcessBuilder().redirectErrorStream(true).directory(new
File(pythonSdkDir)).command(["sh", "-c", shellCommand])
+ Process proc = pb.start();
+
+ // redirectIO does not work for connecting to groovy/gradle stdout
+ BufferedReader reader = new BufferedReader(new
InputStreamReader(proc.getInputStream()));
+ String line
+ while ((line = reader.readLine()) != null) {
+println line
+ }
+ proc.waitFor();
+}
+
+task virtualenv {
+ doLast {
+exec {
+ commandLine "virtualenv", virtualenvDir, "--python=python3"
+}
+execInVirtualenv "pip", "install", "--retries", "10", "--upgrade",
"tox==3.11.1", "--requirement",
"${project.rootDir}/sdks/python/build-requirements.txt"
+execInVirtualenv "python", "setup.py", "build", "--build-base=${buildDir}"
+execInVirtualenv "pip", "install", "-e", "."
+ }
+}
+
+task startLocalJobService {
+ dependsOn virtualenv
+
+ doLast {
+execBackgroundInVirtualenv "python",
+"-m", "apache_beam.runners.portability.local_job_service_main",
+"--background",
+"--stdout_file=${localJobServiceStdoutFile}",
+"--pid_file=${localJobServicePidFile}",
+"--port_file=${localJobServicePortFile}"
Review comment:
It would be nicer for the port to be an output of the task and read it
in by the other task, but that plumbing seems to be unnatural based on gradle
docs I could find.
##
File path: runners/portability/java/build.gradle
##
@@ -31,9 +45,123 @@ dependencies {
compile project(path: ":sdks:java:harness", configuration: "shadow")
compile library.java.vendored_grpc_1_26_0
compile library.java.slf4j_api
+
testCompile project(path: ":runners:core-construction-java", configuration:
"testRuntime")
testCompile library.java.hamcrest_core
testCompile library.java.junit
testCompile library.java.mockito_core
testCompile library.java.slf4j_jdk14
+
+ validatesRunner project(path: ":sdks:java:core", configuration: "shadowTest")
+ validatesRunner project(path: ":runners:core-java", configuration:
