kennknowles commented on a change in pull request #12981:
URL: https://github.com/apache/beam/pull/12981#discussion_r497849873
##########
File path: runners/portability/java/build.gradle
##########
@@ -50,27 +50,32 @@ dependencies {
validatesRunner project(path: project.path, configuration: "testRuntime")
}
-
project.evaluationDependsOn(":sdks:java:core")
+project.evaluationDependsOn(":sdks:python")
project.evaluationDependsOn(":runners:core-java")
-ext.localJobServicePidFile = "${project.buildDir}/local_job_service_pid"
+ext.pythonSdkDir =
project.findProject(":sdks:python").getProjectDir().getAbsoluteFile().toString()
+
+// If this is set via -P then we assume it is already running and will not
start it
+def needsLocalJobService = !project.hasProperty("localJobServicePortFile")
Review comment:
Noting that my previous approach did not work. The property was always
set by the time the conditional was checked. I had only verified it against a
job service that I launched externally so did not catch that the logic was
flawed. This approach does work, by setting `needsLocalJobService` earlier.
##########
File path: runners/portability/java/build.gradle
##########
@@ -50,27 +50,32 @@ dependencies {
validatesRunner project(path: project.path, configuration: "testRuntime")
}
-
project.evaluationDependsOn(":sdks:java:core")
+project.evaluationDependsOn(":sdks:python")
project.evaluationDependsOn(":runners:core-java")
-ext.localJobServicePidFile = "${project.buildDir}/local_job_service_pid"
+ext.pythonSdkDir =
project.findProject(":sdks:python").getProjectDir().getAbsoluteFile().toString()
+
+// If this is set via -P then we assume it is already running and will not
start it
+def needsLocalJobService = !project.hasProperty("localJobServicePortFile")
+
ext.localJobServicePortFile = project.hasProperty("localJobServicePortFile") ?
project.property("localJobServicePortFile") :
"${project.buildDir}/local_job_service_port"
+
ext.localJobServiceStdoutFile = "${project.buildDir}/local_job_service_stdout"
+ext.localJobServicePidFile = "${project.buildDir}/local_job_service_pid"
void execInVirtualenv(String... args) {
String shellCommand = ". ${project.ext.envdir}/bin/activate && " +
args.collect { arg -> "'" + arg.replaceAll("'", "\\'") + "'" }.join(" ")
exec {
- workingDir pythonSdkDir
+ workingDir project.ext.pythonSdkDir
Review comment:
The prior reference to `pythonSdkDir` was actually an unbound variable.
##########
File path: runners/portability/java/build.gradle
##########
@@ -108,95 +120,113 @@ startLocalJobService.finalizedBy stopLocalJobService
/**
* Runs Java ValidatesRunner tests against the Universal Local Runner (ULR)
aka local_job_service_main
- * with subprocess SDK harness environments.
+ * with the specified environment type.
*/
-task ulrValidatesRunnerTests(type: Test) {
- dependsOn ":sdks:java:container:docker"
-
- if (!project.hasProperty("localJobServicePortFile")) {
- dependsOn startLocalJobService
+def createUlrValidatesRunnerTask = { name, environmentType ->
+ Task vrTask = tasks.create(name: name, type: Test, group: "Verification") {
+ description "PortableRunner Java docker ValidatesRunner suite"
+ classpath = configurations.validatesRunner
+ systemProperty "beamTestPipelineOptions", JsonOutput.toJson([
+ "--runner=TestUniversalRunner",
+ "--experiments=beam_fn_api",
+ "--defaultEnvironmentType=${environmentType}",
+ "--localJobServicePortFile=${localJobServicePortFile}"
+ ])
+ testClassesDirs =
files(project(":sdks:java:core").sourceSets.test.output.classesDirs)
+ useJUnit {
+ includeCategories 'org.apache.beam.sdk.testing.ValidatesRunner'
+ excludeCategories 'org.apache.beam.sdk.testing.UsesGaugeMetrics'
+ excludeCategories 'org.apache.beam.sdk.testing.UsesOnWindowExpiration'
+ excludeCategories 'org.apache.beam.sdk.testing.UsesBundleFinalizer'
+ excludeCategories 'org.apache.beam.sdk.testing.UsesMapState'
+ excludeCategories 'org.apache.beam.sdk.testing.UsesSetState'
+ excludeCategories 'org.apache.beam.sdk.testing.UsesOrderedListState'
+ excludeCategories 'org.apache.beam.sdk.testing.UsesTestStream'
Review comment:
Previously, only a special case of `UsesTestStreamWithMultipleStages`
was sickbayed. But I got a failure of being unable to understand the URN
`beam:transforms:teststream:v1` so I interpret that to mean it is not supported
at all.
##########
File path: runners/portability/java/build.gradle
##########
@@ -82,8 +87,15 @@ void execBackgroundInVirtualenv(String... args) {
proc.waitFor();
}
-task startLocalJobService {
+task installBeamPythonInVirtualenv {
dependsOn setupVirtualenv
+ doLast {
+ execInVirtualenv "pip", "install", "-e", "."
Review comment:
I did not understand previously that the Beam SDK and requirements were
not installed into the virtualenv. Again, masked by the fact that I only ran it
with `-PlocalJobServicePortFile` hence did not test the virtualenv setup
properly.
##########
File path: runners/portability/java/build.gradle
##########
@@ -50,27 +50,32 @@ dependencies {
validatesRunner project(path: project.path, configuration: "testRuntime")
}
-
project.evaluationDependsOn(":sdks:java:core")
+project.evaluationDependsOn(":sdks:python")
project.evaluationDependsOn(":runners:core-java")
-ext.localJobServicePidFile = "${project.buildDir}/local_job_service_pid"
+ext.pythonSdkDir =
project.findProject(":sdks:python").getProjectDir().getAbsoluteFile().toString()
+
+// If this is set via -P then we assume it is already running and will not
start it
+def needsLocalJobService = !project.hasProperty("localJobServicePortFile")
+
ext.localJobServicePortFile = project.hasProperty("localJobServicePortFile") ?
project.property("localJobServicePortFile") :
"${project.buildDir}/local_job_service_port"
+
ext.localJobServiceStdoutFile = "${project.buildDir}/local_job_service_stdout"
+ext.localJobServicePidFile = "${project.buildDir}/local_job_service_pid"
void execInVirtualenv(String... args) {
String shellCommand = ". ${project.ext.envdir}/bin/activate && " +
args.collect { arg -> "'" + arg.replaceAll("'", "\\'") + "'" }.join(" ")
exec {
- workingDir pythonSdkDir
+ workingDir project.ext.pythonSdkDir
commandLine "sh", "-c", shellCommand
}
}
// Does not background the process, but allows the process to daemonize itself
void execBackgroundInVirtualenv(String... args) {
String shellCommand = ". ${project.ext.envdir}/bin/activate && " +
args.collect { arg -> "'" + arg.replaceAll("'", "\\'") + "'" }.join(" ")
- println "execBackgroundInVirtualEnv: ${shellCommand}"
Review comment:
Left in accidentally ;_;
##########
File path: runners/portability/java/build.gradle
##########
@@ -108,95 +120,113 @@ startLocalJobService.finalizedBy stopLocalJobService
/**
* Runs Java ValidatesRunner tests against the Universal Local Runner (ULR)
aka local_job_service_main
- * with subprocess SDK harness environments.
+ * with the specified environment type.
*/
-task ulrValidatesRunnerTests(type: Test) {
- dependsOn ":sdks:java:container:docker"
-
- if (!project.hasProperty("localJobServicePortFile")) {
- dependsOn startLocalJobService
+def createUlrValidatesRunnerTask = { name, environmentType ->
+ Task vrTask = tasks.create(name: name, type: Test, group: "Verification") {
+ description "PortableRunner Java docker ValidatesRunner suite"
+ classpath = configurations.validatesRunner
+ systemProperty "beamTestPipelineOptions", JsonOutput.toJson([
+ "--runner=TestUniversalRunner",
+ "--experiments=beam_fn_api",
+ "--defaultEnvironmentType=${environmentType}",
+ "--localJobServicePortFile=${localJobServicePortFile}"
+ ])
+ testClassesDirs =
files(project(":sdks:java:core").sourceSets.test.output.classesDirs)
+ useJUnit {
+ includeCategories 'org.apache.beam.sdk.testing.ValidatesRunner'
+ excludeCategories 'org.apache.beam.sdk.testing.UsesGaugeMetrics'
+ excludeCategories 'org.apache.beam.sdk.testing.UsesOnWindowExpiration'
+ excludeCategories 'org.apache.beam.sdk.testing.UsesBundleFinalizer'
Review comment:
`UsesBundleFinalizer` is also new since the initial setup of the VR
gradle task.
##########
File path: runners/portability/java/build.gradle
##########
@@ -108,95 +120,113 @@ startLocalJobService.finalizedBy stopLocalJobService
/**
* Runs Java ValidatesRunner tests against the Universal Local Runner (ULR)
aka local_job_service_main
- * with subprocess SDK harness environments.
+ * with the specified environment type.
*/
-task ulrValidatesRunnerTests(type: Test) {
- dependsOn ":sdks:java:container:docker"
-
- if (!project.hasProperty("localJobServicePortFile")) {
- dependsOn startLocalJobService
+def createUlrValidatesRunnerTask = { name, environmentType ->
+ Task vrTask = tasks.create(name: name, type: Test, group: "Verification") {
+ description "PortableRunner Java docker ValidatesRunner suite"
+ classpath = configurations.validatesRunner
+ systemProperty "beamTestPipelineOptions", JsonOutput.toJson([
+ "--runner=TestUniversalRunner",
+ "--experiments=beam_fn_api",
+ "--defaultEnvironmentType=${environmentType}",
+ "--localJobServicePortFile=${localJobServicePortFile}"
+ ])
+ testClassesDirs =
files(project(":sdks:java:core").sourceSets.test.output.classesDirs)
+ useJUnit {
+ includeCategories 'org.apache.beam.sdk.testing.ValidatesRunner'
+ excludeCategories 'org.apache.beam.sdk.testing.UsesGaugeMetrics'
+ excludeCategories 'org.apache.beam.sdk.testing.UsesOnWindowExpiration'
+ excludeCategories 'org.apache.beam.sdk.testing.UsesBundleFinalizer'
+ excludeCategories 'org.apache.beam.sdk.testing.UsesMapState'
+ excludeCategories 'org.apache.beam.sdk.testing.UsesSetState'
+ excludeCategories 'org.apache.beam.sdk.testing.UsesOrderedListState'
+ excludeCategories 'org.apache.beam.sdk.testing.UsesTestStream'
+ excludeCategories 'org.apache.beam.sdk.testing.UsesTimersInParDo'
+ }
+ filter {
+ // There is not currently a category for excluding these _only_ in
committed mode
+ // https://issues.apache.org/jira/browse/BEAM-10445
+ excludeTestsMatching
'org.apache.beam.sdk.metrics.MetricsTest$CommittedMetricTests.testCommittedCounterMetrics'
+ // https://issues.apache.org/jira/browse/BEAM-10446
+ excludeTestsMatching
'org.apache.beam.sdk.metrics.MetricsTest$CommittedMetricTests.testCommittedDistributionMetrics'
+
+ // This test seems erroneously labeled ValidatesRunner
+ excludeTestsMatching
'org.apache.beam.sdk.schemas.AvroSchemaTest.testAvroPipelineGroupBy'
+
+ // Teardown not called in exceptions
+ // https://issues.apache.org/jira/browse/BEAM-10447
+ excludeTestsMatching
'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInFinishBundle'
+ excludeTestsMatching
'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInFinishBundleStateful'
+ excludeTestsMatching
'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInProcessElement'
+ excludeTestsMatching
'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInProcessElementStateful'
+ excludeTestsMatching
'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInSetup'
+ excludeTestsMatching
'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInSetupStateful'
+ excludeTestsMatching
'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInStartBundle'
+ excludeTestsMatching
'org.apache.beam.sdk.transforms.ParDoLifecycleTest.testTeardownCalledAfterExceptionInStartBundleStateful'
+
+ // Only known window fns supported, not general window merging
+ // https://issues.apache.org/jira/browse/BEAM-10448
+ excludeTestsMatching
'org.apache.beam.sdk.transforms.windowing.WindowTest.testMergingCustomWindows'
+ excludeTestsMatching
'org.apache.beam.sdk.transforms.windowing.WindowTest.testMergingCustomWindowsKeyedCollection'
+ excludeTestsMatching
'org.apache.beam.sdk.transforms.windowing.WindowingTest.testMergingWindowing'
+ excludeTestsMatching
'org.apache.beam.sdk.transforms.windowing.WindowingTest.testNonPartitioningWindowing'
+ excludeTestsMatching
'org.apache.beam.sdk.transforms.GroupByKeyTest$WindowTests.testGroupByKeyMergingWindows'
+
+ // Flatten with empty PCollections hangs
+ // https://issues.apache.org/jira/browse/BEAM-10450
+ excludeTestsMatching
'org.apache.beam.sdk.transforms.FlattenTest.testEmptyFlattenAsSideInput'
+ excludeTestsMatching
'org.apache.beam.sdk.transforms.FlattenTest.testFlattenPCollectionsEmpty'
+ excludeTestsMatching
'org.apache.beam.sdk.transforms.FlattenTest.testFlattenPCollectionsEmptyThenParDo'
+
+ // Empty side inputs hang
+ // https://issues.apache.org/jira/browse/BEAM-10449
+ excludeTestsMatching
'org.apache.beam.sdk.transforms.ViewTest.testWindowedSideInputFixedToFixedWithDefault'
+ excludeTestsMatching
'org.apache.beam.sdk.transforms.ViewTest.testEmptyIterableSideInput'
+ excludeTestsMatching
'org.apache.beam.sdk.transforms.ViewTest.testEmptySingletonSideInput'
+ excludeTestsMatching
'org.apache.beam.sdk.transforms.ViewTest.testEmptyListSideInput'
+ excludeTestsMatching
'org.apache.beam.sdk.transforms.ViewTest.testEmptyMultimapSideInput'
+ excludeTestsMatching
'org.apache.beam.sdk.transforms.ViewTest.testEmptyMultimapSideInputWithNonDeterministicKeyCoder'
+ excludeTestsMatching
'org.apache.beam.sdk.transforms.ViewTest.testEmptyMapSideInput'
+ excludeTestsMatching
'org.apache.beam.sdk.transforms.ViewTest.testEmptyMapSideInputWithNonDeterministicKeyCoder'
+
+ // Misc failures
+ // https://issues.apache.org/jira/browse/BEAM-10451
+ excludeTestsMatching
'org.apache.beam.sdk.transforms.CombineTest$WindowingTests.testGlobalCombineWithDefaultsAndTriggers'
+ excludeTestsMatching
'org.apache.beam.sdk.transforms.CombineTest$WindowingTests.testSessionsCombine'
+ excludeTestsMatching
'org.apache.beam.sdk.transforms.CombineTest$WindowingTests.testSessionsCombineWithContext'
+
+ // https://issues.apache.org/jira/browse/BEAM-10454
+ excludeTestsMatching
'org.apache.beam.sdk.testing.PAssertTest.testWindowedIsEqualTo'
+
+ // https://issues.apache.org/jira/browse/BEAM-10453
+ excludeTestsMatching
'org.apache.beam.sdk.transforms.ReshuffleTest.testReshuffleWithTimestampsStreaming'
+
+ // https://issues.apache.org/jira/browse/BEAM-10452
+ excludeTestsMatching
'org.apache.beam.sdk.transforms.CombineTest$BasicTests.testHotKeyCombiningWithAccumulationMode'
+
+ // https://issues.apache.org/jira/browse/BEAM-10995
+ excludeTestsMatching
'org.apache.beam.sdk.transforms.windowing.WindowingTest.testWindowPreservation'
Review comment:
This failure is new and seems problematic. It is an elementary test from
the very beginning of Beam. It was not failing before.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
