This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from 05861613f48 Fix typo and remove extra button (#28965)
add a7c6728b004 Fix publish snapshots github actions (#28961)
add a342d2817a9 Github Workflow Replacement for Jenkins Jobs,
beam_LoadTests_Python_GBK_Flink_Batch; beam_Java_LoadTests_Combine_Smoke_PR
(#28897)
add 2dc766512c0 Bump github.com/aws/aws-sdk-go-v2 from 1.21.1 to 1.21.2 in
/sdks (#28978)
add 3a749a9a6ba Add GitHub Workflow Replacements for Jenkins
job_Python_LoadTests_Smoke and job_Java_LoadTests_GBK_Smoke (#28854)
add 71b8e93b6d3 Add GitHub Workflow Replacements for Jenkins
job_Python_ParDo_DataFlow and job_Python_ParDo_Flink (#28894)
add c0af78a9e98 Revert "Setup Java 21 container (#28833)" (#28968)
add dad13cfbc9a Fix `watch_file_pattern` condition in RunInference (#28948)
add 58e3bdd678b Shutdown more Jenkins PreCommits (#28840)
add 3a45ecf4b27 Updated the DoFn documentation with pickling (#28970)
add f4f1c3c22d9 Add LoadTests Java ParDo Dataflow V2 workflows (#28744)
add dd40403752f beam_LoadTests_Python_GBK_reiterate_Dataflow (#28861)
add 155f8507d07 Add CloudML Benchmarks Dataflow workflow (#28941)
add d636dd32aca Add KeyedModelHandler content to the multi-model ML page
(#28987)
add 250483c5f1f Bump org.checkerframework:checkerframework-gradle-plugin
(#28979)
add 47d0fd566f8 Set MIN_PER_PARTITION_MEMORY to 10 MiB instead of 4 MiB to
match the change in the backend limit. (#28964)
add f93f62e9d82 reorder args to avoid newline issues (#28994)
No new revisions were added by this update.
Summary of changes:
.../gradle-command-self-hosted-action/action.yml | 11 +-
...ts.yml => beam_CloudML_Benchmarks_Dataflow.yml} | 44 +++---
...s.yml => beam_Java_LoadTests_Combine_Smoke.yml} | 54 ++++---
...s_Java_CoGBK_Dataflow_V2_Batch_JavaVersions.yml | 8 +-
...va_CoGBK_Dataflow_V2_Streaming_JavaVersions.yml | 8 +-
...LoadTests_Java_GBK_Dataflow_V2_Batch_Java11.yml | 14 +-
...LoadTests_Java_GBK_Dataflow_V2_Batch_Java17.yml | 14 +-
...Tests_Java_GBK_Dataflow_V2_Streaming_Java11.yml | 14 +-
...Tests_Java_GBK_Dataflow_V2_Streaming_Java17.yml | 14 +-
.../workflows/beam_LoadTests_Java_GBK_Smoke.yml | 116 +++++++++++++++
..._Java_ParDo_Dataflow_V2_Batch_JavaVersions.yml} | 74 +++++-----
...a_ParDo_Dataflow_V2_Streaming_JavaVersions.yml} | 74 +++++-----
...oadTests_Python_Combine_Dataflow_Streaming.yml} | 79 +++++------
.../beam_LoadTests_Python_GBK_Flink_Batch.yml | 155 +++++++++++++++++++++
...dTests_Python_GBK_reiterate_Dataflow_Batch.yml} | 51 +++++--
...ts_Python_GBK_reiterate_Dataflow_Streaming.yml} | 51 +++++--
.../beam_LoadTests_Python_ParDo_Dataflow_Batch.yml | 126 +++++++++++++++++
...m_LoadTests_Python_ParDo_Dataflow_Streaming.yml | 126 +++++++++++++++++
.../beam_LoadTests_Python_ParDo_Flink_Batch.yml | 131 +++++++++++++++++
...beam_LoadTests_Python_ParDo_Flink_Streaming.yml | 155 +++++++++++++++++++++
...apshots.yml => beam_LoadTests_Python_Smoke.yml} | 56 +++++---
.../beam_PerformanceTests_xlang_KafkaIO_Python.yml | 4 +-
.../beam_PostCommit_Java_Examples_Dataflow_ARM.yml | 2 +-
...m_PostCommit_Java_Examples_Dataflow_V2_Java.yml | 2 +-
.../beam_PostCommit_Java_Jpms_Dataflow_Java17.yml | 2 +-
.../beam_PostCommit_Java_Jpms_Direct_Java17.yml | 2 +-
...am_PostCommit_Java_Nexmark_Dataflow_V2_Java.yml | 2 +-
.../beam_PostCommit_TransformService_Direct.yml | 2 +-
.../beam_PostCommit_XVR_GoUsingJava_Dataflow.yml | 1 +
...eam_PreCommit_Java_Examples_Dataflow_Java11.yml | 2 +-
...eam_PreCommit_Java_Examples_Dataflow_Java17.yml | 2 +-
.github/workflows/beam_PreCommit_SQL_Java11.yml | 2 +-
.github/workflows/beam_PreCommit_SQL_Java17.yml | 2 +-
.../workflows/beam_Publish_Beam_SDK_Snapshots.yml | 11 ++
.../workflows/beam_Publish_Docker_Snapshots.yml | 11 ++
.../beam_CloudML_Benchmarks_Dataflow_arguments.txt | 23 +++
..._Smoke_CombineLoadTest_load_test_Dataflow-1.txt | 26 ++++
..._Smoke_CombineLoadTest_load_test_Dataflow-2.txt | 26 ++++
..._Smoke_CombineLoadTest_load_test_Dataflow-3.txt | 26 ++++
...Dataflow_V2_Batch_JavaVersions_100_counters.txt | 28 ++++
..._Dataflow_V2_Batch_JavaVersions_10_counters.txt | 28 ++++
...rDo_Dataflow_V2_Batch_JavaVersions_10_times.txt | 28 ++++
...Do_Dataflow_V2_Batch_JavaVersions_200_times.txt | 28 ++++
...flow_V2_Streaming_JavaVersions_100_counters.txt | 29 ++++
...aflow_V2_Streaming_JavaVersions_10_counters.txt | 29 ++++
...Dataflow_V2_Streaming_JavaVersions_10_times.txt | 29 ++++
...ataflow_V2_Streaming_JavaVersions_200_times.txt | 29 ++++
.../java_Smoke_GroupByKey_Dataflow.txt | 23 +++
.../java_Smoke_GroupByKey_Direct.txt | 20 +++
.../java_Smoke_GroupByKey_Flink.txt | 21 +++
.../java_Smoke_GroupByKey_Spark.txt | 22 +++
...bine_Dataflow_Streaming_2GB_10_byte_records.txt | 29 ++++
...hon_Combine_Dataflow_Streaming_2GB_Fanout_4.txt | 30 ++++
...hon_Combine_Dataflow_Streaming_2GB_Fanout_8.txt | 30 ++++
.../python_GBK_Flink_Batch_2GB_of_100B_records.txt | 28 ++++
.../python_GBK_Flink_Batch_2GB_of_10B_records.txt | 28 ++++
...nout_4_times_with_2GB_10-byte_records_total.txt | 28 ++++
...nout_8_times_with_2GB_10-byte_records_total.txt | 28 ++++
...K_Flink_Batch_reiterate_4_times_10kB_values.txt | 28 ++++
...ataflow_Batch_reiterate_4_times_10kB_values.txt | 28 ++++
...Dataflow_Batch_reiterate_4_times_2MB_values.txt | 28 ++++
...low_Streaming_reiterate_4_times_10kB_values.txt | 30 ++++
...flow_Streaming_reiterate_4_times_2MB_values.txt | 30 ++++
.../python_ParDo_Dataflow_Batch_100_Counters.txt | 29 ++++
.../python_ParDo_Dataflow_Batch_10_Counters.txt | 29 ++++
.../python_ParDo_Dataflow_Batch_10_Iterations.txt | 29 ++++
.../python_ParDo_Dataflow_Batch_200_Iterations.txt | 29 ++++
...ython_ParDo_Dataflow_Streaming_100_Counters.txt | 31 +++++
...python_ParDo_Dataflow_Streaming_10_Counters.txt | 31 +++++
...thon_ParDo_Dataflow_Streaming_10_Iterations.txt | 31 +++++
...hon_ParDo_Dataflow_Streaming_200_Iterations.txt | 31 +++++
.../python_ParDo_Flink_Batch_10_Counters.txt | 29 ++++
.../python_ParDo_Flink_Batch_10_Iterations.txt | 29 ++++
.../python_ParDo_Flink_Batch_200_Iterations.txt | 29 ++++
.../python_ParDo_Flink_Streaming_100_Counters.txt | 31 +++++
.../python_ParDo_Flink_Streaming_10_Counters.txt | 31 +++++
.../python_ParDo_Flink_Streaming_10_Iterations.txt | 32 +++++
...python_ParDo_Flink_Streaming_200_Iterations.txt | 31 +++++
.../python_ParDo_Flink_Streaming_5_Iterations.txt | 35 +++++
.../python_Smoke_GroupByKey_Dataflow.txt | 23 +++
.../python_Smoke_GroupByKey_Direct.txt | 23 +++
.test-infra/jenkins/JavaTestProperties.groovy | 7 +-
.test-infra/jenkins/NexmarkBuilder.groovy | 4 +-
.../job_LoadTests_CoGBK_Dataflow_V2_Java11.groovy | 2 +-
.../job_LoadTests_CoGBK_Dataflow_V2_Java17.groovy | 2 +-
.../job_LoadTests_GBK_Dataflow_V2_Java11.groovy | 2 +-
.../job_LoadTests_GBK_Dataflow_V2_Java17.groovy | 2 +-
.../job_LoadTests_ParDo_Dataflow_V2_Java11.groovy | 2 +-
.../job_LoadTests_ParDo_Dataflow_V2_Java17.groovy | 2 +-
...tCommit_Java_Examples_Dataflow_V2_Java11.groovy | 2 +-
...tCommit_Java_Examples_Dataflow_V2_Java17.groovy | 2 +-
...job_PostCommit_Java_Jpms_Dataflow_Java17.groovy | 2 +-
.../job_PostCommit_Java_Jpms_Direct_Java17.groovy | 2 +-
.../job_PostCommit_TransformService_Direct.groovy | 2 +-
.../job_PreCommit_Java_Examples_Dataflow.groovy | 43 ------
..._PreCommit_Java_Examples_Dataflow_Java11.groovy | 58 --------
.test-infra/jenkins/job_PreCommit_Java_IOs.groovy | 54 +++----
...ava_PortableValidatesRunner_Flink_Docker.groovy | 41 ------
.../job_PreCommit_Java_Spark3_Versions.groovy | 37 -----
.../jenkins/job_PreCommit_Portable_Python.groovy | 59 --------
.test-infra/jenkins/job_PreCommit_SQL.groovy | 53 -------
.../jenkins/job_PreCommit_SQL_Java11.groovy | 52 -------
.../jenkins/job_PreCommit_SQL_Java17.groovy | 51 -------
..._Precommit_Java_Examples_Dataflow_Java17.groovy | 58 --------
CHANGES.md | 1 +
build.gradle.kts | 14 +-
buildSrc/build.gradle.kts | 2 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 37 +----
.../runners/core/construction/Environments.java | 6 +-
.../core/construction/EnvironmentsTest.java | 6 +-
.../google-cloud-dataflow-java/arm/build.gradle | 6 +-
runners/google-cloud-dataflow-java/build.gradle | 6 +-
.../beam/runners/dataflow/DataflowRunner.java | 7 +-
runners/spark/spark_runner.gradle | 5 +-
sdks/go.mod | 2 +-
sdks/go.sum | 3 +-
sdks/java/container/Dockerfile | 3 +-
sdks/java/container/common.gradle | 14 +-
sdks/java/container/java21/build.gradle | 30 ----
sdks/java/container/java21/option-jamm.json | 12 --
.../pubsublite/internal/SubscribeTransform.java | 2 +-
sdks/java/testing/jpms-tests/build.gradle | 21 +--
sdks/java/testing/test-utils/build.gradle | 29 ++--
.../testutils/jvmverification/JvmVerification.java | 16 +--
sdks/python/apache_beam/ml/inference/base.py | 5 +-
sdks/python/apache_beam/ml/inference/base_test.py | 22 +++
settings.gradle.kts | 1 -
.../en/documentation/ml/multi-model-pipelines.md | 58 ++++++++
.../content/en/documentation/programming-guide.md | 11 +-
.../documentation/sdks/python-machine-learning.md | 42 +++---
website/www/site/content/en/roadmap/java-sdk.md | 6 +-
131 files changed, 2694 insertions(+), 927 deletions(-)
copy .github/workflows/{beam_Publish_Docker_Snapshots.yml =>
beam_CloudML_Benchmarks_Dataflow.yml} (66%)
copy .github/workflows/{beam_Publish_Docker_Snapshots.yml =>
beam_Java_LoadTests_Combine_Smoke.yml} (52%)
create mode 100644 .github/workflows/beam_LoadTests_Java_GBK_Smoke.yml
copy
.github/workflows/{beam_LoadTests_Java_CoGBK_Dataflow_V2_Batch_JavaVersions.yml
=> beam_LoadTests_Java_ParDo_Dataflow_V2_Batch_JavaVersions.yml} (63%)
copy
.github/workflows/{beam_LoadTests_Java_CoGBK_Dataflow_V2_Streaming_JavaVersions.yml
=> beam_LoadTests_Java_ParDo_Dataflow_V2_Streaming_JavaVersions.yml} (66%)
copy .github/workflows/{beam_PerformanceTests_xlang_KafkaIO_Python.yml =>
beam_LoadTests_Python_Combine_Dataflow_Streaming.yml} (51%)
create mode 100644 .github/workflows/beam_LoadTests_Python_GBK_Flink_Batch.yml
copy .github/workflows/{beam_Publish_Docker_Snapshots.yml =>
beam_LoadTests_Python_GBK_reiterate_Dataflow_Batch.yml} (53%)
copy .github/workflows/{beam_Publish_Docker_Snapshots.yml =>
beam_LoadTests_Python_GBK_reiterate_Dataflow_Streaming.yml} (52%)
create mode 100644
.github/workflows/beam_LoadTests_Python_ParDo_Dataflow_Batch.yml
create mode 100644
.github/workflows/beam_LoadTests_Python_ParDo_Dataflow_Streaming.yml
create mode 100644
.github/workflows/beam_LoadTests_Python_ParDo_Flink_Batch.yml
create mode 100644
.github/workflows/beam_LoadTests_Python_ParDo_Flink_Streaming.yml
copy .github/workflows/{beam_Publish_Docker_Snapshots.yml =>
beam_LoadTests_Python_Smoke.yml} (53%)
create mode 100644
.github/workflows/load-tests-job-configs/beam_CloudML_Benchmarks_Dataflow_arguments.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_LoadTests_Combine_Smoke_CombineLoadTest_load_test_Dataflow-1.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_LoadTests_Combine_Smoke_CombineLoadTest_load_test_Dataflow-2.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_LoadTests_Combine_Smoke_CombineLoadTest_load_test_Dataflow-3.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Batch_JavaVersions_100_counters.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Batch_JavaVersions_10_counters.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Batch_JavaVersions_10_times.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Batch_JavaVersions_200_times.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Streaming_JavaVersions_100_counters.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Streaming_JavaVersions_10_counters.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Streaming_JavaVersions_10_times.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_ParDo_Dataflow_V2_Streaming_JavaVersions_200_times.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_Smoke_GroupByKey_Dataflow.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_Smoke_GroupByKey_Direct.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_Smoke_GroupByKey_Flink.txt
create mode 100644
.github/workflows/load-tests-job-configs/java_Smoke_GroupByKey_Spark.txt
create mode 100644
.github/workflows/load-tests-job-configs/python_Combine_Dataflow_Streaming_2GB_10_byte_records.txt
create mode 100644
.github/workflows/load-tests-job-configs/python_Combine_Dataflow_Streaming_2GB_Fanout_4.txt
create mode 100644
.github/workflows/load-tests-job-configs/python_Combine_Dataflow_Streaming_2GB_Fanout_8.txt
create mode 100644
.github/workflows/load-tests-job-configs/python_GBK_Flink_Batch_2GB_of_100B_records.txt
create mode 100644
.github/workflows/load-tests-job-configs/python_GBK_Flink_Batch_2GB_of_10B_records.txt
create mode 100644
.github/workflows/load-tests-job-configs/python_GBK_Flink_Batch_fanout_4_times_with_2GB_10-byte_records_total.txt
create mode 100644
.github/workflows/load-tests-job-configs/python_GBK_Flink_Batch_fanout_8_times_with_2GB_10-byte_records_total.txt
create mode 100644
.github/workflows/load-tests-job-configs/python_GBK_Flink_Batch_reiterate_4_times_10kB_values.txt
create mode 100644
.github/workflows/load-tests-job-configs/python_GBK_reiterate_Dataflow_Batch_reiterate_4_times_10kB_values.txt
create mode 100644
.github/workflows/load-tests-job-configs/python_GBK_reiterate_Dataflow_Batch_reiterate_4_times_2MB_values.txt
create mode 100644
.github/workflows/load-tests-job-configs/python_GBK_reiterate_Dataflow_Streaming_reiterate_4_times_10kB_values.txt
create mode 100644
.github/workflows/load-tests-job-configs/python_GBK_reiterate_Dataflow_Streaming_reiterate_4_times_2MB_values.txt
create mode 100644
.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Batch_100_Counters.txt
create mode 100644
.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Batch_10_Counters.txt
create mode 100644
.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Batch_10_Iterations.txt
create mode 100644
.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Batch_200_Iterations.txt
create mode 100644
.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Streaming_100_Counters.txt
create mode 100644
.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Streaming_10_Counters.txt
create mode 100644
.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Streaming_10_Iterations.txt
create mode 100644
.github/workflows/load-tests-job-configs/python_ParDo_Dataflow_Streaming_200_Iterations.txt
create mode 100644
.github/workflows/load-tests-job-configs/python_ParDo_Flink_Batch_10_Counters.txt
create mode 100644
.github/workflows/load-tests-job-configs/python_ParDo_Flink_Batch_10_Iterations.txt
create mode 100644
.github/workflows/load-tests-job-configs/python_ParDo_Flink_Batch_200_Iterations.txt
create mode 100644
.github/workflows/load-tests-job-configs/python_ParDo_Flink_Streaming_100_Counters.txt
create mode 100644
.github/workflows/load-tests-job-configs/python_ParDo_Flink_Streaming_10_Counters.txt
create mode 100644
.github/workflows/load-tests-job-configs/python_ParDo_Flink_Streaming_10_Iterations.txt
create mode 100644
.github/workflows/load-tests-job-configs/python_ParDo_Flink_Streaming_200_Iterations.txt
create mode 100644
.github/workflows/load-tests-job-configs/python_ParDo_Flink_Streaming_5_Iterations.txt
create mode 100644
.github/workflows/load-tests-job-configs/python_Smoke_GroupByKey_Dataflow.txt
create mode 100644
.github/workflows/load-tests-job-configs/python_Smoke_GroupByKey_Direct.txt
delete mode 100644
.test-infra/jenkins/job_PreCommit_Java_Examples_Dataflow.groovy
delete mode 100644
.test-infra/jenkins/job_PreCommit_Java_Examples_Dataflow_Java11.groovy
delete mode 100644
.test-infra/jenkins/job_PreCommit_Java_PortableValidatesRunner_Flink_Docker.groovy
delete mode 100644
.test-infra/jenkins/job_PreCommit_Java_Spark3_Versions.groovy
delete mode 100644 .test-infra/jenkins/job_PreCommit_Portable_Python.groovy
delete mode 100644 .test-infra/jenkins/job_PreCommit_SQL.groovy
delete mode 100644 .test-infra/jenkins/job_PreCommit_SQL_Java11.groovy
delete mode 100644 .test-infra/jenkins/job_PreCommit_SQL_Java17.groovy
delete mode 100644
.test-infra/jenkins/job_Precommit_Java_Examples_Dataflow_Java17.groovy
delete mode 100644 sdks/java/container/java21/build.gradle
delete mode 100644 sdks/java/container/java21/option-jamm.json
