This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to tag nightly-master
in repository https://gitbox.apache.org/repos/asf/beam.git.
*** WARNING: tag nightly-master was modified! ***
from 0af8c3b (commit)
to 153876f (commit)
from 0af8c3b Merge pull request #14152 from [BEAM-11926] Improve error
when missing Beam schema for BigqueryIO
add 0f6aacf [BEAM-11910] Increase the bag page limit for continuation
pages from 8MB to 32MB.
add c925ff8 Merge pull request #14154: [BEAM-11910] Increase the bag page
limit for continuation pages
add 9c0f608 [BEAM-9282] Move structured streaming runner into Spark 2
specific module
add 47fd176 [BEAM-9282] Separate modules for Spark 2/3
add 17710e5 [BEAM-9282] Separate modules for Spark 2/3 job-server
add e86862c [BEAM-9282] Separate modules for Spark 2/3 job-server
container
add 44b7a87 [BEAM-7092] Run PostCommit tests for Spark 3 module too
add cc29b5b [BEAM-7092] Update tests invocation for Spark 2 module
add b234bb4 [BEAM-9283] Add Spark 3 test jobs to the CI (Java 11)
add 07e52fc [BEAM-11654] Publish Spark 2 and 3 specific Job-Server
containers
add d9f3157 [BEAM-7092] Add paranamer 2.8 license to container (Spark 3 /
Avro)
add 153876f Merge pull request #14216: [BEAM-7093] Support Spark 3 in
Spark runner
No new revisions were added by this update.
Summary of changes:
.test-infra/jenkins/CommonTestProperties.groovy | 4 +-
...ommit_CrossLanguageValidatesRunner_Spark.groovy | 3 +-
.../job_PostCommit_Java_Nexmark_Spark.groovy | 8 +-
...Java_PortableValidatesRunner_Spark_Batch.groovy | 3 +-
..._PortableValidatesRunner_Spark_Streaming.groovy | 3 +-
...ob_PostCommit_Java_ValidatesRunner_Spark.groovy | 3 +-
...ValidatesRunner_SparkStructuredStreaming.groovy | 2 +-
...ommit_Java_ValidatesRunner_Spark_Java11.groovy} | 13 ++-
build.gradle.kts | 5 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 7 +-
examples/java/build.gradle | 2 +-
examples/kotlin/build.gradle | 2 +-
release/build.gradle.kts | 2 +-
release/src/main/scripts/run_rc_validation.sh | 2 +-
runners/core-construction-java/build.gradle | 3 +
.../dataflow/worker/WindmillStateReader.java | 22 +++--
.../dataflow/worker/WindmillStateReaderTest.java | 20 ++---
runners/{flink/1.8 => spark/2}/build.gradle | 13 +--
.../{flink/1.9 => spark/2}/job-server/build.gradle | 4 +-
.../2/job-server/container}/build.gradle | 5 +-
.../SparkStructuredStreamingPipelineOptions.java | 0
.../SparkStructuredStreamingPipelineResult.java | 0
.../SparkStructuredStreamingRunner.java | 0
.../SparkStructuredStreamingRunnerRegistrar.java | 0
.../aggregators/AggregatorsAccumulator.java | 0
.../aggregators/NamedAggregators.java | 0
.../aggregators/NamedAggregatorsAccumulator.java | 0
.../aggregators/package-info.java | 0
.../structuredstreaming/examples/WordCount.java | 0
.../metrics/AggregatorMetric.java | 0
.../metrics/AggregatorMetricSource.java | 0
.../metrics/CompositeSource.java | 0
.../metrics/MetricsAccumulator.java | 0
.../MetricsContainerStepMapAccumulator.java | 0
.../metrics/SparkBeamMetric.java | 0
.../metrics/SparkBeamMetricSource.java | 0
.../metrics/SparkMetricsContainerStepMap.java | 0
.../metrics/WithMetricsSupport.java | 0
.../structuredstreaming/metrics/package-info.java | 0
.../metrics/sink/CodahaleCsvSink.java | 0
.../metrics/sink/CodahaleGraphiteSink.java | 0
.../metrics/sink/package-info.java | 0
.../spark/structuredstreaming/package-info.java | 0
.../translation/PipelineTranslator.java | 0
.../translation/SchemaHelpers.java | 0
.../translation/SparkTransformOverrides.java | 0
.../translation/TransformTranslator.java | 0
.../translation/TranslationContext.java | 0
.../translation/batch/AggregatorCombiner.java | 0
.../batch/CombinePerKeyTranslatorBatch.java | 0
.../CreatePCollectionViewTranslatorBatch.java | 0
.../translation/batch/DatasetSourceBatch.java | 0
.../translation/batch/DoFnFunction.java | 0
.../translation/batch/DoFnRunnerWithMetrics.java | 0
.../translation/batch/FlattenTranslatorBatch.java | 0
.../batch/GroupByKeyTranslatorBatch.java | 0
.../translation/batch/ImpulseTranslatorBatch.java | 0
.../translation/batch/ParDoTranslatorBatch.java | 0
.../translation/batch/PipelineTranslatorBatch.java | 0
.../translation/batch/ProcessContext.java | 0
.../batch/ReadSourceTranslatorBatch.java | 0
.../batch/ReshuffleTranslatorBatch.java | 0
.../batch/WindowAssignTranslatorBatch.java | 0
.../GroupAlsoByWindowViaOutputBufferFn.java | 0
.../batch/functions/NoOpStepContext.java | 0
.../batch/functions/SparkSideInputReader.java | 0
.../translation/batch/functions/package-info.java | 0
.../translation/batch/package-info.java | 0
.../translation/helpers/CoderHelpers.java | 0
.../translation/helpers/EncoderHelpers.java | 0
.../translation/helpers/KVHelpers.java | 0
.../translation/helpers/MultiOuputCoder.java | 0
.../translation/helpers/RowHelpers.java | 0
.../translation/helpers/SideInputBroadcast.java | 0
.../translation/helpers/WindowingHelpers.java | 0
.../translation/helpers/package-info.java | 0
.../translation/package-info.java | 0
.../streaming/DatasetSourceStreaming.java | 0
.../streaming/PipelineTranslatorStreaming.java | 0
.../streaming/ReadSourceTranslatorStreaming.java | 0
.../translation/streaming/package-info.java | 0
.../translation/utils/CachedSideInputReader.java | 0
.../translation/utils/SideInputStorage.java | 0
.../translation/utils/package-info.java | 0
...parkStructuredStreamingRunnerRegistrarTest.java | 0
.../StructuredStreamingPipelineStateTest.java | 0
.../aggregators/metrics/sink/InMemoryMetrics.java | 0
.../metrics/sink/InMemoryMetricsSinkRule.java | 0
.../metrics/sink/SparkMetricsSinkTest.java | 0
.../metrics/BeamMetricTest.java | 0
.../translation/batch/CombineTest.java | 0
.../translation/batch/ComplexSourceTest.java | 0
.../translation/batch/FlattenTest.java | 0
.../translation/batch/GroupByKeyTest.java | 0
.../translation/batch/ParDoTest.java | 0
.../translation/batch/SimpleSourceTest.java | 0
.../translation/batch/WindowAssignTest.java | 0
.../translation/helpers/EncoderHelpersTest.java | 0
.../translation/streaming/SimpleSourceTest.java | 0
.../utils/SerializationDebugger.java | 0
.../structuredstreaming/utils/package-info.java | 0
runners/{flink/1.8 => spark/3}/build.gradle | 13 +--
.../{flink/1.9 => spark/3}/job-server/build.gradle | 4 +-
.../3/job-server/container}/build.gradle | 5 +-
...ld.gradle => spark_job_server_container.gradle} | 10 +--
.../{build.gradle => spark_job_server.gradle} | 0
.../spark/{build.gradle => spark_runner.gradle} | 96 +++++++++++++++++++---
sdks/go/test/build.gradle | 8 +-
.../container/license_scripts/dep_urls_java.yaml | 2 +
sdks/java/io/hadoop-format/build.gradle | 2 +-
sdks/java/testing/load-tests/build.gradle | 2 +-
sdks/java/testing/nexmark/build.gradle | 4 +-
.../runners/portability/spark_runner.py | 2 +-
.../runners/portability/spark_runner_test.py | 2 +-
.../portability/spark_uber_jar_job_server.py | 2 +-
sdks/python/test-suites/portable/common.gradle | 10 +--
settings.gradle.kts | 9 +-
.../site/content/en/contribute/release-guide.md | 2 +-
.../site/content/en/documentation/runners/spark.md | 4 +-
.../en/documentation/sdks/java/testing/nexmark.md | 4 +-
120 files changed, 205 insertions(+), 102 deletions(-)
copy
.test-infra/jenkins/{job_PostCommit_Java_ValidatesRunner_Flink_Java11.groovy =>
job_PostCommit_Java_ValidatesRunner_Spark_Java11.groovy} (82%)
copy runners/{flink/1.8 => spark/2}/build.gradle (80%)
copy runners/{flink/1.9 => spark/2}/job-server/build.gradle (91%)
copy runners/{flink/1.8/job-server-container =>
spark/2/job-server/container}/build.gradle (85%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingPipelineOptions.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingPipelineResult.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingRunner.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingRunnerRegistrar.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/aggregators/AggregatorsAccumulator.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/aggregators/NamedAggregators.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/aggregators/NamedAggregatorsAccumulator.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/aggregators/package-info.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/examples/WordCount.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/AggregatorMetric.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/AggregatorMetricSource.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/CompositeSource.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/MetricsAccumulator.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/MetricsContainerStepMapAccumulator.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/SparkBeamMetric.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/SparkBeamMetricSource.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/SparkMetricsContainerStepMap.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/WithMetricsSupport.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/package-info.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/sink/CodahaleCsvSink.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/sink/CodahaleGraphiteSink.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/metrics/sink/package-info.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/package-info.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/PipelineTranslator.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/SchemaHelpers.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/SparkTransformOverrides.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TransformTranslator.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/TranslationContext.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/AggregatorCombiner.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/CombinePerKeyTranslatorBatch.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/CreatePCollectionViewTranslatorBatch.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/DatasetSourceBatch.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/DoFnFunction.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/DoFnRunnerWithMetrics.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/FlattenTranslatorBatch.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/GroupByKeyTranslatorBatch.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ImpulseTranslatorBatch.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ParDoTranslatorBatch.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/PipelineTranslatorBatch.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ProcessContext.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ReadSourceTranslatorBatch.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ReshuffleTranslatorBatch.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/WindowAssignTranslatorBatch.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/functions/GroupAlsoByWindowViaOutputBufferFn.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/functions/NoOpStepContext.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/functions/SparkSideInputReader.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/functions/package-info.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/package-info.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/CoderHelpers.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/EncoderHelpers.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/KVHelpers.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/MultiOuputCoder.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/RowHelpers.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/SideInputBroadcast.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/WindowingHelpers.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/package-info.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/package-info.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/DatasetSourceStreaming.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/PipelineTranslatorStreaming.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/ReadSourceTranslatorStreaming.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/package-info.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/utils/CachedSideInputReader.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/utils/SideInputStorage.java
(100%)
rename runners/spark/{ =>
2}/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/utils/package-info.java
(100%)
rename runners/spark/{ =>
2}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/SparkStructuredStreamingRunnerRegistrarTest.java
(100%)
rename runners/spark/{ =>
2}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/StructuredStreamingPipelineStateTest.java
(100%)
rename runners/spark/{ =>
2}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/aggregators/metrics/sink/InMemoryMetrics.java
(100%)
rename runners/spark/{ =>
2}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/aggregators/metrics/sink/InMemoryMetricsSinkRule.java
(100%)
rename runners/spark/{ =>
2}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/aggregators/metrics/sink/SparkMetricsSinkTest.java
(100%)
rename runners/spark/{ =>
2}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/metrics/BeamMetricTest.java
(100%)
rename runners/spark/{ =>
2}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/CombineTest.java
(100%)
rename runners/spark/{ =>
2}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ComplexSourceTest.java
(100%)
rename runners/spark/{ =>
2}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/FlattenTest.java
(100%)
rename runners/spark/{ =>
2}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/GroupByKeyTest.java
(100%)
rename runners/spark/{ =>
2}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/ParDoTest.java
(100%)
rename runners/spark/{ =>
2}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/SimpleSourceTest.java
(100%)
rename runners/spark/{ =>
2}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/batch/WindowAssignTest.java
(100%)
rename runners/spark/{ =>
2}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/EncoderHelpersTest.java
(100%)
rename runners/spark/{ =>
2}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/translation/streaming/SimpleSourceTest.java
(100%)
rename runners/spark/{ =>
2}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/utils/SerializationDebugger.java
(100%)
rename runners/spark/{ =>
2}/src/test/java/org/apache/beam/runners/spark/structuredstreaming/utils/package-info.java
(100%)
copy runners/{flink/1.8 => spark/3}/build.gradle (80%)
copy runners/{flink/1.9 => spark/3}/job-server/build.gradle (91%)
copy runners/{flink/1.8/job-server-container =>
spark/3/job-server/container}/build.gradle (85%)
rename runners/spark/job-server/container/{build.gradle =>
spark_job_server_container.gradle} (87%)
rename runners/spark/job-server/{build.gradle => spark_job_server.gradle}
(100%)
rename runners/spark/{build.gradle => spark_runner.gradle} (79%)
