This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to tag nightly-master
in repository https://gitbox.apache.org/repos/asf/beam.git.
*** WARNING: tag nightly-master was modified! ***
from e24892b (commit)
to 47d3326 (commit)
from e24892b Merge pull request #13880 from [BEAM-11739] Add tag to docker
image when pushing to container registry
add afb6455 Document recommended pipeline run pattern for Python.
add eb733a4 Merge pull request #13873 from robertwb/pipeline-run
add 489ab66 [BEAM-11531] Allow pandas <1.3.0 (#13681)
add 816f297 Add transform translator for GroupIntoBatches in Java.
add 9b92f41 Fix checkStyle error; add a condition to disable autosharding
for JRH
add b4a9a99 Fail jobs that would otherwise fall back to the default
implementation of GIB
add 4a10c01 Fix failed test; update checkArgument logs.
add b56b61a Merge pull request #13805 from [BEAM-10475] Add transform
translator for GroupIntoBatches in Java.
add 7962018 [BEAM-11705] Fixing ignore_insert_id implementation
add 9e18310 Merge pull request #13827 from [BEAM-11705] Fixing
ignore_insert_id implementation
add 9cbef09 Bump Dataflow worker container version.
add 4be9e20 Merge pull request #13890 from Bump Dataflow worker container
version.
add 291ced1 Create Spark Metrics in directory using Spark History Server
format
add f044ec1 java spotless apply
add c578c84 add --spark-history-dir option in spark job server
add cb1b46f add driver distribution logs
add 864603a remove start import check
add f0c7412 spotless apply
add ad359f5 render all metrics & add eventLogEnabled
add bb7c05e add exception when eventLogEnabled is true
add cac34a7 handle eventLogEnabled
add e669894 apply spotless
add 8be02d6 remove star import
add 5392f66 run java spotless
add 1da04b4 fix renderName is static from now on
add 9bec263 spark.executor.id loop and switch to boolean
add 31f86b8 spotless apply
add c98a89e remove uncessary code
add 6529c7a scope.Option.apply
add 03c2457 remove options from spark job server configuration & fix the
spark listener executor added logic
add d3b7838 remove options from spark job server configuration & fix the
spark listener executor added logic
add ba68363 remove options from spark job server configuration & fix the
spark listener executor added logic
add 6acf592 remove options from spark job server configuration & fix the
spark listener executor added logic
add a513fbf remove options from spark job server configuration & fix the
spark listener executor added logic
add f383e17 remove options from spark job server configuration & fix the
spark listener executor added logic
add e105b9f minor cleanup
add 068a33e minor cleanup
add b35c282 minor improvments
add 4b64b5f remove whitespace changes
add 6329b57 remove whitespace changes
add 654ad2b Merge pull request #13743 from tszerszen/sparkmetrics
add 3fdb136 [BEAM-11476] Resolve flaky tests (#13881)
add 8da6a6c [BEAM-11519] Adding PYPI_INSTALL_REQ role and SDK container
support
add 7cb5691 move idCounter variable and add comments
add c2b7fba simplify the branches, change _req to _requirement
add 715bcd0 change urn for pip install requirements file
add cb52848 Merge pull request #13606 from ihji/BEAM-11519
add 842711d [BEAM-11657] Avoid repeated reflection calls for Kafka
deserialization when not using headers
add 2a8daff Merge pull request #13782: [BEAM-11657] Reduce reflection
calls when reading from Kafka
add 0cfa80f Remove an unused reference to staleTimerSet and reword the
commentary. (#13268)
add ef5aeef [BEAM-10925] Enable user-defined Java scalar functions in
ZetaSQL.
add feee800 address review comments
add 9bbd5bd Merge pull request #13891 from ibzib/scalarfn-test
add ebd2a3a Fix the check on maxBufferingDuration
add 5c31997 Merge pull request #13894 from [BEAM-11746] Fix test
flakiness: GroupIntoBatches.testInGlobalWindow
add 0fbb21f [BEAM-11325] Support KafkaIO dynamic read
add 47d3326 Merge pull request #13750 from [BEAM-11325] Kafka Dynamic Read
No new revisions were added by this update.
Summary of changes:
.../pipeline/src/main/proto/beam_runner_api.proto | 6 +-
runners/core-construction-java/build.gradle | 1 +
.../construction/GroupIntoBatchesTranslation.java | 100 +++++
.../core/construction/PTransformTranslation.java | 5 +
.../GroupIntoBatchesTranslationTest.java | 106 ++++++
runners/google-cloud-dataflow-java/build.gradle | 2 +-
.../beam/runners/dataflow/DataflowRunner.java | 31 +-
.../runners/dataflow/GroupIntoBatchesOverride.java | 5 +-
.../dataflow/DataflowPipelineTranslatorTest.java | 146 +++++--
.../beam/runners/dataflow/DataflowRunnerTest.java | 11 +-
.../beam/runners/spark/SparkPipelineOptions.java | 12 +
.../beam/runners/spark/SparkPipelineRunner.java | 58 ++-
.../runners/spark/metrics/SparkBeamMetric.java | 24 +-
.../runners/spark/metrics/SparkBeamMetricTest.java | 2 +-
sdks/go/container/boot.go | 5 +-
sdks/go/pkg/beam/artifact/materialize.go | 115 +++++-
sdks/go/pkg/beam/artifact/materialize_test.go | 9 +-
sdks/go/pkg/beam/core/runtime/xlangx/resolve.go | 12 +-
sdks/java/container/boot.go | 9 +-
.../beam/sdk/transforms/GroupIntoBatches.java | 68 ++--
.../extensions/sql/provider/UdfTestProvider.java | 38 +-
sdks/java/extensions/sql/zetasql/build.gradle | 7 +
.../sdk/extensions/sql/zetasql/SqlAnalyzer.java | 54 ++-
.../extensions/sql/zetasql/ZetaSQLPlannerImpl.java | 61 ++-
.../zetasql/translation/ExpressionConverter.java | 28 +-
.../extensions/sql/zetasql/ZetaSqlJavaUdfTest.java | 248 ++++++++++++
.../sdk/extensions/sql/zetasql/ZetaSqlUdfTest.java | 26 ++
.../org/apache/beam/sdk/io/kafka/ConsumerSpEL.java | 97 +++--
.../beam/sdk/io/kafka/KafkaCommitOffset.java | 2 -
.../java/org/apache/beam/sdk/io/kafka/KafkaIO.java | 164 +++++++-
.../beam/sdk/io/kafka/KafkaUnboundedReader.java | 20 +-
.../beam/sdk/io/kafka/KafkaUnboundedSource.java | 4 +
.../beam/sdk/io/kafka/ReadFromKafkaDoFn.java | 22 +-
.../beam/sdk/io/kafka/TopicPartitionCoder.java | 56 +++
.../sdk/io/kafka/WatchKafkaTopicPartitionDoFn.java | 156 ++++++++
.../org/apache/beam/sdk/io/kafka/KafkaIOTest.java | 53 +--
.../sdk/io/kafka/TopicPartitionCoderTest.java} | 29 +-
.../io/kafka/WatchKafkaTopicPartitionDoFnTest.java | 422 +++++++++++++++++++++
sdks/python/apache_beam/dataframe/schemas.py | 15 +-
sdks/python/apache_beam/dataframe/schemas_test.py | 22 +-
sdks/python/apache_beam/io/gcp/bigquery.py | 5 +-
.../interactive/interactive_environment_test.py | 144 ++++---
sdks/python/container/boot.go | 22 +-
sdks/python/setup.py | 4 +-
.../www/site/content/en/blog/timely-processing.md | 9 +-
.../content/en/get-started/wordcount-example.md | 4 +-
46 files changed, 2049 insertions(+), 390 deletions(-)
create mode 100644
runners/core-construction-java/src/main/java/org/apache/beam/runners/core/construction/GroupIntoBatchesTranslation.java
create mode 100644
runners/core-construction-java/src/test/java/org/apache/beam/runners/core/construction/GroupIntoBatchesTranslationTest.java
create mode 100644
sdks/java/extensions/sql/zetasql/src/test/java/org/apache/beam/sdk/extensions/sql/zetasql/ZetaSqlJavaUdfTest.java
create mode 100644
sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/TopicPartitionCoder.java
create mode 100644
sdks/java/io/kafka/src/main/java/org/apache/beam/sdk/io/kafka/WatchKafkaTopicPartitionDoFn.java
copy
sdks/java/{core/src/test/java/org/apache/beam/sdk/transforms/windowing/NeverTest.java
=>
io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/TopicPartitionCoderTest.java}
(59%)
create mode 100644
sdks/java/io/kafka/src/test/java/org/apache/beam/sdk/io/kafka/WatchKafkaTopicPartitionDoFnTest.java
