This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to tag nightly-master
in repository https://gitbox.apache.org/repos/asf/beam.git.
*** WARNING: tag nightly-master was modified! ***
from 5ddb69f (commit)
to bee495f (commit)
from 5ddb69f Merge pull request #14396 from [BEAM-12079] Fix
_StreamToBigQuery/CommitInsertIds/GroupByKey failure
add df055c1 Support multilayer ZetaSQL UNNEST (#14342)
add 2c619c8 [BEAM-7372] delete codes for compatibility of py2 from
apache_beam/io except for io/gcp (#14352)
add 9e0b378 [BEAM-11213] Display Beam Metrics in Spark History Server for
Classic Runner too
add 1e60f38 Merge pull request #14409: [BEAM-11213] Display Beam Metrics
in Spark History Server for Classic Runner too
add 9863e82 [BEAM-11747] Narrow list of unsupported types in
BeamJavaUdfCalcRule. (#14241)
add 6151674 [BEAM-12079] Enforce callable destination arg type for
WriteToBigQuery schema argument (#14415)
add 19a4ac0 Re-enable CrossLanguageKafkaIOTest
add e5b3333 Merge pull request #14414 [BEAM-10663] Re-enable
CrossLanguageKafkaIOTest
add 32af8d2 [BEAM-12088] Make file staging uniform among Spark Runners
add 5f13eff Merge pull request #14417: [BEAM-12088] Make file staging
uniform among Spark Runners
add f59aef2 [BEAM-12060] Fix failing Go Postcommits, jenkins support for
Gradle tasks.
add bee495f Merge pull request #14407: [BEAM-12060] Fix failing Go
Postcommits, jenkins support for Gradle tasks.
No new revisions were added by this update.
Summary of changes:
.test-infra/jenkins/job_PostCommit_Go.groovy | 1 +
.../job_PostCommit_Go_ValidatesRunner_Flink.groovy | 1 +
.../job_PostCommit_Go_ValidatesRunner_Spark.groovy | 1 +
.../jenkins/job_PreCommit_Go_Portable.groovy | 1 +
build.gradle.kts | 2 +-
.../SparkStructuredStreamingRunner.java | 7 +-
.../translation/PipelineTranslator.java | 21 ----
.../runners/spark/SparkCommonPipelineOptions.java | 36 +++++++
.../beam/runners/spark/SparkPipelineOptions.java | 49 ---------
.../beam/runners/spark/SparkPipelineRunner.java | 41 ++-----
.../org/apache/beam/runners/spark/SparkRunner.java | 65 +++++++----
.../beam/runners/spark/util/SparkCommon.java | 79 ++++++++++++++
.../beam/runners/spark/util/SparkCompat.java | 6 +-
.../spark/metrics/SparkMetricsPusherTest.java | 2 +-
sdks/go/test/build.gradle | 40 +++----
.../extensions/sql/impl/rule/BeamUnnestRule.java | 2 +-
.../sql/zetasql/BeamJavaUdfCalcRule.java | 119 ++++++++++++++++++++-
.../sql/zetasql/BeamZetaSqlCalcRule.java | 31 +++++-
.../sql/zetasql/ZetaSQLQueryPlanner.java | 118 --------------------
.../sql/zetasql/unnest/BeamZetaSqlUnnestRule.java | 6 +-
.../beam/sdk/extensions/sql/zetasql/TestInput.java | 13 +++
.../sql/zetasql/ZetaSqlDialectSpecTest.java | 38 +++++++
.../extensions/sql/zetasql/ZetaSqlTestBase.java | 2 +
.../java/org/apache/beam/sdk/io/kafka/KafkaIO.java | 4 +-
sdks/python/apache_beam/io/__init__.py | 2 -
sdks/python/apache_beam/io/avroio.py | 3 -
sdks/python/apache_beam/io/avroio_test.py | 6 --
sdks/python/apache_beam/io/aws/__init__.py | 2 -
sdks/python/apache_beam/io/aws/clients/__init__.py | 1 -
.../apache_beam/io/aws/clients/s3/__init__.py | 1 -
.../apache_beam/io/aws/clients/s3/boto3_client.py | 2 -
.../apache_beam/io/aws/clients/s3/fake_client.py | 2 -
.../apache_beam/io/aws/clients/s3/messages.py | 2 -
sdks/python/apache_beam/io/aws/s3filesystem.py | 8 +-
.../python/apache_beam/io/aws/s3filesystem_test.py | 2 -
sdks/python/apache_beam/io/aws/s3io.py | 3 -
sdks/python/apache_beam/io/aws/s3io_test.py | 2 -
sdks/python/apache_beam/io/azure/__init__.py | 2 -
.../apache_beam/io/azure/blobstoragefilesystem.py | 6 +-
.../io/azure/blobstoragefilesystem_test.py | 4 -
sdks/python/apache_beam/io/azure/blobstorageio.py | 3 -
.../apache_beam/io/azure/blobstorageio_test.py | 2 -
sdks/python/apache_beam/io/concat_source.py | 4 -
sdks/python/apache_beam/io/concat_source_test.py | 4 -
sdks/python/apache_beam/io/external/__init__.py | 2 -
.../python/apache_beam/io/external/gcp/__init__.py | 2 -
sdks/python/apache_beam/io/external/gcp/pubsub.py | 18 ++--
.../apache_beam/io/external/generate_sequence.py | 2 -
.../io/external/generate_sequence_test.py | 3 -
sdks/python/apache_beam/io/external/kafka.py | 2 -
sdks/python/apache_beam/io/external/snowflake.py | 2 -
.../io/external/xlang_jdbcio_it_test.py | 6 +-
.../io/external/xlang_kafkaio_it_test.py | 3 -
.../io/external/xlang_kinesisio_it_test.py | 2 -
.../io/external/xlang_parquetio_test.py | 3 -
.../io/external/xlang_snowflakeio_it_test.py | 2 -
sdks/python/apache_beam/io/filebasedsink.py | 17 +--
sdks/python/apache_beam/io/filebasedsink_test.py | 5 -
sdks/python/apache_beam/io/filebasedsource.py | 13 +--
sdks/python/apache_beam/io/filebasedsource_test.py | 7 --
sdks/python/apache_beam/io/fileio.py | 6 +-
sdks/python/apache_beam/io/fileio_test.py | 2 -
sdks/python/apache_beam/io/filesystem.py | 15 +--
sdks/python/apache_beam/io/filesystem_test.py | 7 +-
sdks/python/apache_beam/io/filesystemio.py | 9 +-
sdks/python/apache_beam/io/filesystemio_test.py | 3 -
sdks/python/apache_beam/io/filesystems.py | 7 +-
sdks/python/apache_beam/io/filesystems_test.py | 4 -
sdks/python/apache_beam/io/flink/__init__.py | 2 -
.../io/flink/flink_streaming_impulse_source.py | 2 -
.../flink/flink_streaming_impulse_source_test.py | 2 -
sdks/python/apache_beam/io/gcp/bigquery.py | 6 +-
sdks/python/apache_beam/io/hadoopfilesystem.py | 3 -
.../python/apache_beam/io/hadoopfilesystem_test.py | 8 +-
sdks/python/apache_beam/io/iobase.py | 5 -
sdks/python/apache_beam/io/iobase_test.py | 2 -
sdks/python/apache_beam/io/jdbc.py | 22 ++--
sdks/python/apache_beam/io/kafka.py | 18 ++--
sdks/python/apache_beam/io/kinesis.py | 32 +++---
sdks/python/apache_beam/io/localfilesystem.py | 3 -
sdks/python/apache_beam/io/localfilesystem_test.py | 4 -
sdks/python/apache_beam/io/mongodbio.py | 3 -
sdks/python/apache_beam/io/mongodbio_it_test.py | 2 -
sdks/python/apache_beam/io/mongodbio_test.py | 3 -
sdks/python/apache_beam/io/parquetio.py | 2 -
sdks/python/apache_beam/io/parquetio_it_test.py | 3 -
sdks/python/apache_beam/io/parquetio_test.py | 2 -
sdks/python/apache_beam/io/range_trackers.py | 12 +--
sdks/python/apache_beam/io/range_trackers_test.py | 7 +-
sdks/python/apache_beam/io/restriction_trackers.py | 4 -
.../apache_beam/io/restriction_trackers_test.py | 2 -
sdks/python/apache_beam/io/snowflake.py | 70 ++++++------
sdks/python/apache_beam/io/source_test_utils.py | 6 --
.../apache_beam/io/source_test_utils_test.py | 3 -
sdks/python/apache_beam/io/sources_test.py | 2 -
sdks/python/apache_beam/io/textio.py | 8 +-
sdks/python/apache_beam/io/textio_test.py | 4 -
sdks/python/apache_beam/io/tfrecordio.py | 3 -
sdks/python/apache_beam/io/tfrecordio_test.py | 5 -
sdks/python/apache_beam/io/utils.py | 4 -
sdks/python/apache_beam/io/utils_test.py | 2 -
sdks/python/apache_beam/io/watermark_estimators.py | 2 -
.../apache_beam/io/watermark_estimators_test.py | 2 -
sdks/python/test-suites/portable/common.gradle | 1 +
104 files changed, 498 insertions(+), 651 deletions(-)
create mode 100644
runners/spark/src/main/java/org/apache/beam/runners/spark/util/SparkCommon.java
