This is an automated email from the ASF dual-hosted git repository. github-bot pushed a change to tag nightly-master in repository https://gitbox.apache.org/repos/asf/beam.git.
*** WARNING: tag nightly-master was modified! *** from c7b0450 (commit) to 1fbc55e (commit) from c7b0450 Merge pull request #12181 from KevinGG/fix_graph add 50c63e7 [BEAM-10387] Allow customizing expansion server for SqlTransform add 00a756c [BEAM-10385] Integrate SQL expansion into Flink job server add 88b6fe8 [BEAM-7252] Exclude SQL test from FlinkRunnerOptimized add 1b01fb1 Merge pull request #12133: [BEAM-10385] Integrate SQL expansion into Flink job server add 7776ca6 [BEAM-5414] - Update grpcio-tools to its latest version. (#12179) add 76b764d [BEAM-10093] Run ZetaSQL Nexmark in postcommit add d141981 Merge pull request #12159: [BEAM-10093] Run ZetaSQL Nexmark in postcommit add 7ea4a69 Fix a class cast bug: TupleTagList is not a collection. (#12183) add 916f250 Adds Python 3.8 ValidatesRunner tests for Dataflow. add 6ff5d80 [BEAM-9754] Adds Python 3.8 ValidatesRunner tests for Dataflow. add 28c2c77 [BEAM-10336,BEAM-10337] Add SchemaIO abstraction and implement for PubSub (#12090) add 6d68161 Fix typos in programming guide add e2b41c7 Merge pull request #12190 from ceh-forks/ceh-programming-guide-fix-typos add 21959c9 [BEAM-10283] Add new overloads of withKeyRanges and withRowFilter met… (#12089) add 5b66382 [BEAM-9953[SQL][ZetaSQL] Support Pure SQL user-defined table-valued function. (#12169) add 1afbba9 Add checkerFramework to applyJavaNature and disable everywhere since every single Java module has errors add 07ef03e Fix nullability errors in metrics add 48cf39e Fix nullability errors in DisplayData add 0820ad4 Fix some null errors in ZetaSQL translator add 57c70d7 Fix null error in Beam SQL add 763b457 Merge pull request #12162 from kennknowles/checker add 49bd059 Fix threading issue with default artifact resolver registration. add aa6b50a Merge pull request #12186 Fix threading issue with default artifact resolver registration. add 31aa8c0 [BEAM-10308] Make component ID assignments consistent across PipelineContext instances (#12067) add 41ea60d [BEAM-4605] Remove portable runner GBK hack. add 1fbc55e Merge pull request #12182 [BEAM-4605] Remove portable runner GBK hack. No new revisions were added by this update. Summary of changes: .test-infra/jenkins/NexmarkBuilder.groovy | 11 + .../job_PostCommit_Java_Nexmark_Dataflow.groovy | 46 ++++ .../job_PostCommit_Java_Nexmark_Direct.groovy | 36 +++ .../job_PostCommit_Java_Nexmark_Flink.groovy | 35 +++ ...stCommit_Python_ValidatesRunner_Dataflow.groovy | 2 + CHANGES.md | 14 +- buildSrc/build.gradle | 1 + .../org/apache/beam/gradle/BeamModulePlugin.groovy | 25 ++ examples/java/build.gradle | 2 +- examples/kotlin/build.gradle | 2 +- runners/core-construction-java/build.gradle | 2 +- .../core/construction/DefaultArtifactResolver.java | 11 +- runners/core-java/build.gradle | 2 +- runners/direct-java/build.gradle | 2 +- runners/extensions-java/metrics/build.gradle | 5 +- runners/flink/flink_runner.gradle | 4 +- runners/flink/job-server/flink_job_server.gradle | 13 +- runners/google-cloud-dataflow-java/build.gradle | 2 +- .../examples-streaming/build.gradle | 2 +- .../examples/build.gradle | 2 +- .../google-cloud-dataflow-java/worker/build.gradle | 1 + .../worker/legacy-worker/build.gradle | 1 + runners/java-fn-execution/build.gradle | 2 +- runners/java-job-service/build.gradle | 2 +- runners/jet/build.gradle | 2 +- runners/local-java/build.gradle | 3 +- runners/portability/java/build.gradle | 2 +- runners/samza/build.gradle | 2 +- runners/samza/job-server/build.gradle | 1 + runners/spark/build.gradle | 2 +- runners/spark/job-server/build.gradle | 1 + runners/twister2/build.gradle | 2 +- sdks/java/build-tools/build.gradle | 2 +- sdks/java/core/build.gradle | 1 + .../apache/beam/sdk/metrics/MetricFiltering.java | 30 ++- .../beam/sdk/metrics/MetricsEnvironment.java | 14 +- .../InvalidConfigurationException.java} | 21 +- .../InvalidSchemaException.java} | 21 +- .../sdk/schemas/io/SchemaCapableIOProvider.java | 51 ++++ .../org/apache/beam/sdk/schemas/io/SchemaIO.java | 51 ++++ .../apache/beam/sdk/schemas/io}/package-info.java | 4 +- .../beam/sdk/transforms/display/DisplayData.java | 1 + sdks/java/expansion-service/build.gradle | 2 +- .../sdk/expansion/service/ExpansionService.java | 10 +- sdks/java/extensions/euphoria/build.gradle | 2 +- .../google-cloud-platform-core/build.gradle | 2 +- sdks/java/extensions/jackson/build.gradle | 2 +- sdks/java/extensions/join-library/build.gradle | 2 +- sdks/java/extensions/kryo/build.gradle | 2 +- sdks/java/extensions/ml/build.gradle | 2 +- sdks/java/extensions/protobuf/build.gradle | 2 +- sdks/java/extensions/sketching/build.gradle | 2 +- sdks/java/extensions/sorter/build.gradle | 2 +- sdks/java/extensions/sql/build.gradle | 3 +- sdks/java/extensions/sql/datacatalog/build.gradle | 2 +- .../extensions/sql/expansion-service/build.gradle | 2 +- sdks/java/extensions/sql/hcatalog/build.gradle | 2 +- sdks/java/extensions/sql/jdbc/build.gradle | 2 +- sdks/java/extensions/sql/perf-tests/build.gradle | 2 +- .../sql/impl/SqlConversionException.java | 4 +- ...qlUserDefinedSQLNativeTableValuedFunction.java} | 48 ++-- .../sql/impl/rel/BeamTableFunctionScanRel.java | 22 +- .../sql/impl/utils/TVFStreamingUtils.java | 5 - .../meta/provider/pubsub/PubsubIOJsonTable.java | 77 +----- .../provider/pubsub/PubsubJsonTableProvider.java | 159 +++---------- sdks/java/extensions/sql/zetasql/build.gradle | 2 +- .../sdk/extensions/sql/zetasql/SqlAnalyzer.java | 34 ++- .../extensions/sql/zetasql/ZetaSQLPlannerImpl.java | 12 +- .../sql/zetasql/ZetaSQLQueryPlanner.java | 2 +- .../sql/zetasql/translation/ConversionContext.java | 51 +++- .../zetasql/translation/ExpressionConverter.java | 50 +++- .../zetasql/translation/FilterScanConverter.java | 3 +- .../sql/zetasql/translation/TVFScanConverter.java | 76 +++--- .../sql/zetasql/ZetaSQLDialectSpecTest.java | 52 +++++ sdks/java/extensions/zetasketch/build.gradle | 2 +- sdks/java/fn-execution/build.gradle | 2 +- sdks/java/harness/build.gradle | 2 +- sdks/java/io/amazon-web-services/build.gradle | 2 +- sdks/java/io/amazon-web-services2/build.gradle | 2 +- sdks/java/io/amqp/build.gradle | 2 +- sdks/java/io/bigquery-io-perf-tests/build.gradle | 2 +- sdks/java/io/cassandra/build.gradle | 2 +- sdks/java/io/clickhouse/build.gradle | 2 +- sdks/java/io/common/build.gradle | 2 +- .../elasticsearch-tests-2/build.gradle | 2 +- .../elasticsearch-tests-5/build.gradle | 2 +- .../elasticsearch-tests-6/build.gradle | 2 +- .../elasticsearch-tests-7/build.gradle | 2 +- .../elasticsearch-tests-common/build.gradle | 2 +- sdks/java/io/elasticsearch/build.gradle | 2 +- sdks/java/io/expansion-service/build.gradle | 2 +- sdks/java/io/file-based-io-tests/build.gradle | 2 +- sdks/java/io/google-cloud-platform/build.gradle | 2 +- .../beam/sdk/io/gcp/bigtable/BigtableIO.java | 139 +++++------ .../sdk/io/gcp/bigtable/BigtableReadOptions.java | 91 ++++++++ .../apache/beam/sdk/io/gcp/healthcare/HL7v2IO.java | 5 +- .../sdk/io/gcp}/pubsub/PubsubMessageToRow.java | 7 +- .../gcp/pubsub/PubsubSchemaCapableIOProvider.java | 257 +++++++++++++++++++++ .../sdk/io/gcp}/pubsub/RowToPubsubMessage.java | 33 ++- .../beam/sdk/io/gcp/bigtable/BigtableIOTest.java | 139 +++++++---- .../sdk/io/gcp}/pubsub/PubsubMessageToRowTest.java | 15 +- sdks/java/io/hadoop-common/build.gradle | 2 +- sdks/java/io/hadoop-file-system/build.gradle | 2 +- sdks/java/io/hadoop-format/build.gradle | 2 +- sdks/java/io/hbase/build.gradle | 2 +- sdks/java/io/hcatalog/build.gradle | 2 +- sdks/java/io/jdbc/build.gradle | 2 +- sdks/java/io/jms/build.gradle | 2 +- sdks/java/io/kafka/build.gradle | 2 +- sdks/java/io/kinesis/build.gradle | 2 +- sdks/java/io/kudu/build.gradle | 2 +- sdks/java/io/mongodb/build.gradle | 2 +- sdks/java/io/mqtt/build.gradle | 2 +- sdks/java/io/parquet/build.gradle | 2 +- sdks/java/io/rabbitmq/build.gradle | 2 +- sdks/java/io/redis/build.gradle | 2 +- sdks/java/io/snowflake/build.gradle | 2 +- sdks/java/io/solr/build.gradle | 2 +- sdks/java/io/splunk/build.gradle | 2 +- sdks/java/io/synthetic/build.gradle | 2 +- sdks/java/io/thrift/build.gradle | 2 +- sdks/java/io/tika/build.gradle | 2 +- sdks/java/io/xml/build.gradle | 2 +- sdks/java/javadoc/build.gradle | 2 +- sdks/java/maven-archetypes/examples/build.gradle | 2 +- sdks/java/maven-archetypes/starter/build.gradle | 2 +- sdks/java/testing/expansion-service/build.gradle | 2 +- sdks/java/testing/kafka-service/build.gradle | 2 +- sdks/java/testing/load-tests/build.gradle | 2 +- sdks/java/testing/nexmark/build.gradle | 2 +- sdks/java/testing/test-utils/build.gradle | 2 +- sdks/python/apache_beam/pipeline.py | 35 +++ .../python/apache_beam/runners/pipeline_context.py | 23 +- .../apache_beam/runners/pipeline_context_test.py | 13 ++ .../runners/portability/flink_runner_test.py | 42 +++- .../runners/portability/portable_runner.py | 9 - sdks/python/apache_beam/transforms/external.py | 3 +- sdks/python/apache_beam/transforms/sql.py | 14 +- sdks/python/apache_beam/transforms/sql_test.py | 15 ++ sdks/python/build-requirements.txt | 3 +- vendor/sdks-java-extensions-protobuf/build.gradle | 1 + .../content/en/documentation/programming-guide.md | 20 +- 142 files changed, 1448 insertions(+), 590 deletions(-) copy sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/{Factory.java => io/InvalidConfigurationException.java} (69%) copy sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/{Factory.java => io/InvalidSchemaException.java} (70%) create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/SchemaCapableIOProvider.java create mode 100644 sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io/SchemaIO.java copy {runners/core-java/src/main/java/org/apache/beam/runners/core => sdks/java/core/src/main/java/org/apache/beam/sdk/schemas/io}/package-info.java (91%) copy sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/impl/{planner/BeamJavaTypeFactory.java => ZetaSqlUserDefinedSQLNativeTableValuedFunction.java} (55%) create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigtable/BigtableReadOptions.java rename sdks/java/{extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider => io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp}/pubsub/PubsubMessageToRow.java (97%) create mode 100644 sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubSchemaCapableIOProvider.java rename sdks/java/{extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider => io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp}/pubsub/RowToPubsubMessage.java (69%) rename sdks/java/{extensions/sql/src/test/java/org/apache/beam/sdk/extensions/sql/meta/provider => io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp}/pubsub/PubsubMessageToRowTest.java (95%)