This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from 2cc2b8e4488 Bump braces from 3.0.2 to 3.0.3 in /sdks/typescript
(#31664)
add 6ec1fb23ece move heartbeat processor to where it is being used (#31298)
add 3588d195335 Add spark mapstate (#31669)
add fb1e9cf72d4 Allow override beam version for PythonExternalTransform
via pipeline option (#31691)
add d80a58c09f3 Eliminate the use of testRuntimeMigration for
sdks:java:io:common (#31693)
add 20aa916931f Properly close Storage API batch connections (#31710)
add f8b63ff3d55 Create CsvIOParseResult (#31706)
add 2f81e16f3ea [#31403] Python wrapper to download, use, or build and run
prism. (#31583)
add 9bdcb672d08 Create CsvIOParseConfiguration class (#31714)
add c3756c04e39 Remove excessive logging in test. (#31715)
add 5a090958992 Move CsvIOParseResult (#31722)
No new revisions were added by this update.
Summary of changes:
.../beam/runners/core/StateInternalsTest.java | 6 +
.../dataflow/worker/StreamingDataflowWorker.java | 172 +++++++++-------
.../windmill/client/grpc/GrpcWindmillServer.java | 53 ++---
.../client/grpc/GrpcWindmillStreamFactory.java | 101 ++++++----
.../client/grpc/StreamingEngineClient.java | 22 +--
.../windmill/client/grpc/WindmillStreamSender.java | 19 +-
.../client/grpc/StreamingEngineClientTest.java | 5 +-
.../client/grpc/WindmillStreamSenderTest.java | 14 +-
.../budget/EvenGetWorkBudgetDistributorTest.java | 3 +-
.../spark/stateful/SparkStateInternals.java | 148 +++++++++++++-
.../spark/stateful/SparkStateInternalsTest.java | 10 +-
.../extensions/python/PythonExternalTransform.java | 13 +-
.../python/PythonExternalTransformOptions.java | 10 +
.../beam/sdk/extensions/python/PythonService.java | 19 +-
sdks/java/io/amazon-web-services/build.gradle | 2 +-
sdks/java/io/amazon-web-services2/build.gradle | 2 +-
sdks/java/io/bigquery-io-perf-tests/build.gradle | 2 +-
sdks/java/io/cassandra/build.gradle | 2 +-
sdks/java/io/cdap/build.gradle | 3 +-
sdks/java/io/common/build.gradle | 17 +-
.../beam/sdk/io/common/DatabaseTestHelper.java | 70 +++----
.../org/apache/beam/sdk/io/common/HashingFn.java | 4 +-
.../org/apache/beam/sdk/io/common/IOITHelper.java | 0
.../beam/sdk/io/common/IOTestPipelineOptions.java | 0
.../beam/sdk/io/common/NetworkTestHelper.java | 0
.../io/common/PostgresIOTestPipelineOptions.java | 0
.../beam/sdk/io/common/SchemaAwareJavaBeans.java | 12 +-
.../org/apache/beam/sdk/io/common/TestRow.java | 17 +-
.../apache/beam/sdk/io/common}/package-info.java | 5 +-
sdks/java/io/csv/build.gradle | 2 +-
.../beam/sdk/io/csv/CsvIOParseConfiguration.java | 69 +++++++
.../apache/beam/sdk/io/csv/CsvIOParseResult.java | 61 +++---
.../sdk/io/csv/CsvIOParseConfigurationTest.java | 12 +-
sdks/java/io/debezium/build.gradle | 4 +-
sdks/java/io/elasticsearch/build.gradle | 2 +-
sdks/java/io/file-based-io-tests/build.gradle | 2 +-
sdks/java/io/file-schema-transform/build.gradle | 2 +-
sdks/java/io/google-ads/build.gradle | 2 +-
sdks/java/io/google-cloud-platform/build.gradle | 2 +-
.../beam/sdk/io/gcp/bigquery/AppendClientInfo.java | 19 +-
.../bigquery/StorageApiWriteUnshardedRecords.java | 5 +
.../gcp/bigquery/StorageApiSinkSchemaUpdateIT.java | 2 +-
sdks/java/io/hadoop-format/build.gradle | 2 +-
sdks/java/io/hbase/build.gradle | 2 +-
sdks/java/io/hcatalog/build.gradle | 11 +-
sdks/java/io/influxdb/build.gradle | 2 +-
sdks/java/io/jdbc/build.gradle | 2 +-
sdks/java/io/jms/build.gradle | 2 +-
sdks/java/io/json/build.gradle | 2 +-
sdks/java/io/kafka/build.gradle | 2 +-
sdks/java/io/kinesis/build.gradle | 2 +-
sdks/java/io/kudu/build.gradle | 2 +-
sdks/java/io/mongodb/build.gradle | 2 +-
sdks/java/io/mqtt/build.gradle | 2 +-
sdks/java/io/neo4j/build.gradle | 2 +-
sdks/java/io/rabbitmq/build.gradle | 2 +-
sdks/java/io/redis/build.gradle | 2 +-
sdks/java/io/rrio/build.gradle | 2 +-
sdks/java/io/singlestore/build.gradle | 2 +-
sdks/java/io/snowflake/build.gradle | 2 +-
sdks/java/io/solace/build.gradle | 2 +-
sdks/java/io/sparkreceiver/2/build.gradle | 2 +-
sdks/python/apache_beam/io/parquetio_test.py | 3 +-
.../python/apache_beam/options/pipeline_options.py | 19 ++
.../runners/portability/prism_runner.py | 216 +++++++++++++++++++++
.../{flink_runner_test.py => prism_runner_test.py} | 136 ++-----------
66 files changed, 868 insertions(+), 470 deletions(-)
rename sdks/java/io/common/src/{test =>
main}/java/org/apache/beam/sdk/io/common/DatabaseTestHelper.java (79%)
rename sdks/java/io/common/src/{test =>
main}/java/org/apache/beam/sdk/io/common/HashingFn.java (98%)
rename sdks/java/io/common/src/{test =>
main}/java/org/apache/beam/sdk/io/common/IOITHelper.java (100%)
rename sdks/java/io/common/src/{test =>
main}/java/org/apache/beam/sdk/io/common/IOTestPipelineOptions.java (100%)
rename sdks/java/io/common/src/{test =>
main}/java/org/apache/beam/sdk/io/common/NetworkTestHelper.java (100%)
rename sdks/java/io/common/src/{test =>
main}/java/org/apache/beam/sdk/io/common/PostgresIOTestPipelineOptions.java
(100%)
rename sdks/java/io/common/src/{test =>
main}/java/org/apache/beam/sdk/io/common/SchemaAwareJavaBeans.java (98%)
rename sdks/java/io/common/src/{test =>
main}/java/org/apache/beam/sdk/io/common/TestRow.java (83%)
copy {examples/java/cdap/src/main/java/org/apache/beam/examples/complete/cdap
=>
sdks/java/io/common/src/main/java/org/apache/beam/sdk/io/common}/package-info.java
(90%)
create mode 100644
sdks/java/io/csv/src/main/java/org/apache/beam/sdk/io/csv/CsvIOParseConfiguration.java
copy
.test-infra/pipelines/src/main/java/org/apache/beam/testinfra/pipelines/conversions/RowConversionResult.java
=>
sdks/java/io/csv/src/main/java/org/apache/beam/sdk/io/csv/CsvIOParseResult.java
(54%)
copy learning/katas/java/IO/Built-in IOs/Built-in
IOs/test/org/apache/beam/learning/katas/io/builtinios/TaskTest.java =>
sdks/java/io/csv/src/test/java/org/apache/beam/sdk/io/csv/CsvIOParseConfigurationTest.java
(72%)
create mode 100644 sdks/python/apache_beam/runners/portability/prism_runner.py
copy sdks/python/apache_beam/runners/portability/{flink_runner_test.py =>
prism_runner_test.py} (67%)
