This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git.
from bee56a6 [BEAM-13288] improve logging for no rows present error
(#16096)
add 45955da Merge pull request #16117 from
[BEAM-13368][Playground][Bugfix] Fix CI and failed unit tests on master
add 0830a02 Run python SpannerIO IT with python 3.7 only to avoid
overload spanner instance (#16112)
add 862ece1 Merge pull request #15378 from [RFC] Define and document
per-key ordering semantics for runners
add 3422774 Bump python containers to beam-master-20211202 (#16129)
add 29213ce [BEAM-13354, BEAM-13015, BEAM-12802, BEAM-12588] Support
prefetch for multimap and set state making loading keys and values truly lazy
(#16092)
add 09bbb48 [BEAM-11936] Fix errorprone UnusedVariable in IO (#16036)
add 3c948fc Fix failing RecommendationAICatalogItemIT
add 89e2b23 Merge pull request #16132 from y1chi/fix_post
add 91950d6 Clarify instructions on how to get contributor list.
add dfb5726 Merge pull request #16133 from tvalentyn/tags_command
add 6b9e374 Revert "Revert "Allow wildcards for java class lookup
transform providers.""
add 8faf018 Move stand-alone expansion service jar into its own project.
add bfff840 Merge pull request #16044 Restore "Allow wildcards for java
class lookup transform providers."
No new revisions were added by this update.
Summary of changes:
...Java_PortableValidatesRunner_Flink_Batch.groovy | 2 +-
build.gradle.kts | 3 +
playground/backend/build.gradle.kts | 2 +-
playground/backend/cmd/server/controller_test.go | 21 +-
runners/flink/job-server/flink_job_server.gradle | 2 +
runners/spark/job-server/spark_job_server.gradle | 1 +
...edMetrics.java => UsesPerKeyOrderInBundle.java} | 10 +-
...leParDo.java => UsesPerKeyOrderedDelivery.java} | 9 +-
.../org/apache/beam/sdk/transforms/ParDoTest.java | 3 +-
.../beam/sdk/transforms/PerKeyOrderingTest.java | 328 +++++++++++++++
.../app}/build.gradle | 21 +-
sdks/java/expansion-service/build.gradle | 5 +-
.../expansion/service/ExpansionServiceOptions.java | 10 +-
.../service/JavaClassLookupTransformProvider.java | 83 +++-
.../JavaClassLookupTransformProviderTest.java | 42 ++
.../ml/RecommendationAICatalogItemIT.java | 4 +-
.../beam/fn/harness/state/FnApiStateAccessor.java | 19 +-
.../beam/fn/harness/state/MultimapUserState.java | 165 +++++---
.../fn/harness/state/MultimapUserStateTest.java | 449 +++++++++++++++------
sdks/java/io/amazon-web-services/build.gradle | 1 -
sdks/java/io/amazon-web-services2/build.gradle | 1 -
.../apache/beam/sdk/io/aws2/kinesis/KinesisIO.java | 6 -
.../beam/sdk/io/aws2/s3/S3FileSystemTest.java | 2 +-
sdks/java/io/amqp/build.gradle | 2 +-
.../java/org/apache/beam/sdk/io/amqp/AmqpIO.java | 8 +-
.../beam/sdk/io/amqp/AmqpMessageCoderTest.java | 2 +-
sdks/java/io/azure/build.gradle | 1 -
.../beam/sdk/io/azure/options/AzureModule.java | 6 -
sdks/java/io/bigquery-io-perf-tests/build.gradle | 2 +-
.../beam/sdk/bigqueryioperftests/BigQueryIOIT.java | 4 -
sdks/java/io/cassandra/build.gradle | 1 -
.../apache/beam/sdk/io/cassandra/CassandraIO.java | 1 -
.../beam/sdk/io/cassandra/CassandraIOTest.java | 1 -
sdks/java/io/clickhouse/build.gradle | 1 -
sdks/java/io/common/build.gradle | 2 +-
sdks/java/io/contextualtextio/build.gradle | 1 -
.../contextualtextio/ContextualTextIOSource.java | 4 -
sdks/java/io/debezium/build.gradle | 1 -
.../io/debezium/expansion-service/build.gradle | 1 -
.../io/debezium/DebeziumTransformRegistrar.java | 3 -
.../apache/beam/io/debezium/SourceRecordJson.java | 4 +-
.../apache/beam/io/debezium/DebeziumIOTest.java | 4 +-
.../io/debezium/KafkaSourceConsumerFnTest.java | 23 +-
.../elasticsearch-tests-5/build.gradle | 1 -
.../elasticsearch-tests-6/build.gradle | 1 -
.../elasticsearch-tests-7/build.gradle | 1 -
.../elasticsearch-tests-common/build.gradle | 1 -
sdks/java/io/elasticsearch/build.gradle | 2 +-
sdks/java/io/expansion-service/build.gradle | 1 -
sdks/java/io/file-based-io-tests/build.gradle | 2 +-
.../java/org/apache/beam/sdk/io/avro/AvroIOIT.java | 4 -
.../apache/beam/sdk/io/parquet/ParquetIOIT.java | 4 -
.../java/org/apache/beam/sdk/io/text/TextIOIT.java | 7 -
.../apache/beam/sdk/io/tfrecord/TFRecordIOIT.java | 4 -
.../java/org/apache/beam/sdk/io/xml/XmlIOIT.java | 4 -
sdks/java/io/google-cloud-platform/build.gradle | 1 -
.../expansion-service/build.gradle | 1 -
.../sdk/io/gcp/bigquery/BigQueryQuerySource.java | 4 -
.../sdk/io/gcp/bigquery/BigQueryServicesImpl.java | 3 -
.../gcp/bigquery/BigQueryStorageArrowReader.java | 5 -
.../sdk/io/gcp/bigquery/BigQueryTableSource.java | 3 -
.../beam/sdk/io/gcp/bigquery/CreateTables.java | 6 -
.../beam/sdk/io/gcp/bigquery/StorageApiLoads.java | 3 -
.../bigquery/StorageApiWritesShardedRecords.java | 5 +-
.../beam/sdk/io/gcp/bigtable/BigtableIO.java | 1 -
.../apache/beam/sdk/io/gcp/healthcare/FhirIO.java | 1 -
.../io/gcp/healthcare/HttpHealthcareApiClient.java | 2 +-
.../sdk/io/gcp/pubsub/PubsubUnboundedSink.java | 4 -
.../beam/sdk/io/gcp/pubsub/TestPubsubSignal.java | 9 +-
.../sdk/io/gcp/spanner/MutationKeyEncoder.java | 4 +-
.../sdk/io/gcp/testing/FakeDatasetService.java | 1 -
.../io/gcp/bigquery/BigQueryIOStorageReadTest.java | 3 -
.../io/gcp/bigquery/BigQueryNestedRecordsIT.java | 2 -
.../io/gcp/bigquery/BigQueryServicesImplTest.java | 4 -
.../sdk/io/gcp/bigquery/BigQueryToTableIT.java | 5 +-
.../beam/sdk/io/gcp/bigtable/BigtableIOTest.java | 2 -
.../beam/sdk/io/gcp/datastore/DatastoreV1Test.java | 3 -
.../gcp/firestore/BaseFirestoreV1WriteFnTest.java | 5 +-
.../beam/sdk/io/gcp/healthcare/FhirIOLROIT.java | 4 +-
.../beam/sdk/io/gcp/healthcare/FhirIOTestUtil.java | 3 -
.../beam/sdk/io/gcp/pubsub/PubsubReadIT.java | 3 -
.../sdk/io/gcp/spanner/ReadSpannerSchemaTest.java | 2 -
.../beam/sdk/io/gcp/spanner/SpannerIOReadTest.java | 30 +-
.../sdk/io/gcp/spanner/SpannerIOWriteTest.java | 1 -
sdks/java/io/hadoop-common/build.gradle | 2 +-
sdks/java/io/hadoop-file-system/build.gradle | 1 -
sdks/java/io/hadoop-format/build.gradle | 1 -
.../sdk/io/hadoop/format/HadoopFormatIOIT.java | 4 -
.../io/hadoop/format/HadoopFormatIOReadTest.java | 3 -
sdks/java/io/hbase/build.gradle | 2 +-
.../java/org/apache/beam/sdk/io/hbase/HBaseIO.java | 2 -
sdks/java/io/hcatalog/build.gradle | 2 +-
sdks/java/io/influxdb/build.gradle | 3 +-
.../apache/beam/sdk/io/influxdb/InfluxDbIO.java | 3 -
sdks/java/io/jdbc/build.gradle | 1 -
.../java/org/apache/beam/sdk/io/jdbc/JdbcIOIT.java | 4 -
.../org/apache/beam/sdk/io/jdbc/JdbcIOTest.java | 26 +-
sdks/java/io/jms/build.gradle | 1 -
sdks/java/io/kafka/build.gradle | 1 -
.../beam/sdk/io/kafka/KafkaExactlyOnceSink.java | 4 +-
.../sdk/io/kafka/WatchKafkaTopicPartitionDoFn.java | 5 +-
.../beam/sdk/io/kafka/KafkaIOExternalTest.java | 3 +-
sdks/java/io/kinesis/build.gradle | 2 +-
.../java/io/kinesis/expansion-service/build.gradle | 1 -
.../beam/sdk/io/kinesis/KinesisIOReadTest.java | 2 -
.../beam/sdk/io/kinesis/serde/AwsModuleTest.java | 2 -
sdks/java/io/kudu/build.gradle | 1 -
.../java/org/apache/beam/sdk/io/kudu/KuduIO.java | 3 -
.../java/org/apache/beam/sdk/io/kudu/KuduIOIT.java | 3 -
.../org/apache/beam/sdk/io/kudu/KuduIOTest.java | 1 -
sdks/java/io/mongodb/build.gradle | 2 +-
.../org/apache/beam/sdk/io/mongodb/MongoDbIO.java | 1 -
.../apache/beam/sdk/io/mongodb/MongoDBIOIT.java | 4 -
sdks/java/io/mqtt/build.gradle | 2 +-
sdks/java/io/parquet/build.gradle | 1 -
.../apache/beam/sdk/io/parquet/ParquetIOTest.java | 1 -
sdks/java/io/rabbitmq/build.gradle | 2 +-
.../beam/sdk/io/rabbitmq/RabbitMqIOTest.java | 3 -
sdks/java/io/redis/build.gradle | 2 +-
sdks/java/io/snowflake/build.gradle | 1 -
.../io/snowflake/expansion-service/build.gradle | 1 -
.../apache/beam/sdk/io/snowflake/SnowflakeIO.java | 3 +-
.../services/SnowflakeBatchServiceImpl.java | 3 -
.../services/SnowflakeStreamingServiceImpl.java | 4 +-
.../test/unit/DataSourceConfigurationTest.java | 4 -
.../test/unit/write/SchemaDispositionTest.java | 1 -
.../test/unit/write/StreamingWriteTest.java | 4 +-
sdks/java/io/solr/build.gradle | 1 -
sdks/java/io/splunk/build.gradle | 2 +-
.../beam/sdk/io/splunk/SplunkEventWriter.java | 3 +-
sdks/java/io/synthetic/build.gradle | 1 -
sdks/java/io/thrift/build.gradle | 1 -
.../apache/beam/sdk/io/thrift/ThriftIOTest.java | 6 +-
sdks/java/io/tika/build.gradle | 2 +-
sdks/java/io/xml/build.gradle | 2 +-
.../io/gcp/experimental/spannerio_read_it_test.py | 20 +-
.../io/gcp/experimental/spannerio_write_it_test.py | 10 +-
.../apache_beam/runners/dataflow/internal/names.py | 4 +-
sdks/python/apache_beam/transforms/external.py | 56 ++-
.../python/apache_beam/transforms/external_test.py | 24 ++
sdks/python/apache_beam/utils/processes_test.py | 8 +
sdks/python/apache_beam/utils/subprocess_server.py | 47 ++-
.../apache_beam/utils/subprocess_server_test.py | 51 +++
sdks/python/test-suites/dataflow/common.gradle | 24 ++
sdks/python/test-suites/direct/common.gradle | 20 +
sdks/python/test-suites/portable/common.gradle | 38 +-
settings.gradle.kts | 1 +
.../site/content/en/contribute/release-guide.md | 2 +-
.../site/content/en/documentation/runtime/model.md | 50 ++-
website/www/site/data/capability_matrix.yaml | 39 ++
.../documentation/capability-matrix-row.html | 2 +
.../site/layouts/shortcodes/capability-matrix.html | 2 +
.../documentation/capability-matrix-single.html | 6 +-
153 files changed, 1391 insertions(+), 566 deletions(-)
copy
sdks/java/core/src/main/java/org/apache/beam/sdk/testing/{UsesCommittedMetrics.java
=> UsesPerKeyOrderInBundle.java} (71%)
copy
sdks/java/core/src/main/java/org/apache/beam/sdk/testing/{UsesBoundedSplittableParDo.java
=> UsesPerKeyOrderedDelivery.java} (75%)
create mode 100644
sdks/java/core/src/test/java/org/apache/beam/sdk/transforms/PerKeyOrderingTest.java
copy sdks/java/{io/snowflake/expansion-service =>
expansion-service/app}/build.gradle (72%)
