This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from f50c8d503af Deprecate Cloud Debugger and its pipeline options (#25998)
add fc7a240eb4e BigQuery Storage Write API wrapper for Python SDK (#25685)
add 91bfbac7a90 Update Java and Python base image versions, resolve CI
build issue (#25992)
add a4f55afa052 Bump tensorflow (#25974)
add 08ae6c7acff [Playground] Implement Java & Go multifile examples
execution (#24874)
add 567bdaa4563 fix: Porting more BigtableOptions (#25955)
add 469101dd378 [Playground] "always_run" flag in examples' tag (#25928)
add c0f0f2d3c15 Overload HbaseIO with KV<RowKey, RowMutations> (#25864)
add 14a77825928 Add explicit language_level=3 arg to Cythonize (#26015)
add 6484a47c3d5 Fix TestPipeline.runWithAdditionalOptionArgs ignore
additionalArgs (#25937)
add 3efd3c38b99 Update pip-licenses in Dockerfile to support Python 3.11
(#26019)
add 5bdc4527a47 [#22737] Re-write Go SDK data plane to support timers.
(#25982)
add 4911a3e150b [Playground] Guard router methods from being executed on
runners (#25931)
add 7bfbdf2d0f1 Remove dead session code & noisy logging (#22241) (#26029)
add 04288fed142 use example id for cloud path (#26027)
add 5569085c0b0 [#26030] Add PTransformID to logged messages. (#26031)
add 652e30a383b Update container tags (#26022)
add c196cd15e99 Update CHANGES.md for HBaseIO.WriteRowMutations (#26032)
add 87783393936 Fix small doc typo (#26033)
add 9c614557c51 [Go SDK]: Add fileio MatchFiles, MatchAll and ReadMatches
transforms (#25809)
add 9f582e564fe Add key param support for helper Combiners (#25895)
add 52531b73cec Add #24599 to CHANGES.md (#26020)
add 915da25323d Update pillow requirement from <8,>=7.1.1 to >=7.1.1,<10
in /sdks/python (#25788)
add d9321d8cdb3 Recommend `yield from` for emitting elements from
iterables. (#25893)
No new revisions were added by this update.
Summary of changes:
.github/workflows/playground_backend_precommit.yml | 4 +-
.../workflows/playground_examples_ci_reusable.yml | 6 +-
...Commit_Python_CrossLanguage_Gcp_Dataflow.groovy | 55 ++
...stCommit_Python_CrossLanguage_Gcp_Direct.groovy | 55 ++
CHANGES.md | 6 +
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 144 +++++
.../backend/playground_api/api/v1/api.pb.go | 588 +++++++++----------
playground/api/v1/api.proto | 1 +
playground/backend/cmd/server/controller.go | 56 ++
playground/backend/cmd/server/controller_test.go | 80 ++-
playground/backend/containers/java/Dockerfile | 8 +-
playground/backend/containers/python/Dockerfile | 4 +-
playground/backend/containers/python/entrypoint.sh | 2 +-
playground/backend/containers/scio/Dockerfile | 2 +-
playground/backend/internal/api/v1/api.pb.go | 588 +++++++++----------
.../internal/code_processing/code_processing.go | 10 +-
.../code_processing/code_processing_test.go | 2 +-
.../backend/internal/db/datastore/datastore_db.go | 2 +-
.../backend/internal/db/dto/precompiled_object.go | 1 +
playground/backend/internal/db/entity/example.go | 1 +
playground/backend/internal/db/mapper/mapper.go | 2 +-
.../db/mapper/precompiled_object_mapper.go | 23 +-
.../db/mapper/precompiled_object_mapper_test.go | 2 +-
.../internal/environment/environment_service.go | 2 +-
.../environment/environment_service_test.go | 18 +-
playground/backend/internal/executors/executor.go | 10 +-
.../backend/internal/executors/executor_builder.go | 30 +-
.../internal/executors/executor_builder_test.go | 2 +-
.../backend/internal/executors/executor_test.go | 8 +-
playground/backend/internal/fs_tool/fs.go | 6 +-
playground/backend/internal/fs_tool/fs_test.go | 4 +-
playground/backend/internal/fs_tool/go_fs.go | 4 +-
playground/backend/internal/fs_tool/go_fs_test.go | 4 +-
playground/backend/internal/fs_tool/java_fs.go | 130 ++++-
.../backend/internal/fs_tool/java_fs_test.go | 215 ++++++-
.../fs_tool/java_testdata/HasIncorrectMain.java | 28 +
.../fs_tool/java_testdata/HasMainTest1.java | 28 +
.../fs_tool/java_testdata/HasMainTest2.java | 29 +
.../internal/fs_tool/java_testdata/HasNoMain.java | 28 +
playground/backend/internal/fs_tool/scio_fs.go | 2 +-
.../internal/setup_tools/builder/setup_builder.go | 51 +-
.../setup_tools/builder/setup_builder_test.go | 149 ++++-
.../setup_tools/life_cycle/life_cycle_setuper.go | 4 +-
playground/backend/internal/utils/file_utils.go | 4 +
.../lib/src/api/v1/api.pb.dart | 14 +
.../lib/src/api/v1/api.pbjson.dart | 3 +-
.../lib/src/cache/example_cache.dart | 48 +-
.../example_loaders/content_example_loader.dart | 1 +
.../controllers/snippet_editing_controller.dart | 1 +
.../lib/src/models/example_base.dart | 2 +
.../content_example_loading_descriptor.dart | 11 +-
.../example_client/grpc_example_client.dart | 1 +
.../content_example_loading_descriptor_test.dart | 2 +
playground/infrastructure/api/v1/api_pb2.py | 120 ++--
playground/infrastructure/api/v1/api_pb2.pyi | 6 +-
playground/infrastructure/datastore_client.py | 10 +-
playground/infrastructure/models.py | 1 +
sdks/go/pkg/beam/core/metrics/metrics.go | 32 +-
sdks/go/pkg/beam/core/runtime/exec/data.go | 16 +-
sdks/go/pkg/beam/core/runtime/exec/datasource.go | 182 ++++--
.../pkg/beam/core/runtime/exec/datasource_test.go | 115 ++--
.../go/pkg/beam/core/runtime/exec/dynsplit_test.go | 14 +-
sdks/go/pkg/beam/core/runtime/harness/datamgr.go | 514 +++++++++++------
.../pkg/beam/core/runtime/harness/datamgr_test.go | 403 ++++++++++---
sdks/go/pkg/beam/core/runtime/harness/harness.go | 25 +-
sdks/go/pkg/beam/core/runtime/harness/logging.go | 92 +--
.../pkg/beam/core/runtime/harness/logging_test.go | 65 ++-
sdks/go/pkg/beam/core/runtime/harness/session.go | 232 --------
.../go/pkg/beam/core/runtime/harness/session.proto | 71 ---
.../core/runtime/harness/session/session.pb.go | 628 ---------------------
sdks/go/pkg/beam/io/fileio/example_test.go | 74 +++
sdks/go/pkg/beam/io/fileio/file.go | 142 +++++
sdks/go/pkg/beam/io/fileio/file_test.go | 254 +++++++++
sdks/go/pkg/beam/io/fileio/gzip.go | 60 ++
sdks/go/pkg/beam/io/fileio/helper_test.go | 88 +++
sdks/go/pkg/beam/io/fileio/match.go | 189 +++++++
sdks/go/pkg/beam/io/fileio/match_test.go | 285 ++++++++++
sdks/go/pkg/beam/io/fileio/read.go | 140 +++++
sdks/go/pkg/beam/io/fileio/read_test.go | 181 ++++++
.../java/org/apache/beam/sdk/schemas/Schema.java | 6 +-
.../org/apache/beam/sdk/testing/TestPipeline.java | 23 +-
.../org/apache/beam/sdk/schemas/SchemaTest.java | 1 -
.../apache/beam/sdk/testing/TestPipelineTest.java | 26 +
...xpansionServiceSchemaTransformProviderTest.java | 14 +-
.../expansion-service/build.gradle | 1 -
.../io/gcp/bigquery/BeamRowToStorageApiProto.java | 2 +-
.../beam/sdk/io/gcp/bigquery/BigQueryUtils.java | 11 +-
...ueryStorageWriteApiSchemaTransformProvider.java | 60 +-
.../io/gcp/bigtable/BigtableConfigTranslator.java | 9 +
.../beam/sdk/io/gcp/bigtable/BigtableIO.java | 8 +-
.../sdk/io/gcp/bigtable/BigtableReadOptions.java | 7 +
.../io/gcp/bigtable/BigtableServiceFactory.java | 7 +-
.../sdk/io/gcp/bigtable/BigtableServiceImpl.java | 24 +-
.../sdk/io/gcp/bigtable/BigtableWriteOptions.java | 5 +
.../gcp/bigquery/BeamRowToStorageApiProtoTest.java | 2 +-
...StorageWriteApiSchemaTransformProviderTest.java | 155 ++---
.../sdk/io/hbase/HBaseCoderProviderRegistrar.java | 1 +
.../java/org/apache/beam/sdk/io/hbase/HBaseIO.java | 221 ++++++++
.../beam/sdk/io/hbase/HBaseRowMutationsCoder.java | 150 +++++
.../beam/sdk/io/hbase/HBaseSharedConnection.java | 153 +++++
.../io/hbase/HBaseCoderProviderRegistrarTest.java | 6 +
.../sdk/io/hbase/HBaseSharedConnectionTest.java | 100 ++++
.../sdk/io/hbase/HbaseIOWriteRowMutationsTest.java | 214 +++++++
.../beam/sdk/io/hbase/RowMutationsCoderTest.java | 153 +++++
.../apache/beam/sdk/io/hbase/utils/HashUtils.java | 151 +++++
.../beam/sdk/io/hbase/utils/TestConstants.java | 37 ++
.../beam/sdk/io/hbase/utils/TestHBaseUtils.java | 73 +++
.../examples/ml-orchestration/tfx/requirements.txt | 2 +-
sdks/python/apache_beam/io/gcp/bigquery.py | 213 ++++++-
sdks/python/apache_beam/io/gcp/bigquery_tools.py | 119 ++++
.../apache_beam/io/gcp/bigquery_tools_test.py | 238 ++++++++
.../apache_beam/io/gcp/bigquery_write_it_test.py | 190 +++++++
.../apache_beam/runners/dataflow/internal/names.py | 4 +-
sdks/python/apache_beam/transforms/combiners.py | 20 +-
.../apache_beam/transforms/combiners_test.py | 14 +
sdks/python/apache_beam/transforms/external.py | 30 +
.../python/apache_beam/transforms/external_test.py | 43 ++
sdks/python/container/Dockerfile | 2 +-
sdks/python/pytest.ini | 1 +
sdks/python/setup.py | 9 +-
sdks/python/test-suites/dataflow/common.gradle | 24 +
sdks/python/test-suites/direct/common.gradle | 25 +
sdks/python/test-suites/xlang/build.gradle | 62 ++
settings.gradle.kts | 1 +
.../en/documentation/ml/inference-overview.md | 4 +-
.../content/en/documentation/programming-guide.md | 7 +-
126 files changed, 6658 insertions(+), 2390 deletions(-)
create mode 100644
.test-infra/jenkins/job_PostCommit_Python_CrossLanguage_Gcp_Dataflow.groovy
create mode 100644
.test-infra/jenkins/job_PostCommit_Python_CrossLanguage_Gcp_Direct.groovy
create mode 100644
playground/backend/internal/fs_tool/java_testdata/HasIncorrectMain.java
create mode 100644
playground/backend/internal/fs_tool/java_testdata/HasMainTest1.java
create mode 100644
playground/backend/internal/fs_tool/java_testdata/HasMainTest2.java
create mode 100644
playground/backend/internal/fs_tool/java_testdata/HasNoMain.java
delete mode 100644 sdks/go/pkg/beam/core/runtime/harness/session.go
delete mode 100644 sdks/go/pkg/beam/core/runtime/harness/session.proto
delete mode 100644 sdks/go/pkg/beam/core/runtime/harness/session/session.pb.go
create mode 100644 sdks/go/pkg/beam/io/fileio/example_test.go
create mode 100644 sdks/go/pkg/beam/io/fileio/file.go
create mode 100644 sdks/go/pkg/beam/io/fileio/file_test.go
create mode 100644 sdks/go/pkg/beam/io/fileio/gzip.go
create mode 100644 sdks/go/pkg/beam/io/fileio/helper_test.go
create mode 100644 sdks/go/pkg/beam/io/fileio/match.go
create mode 100644 sdks/go/pkg/beam/io/fileio/match_test.go
create mode 100644 sdks/go/pkg/beam/io/fileio/read.go
create mode 100644 sdks/go/pkg/beam/io/fileio/read_test.go
create mode 100644
sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseRowMutationsCoder.java
create mode 100644
sdks/java/io/hbase/src/main/java/org/apache/beam/sdk/io/hbase/HBaseSharedConnection.java
create mode 100644
sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/HBaseSharedConnectionTest.java
create mode 100644
sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/HbaseIOWriteRowMutationsTest.java
create mode 100644
sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/RowMutationsCoderTest.java
create mode 100644
sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/utils/HashUtils.java
create mode 100644
sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/utils/TestConstants.java
create mode 100644
sdks/java/io/hbase/src/test/java/org/apache/beam/sdk/io/hbase/utils/TestHBaseUtils.java
create mode 100644 sdks/python/test-suites/xlang/build.gradle
