This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to branch nightly-refs/heads/master
in repository https://gitbox.apache.org/repos/asf/beam.git
from f72f6ce0e81 Remove CsvIOParseResult (#31819)
add 5579a16de7d Introduce support for emitting lineage in BiqQueryIOs
add dded4f06d82 Be spotless
add 024692647b4 A couple improvements to BQ source lineage.
add c827bbac387 Update contains test.
add c9adc8ee6c6 Merge pull request #31805 Introduce support for emitting
lineage in BQ Source.
add 018bcdf592c Add missing params to Python Bigtable MutationsBatcher
(#31791)
add a2260949431 Avoid publishing string set metrics on the Dataflow legacy
runner. (#31825)
add b9a44126622 Add changelog notes regarding Solace read connector
(#31826)
add 6c829db657b Bump google.golang.org/grpc from 1.64.0 to 1.64.1 in /sdks
(#31817)
add 4df89c704b0 Allow Firestore project to be configurable (#31808)
add 080c80a9573 Moving to 2.59.0-SNAPSHOT on master branch.
add 43b237e5040 Modernize python type hints for apache_beam
add cd495e9cf8a Modernize python type hints for apache_beam/coders
add 6143cd0e0a8 Modernize python type hints for apache_beam/dataframe
add d75916b6342 Modernize python type hints for
apache_beam/examples/cookbook
add c842252b4b5 Modernize python type hints for apache_beam/internal
add 33bde4de9c5 Modernize python type hints for
apache_beam/internal/metrics
add d73982af054 Modernize python type hints for apache_beam/io
add 8f6f24dc1ff Modernize python type hints for apache_beam/io/azure
add b8029e9aa1e Modernize python type hints for apache_beam/io/flink
add f49a29a4685 Modernize python type hints for apache_beam/io/gcp
add 5d5a09b6b3e Modernize python type hints for apache_beam/metrics
add fbafe8d780b Modernize python type hints for apache_beam/ml/gcp
add 0eab29802a3 Modernize python type hints for apache_beam/options
add 842b8ecab01 Modernize python type hints for apache_beam/runners
add 0763d7e50f3 Modernize python type hints for apache_beam/runners/direct
add 3cf0c5512a3 Modernize python type hints for
apache_beam/runners/interactive
add b41698210df Modernize python type hints for apache_beam/runners/job
add 8b540eb5075 Modernize python type hints for
apache_beam/runners/portability
add 8fdbe88dc6e Modernize python type hints for apache_beam/runners/worker
add 77d81895109 Modernize python type hints for
apache_beam/testing/benchmarks
add acfd72c7c06 Modernize python type hints for
apache_beam/testing/load_tests
add 4402f2d44e9 Modernize python type hints for apache_beam/transforms
add 79d4ffd20ba Modernize python type hints for apache_beam/typehints
add f2ffa5ec49e Modernize python type hints for apache_beam/utils
add abdb1b742a9 Fix circular references, mypy complaints.
add d4de077a2a9 Fix bad type declarations.
add 14c52d66ec6 Fix bad typing in PubSub tests.
add 64e6194b948 Preserve existing linter comments.
add a0ba8dea7d8 isort
add 36961405769 Merge pull request #31755 Modernize type hints.
add dd0912460c4 add doc warning against using icebergio directly (#31833)
add 7c0cf39001a Merge pull request #31823 Add lineage information for
BigQuery sinks.
add 8d5c3b5ee2c Locate and download Prism binary (#31796)
add 00bf1c6d036 Change orphan file log to warning (#31835)
No new revisions were added by this update.
Summary of changes:
.asf.yaml | 1 +
CHANGES.md | 2 +-
gradle.properties | 4 +-
.../dataflow/worker/BatchModeExecutionContext.java | 19 +-
.../worker/StreamingStepMetricsContainer.java | 7 +-
.../worker/StreamingStepMetricsContainerTest.java | 2 +
.../apache/beam/runners/prism/PrismLocator.java | 221 ++++++++++++++
.../beam/runners/prism/PrismLocatorTest.java | 125 ++++++++
sdks/go.mod | 2 +-
sdks/go.sum | 4 +-
sdks/go/pkg/beam/core/core.go | 2 +-
.../sdk/metrics/{NoOpCounter.java => Lineage.java} | 36 +--
.../beam/sdk/io/gcp/bigquery/BigQueryHelpers.java | 20 ++
.../sdk/io/gcp/bigquery/BigQuerySourceBase.java | 7 +-
.../io/gcp/bigquery/BigQueryStorageSourceBase.java | 13 +-
.../beam/sdk/io/gcp/bigquery/CreateTables.java | 8 +-
.../bigquery/StorageApiWriteUnshardedRecords.java | 14 +
.../bigquery/StorageApiWritesShardedRecords.java | 6 +
.../beam/sdk/io/gcp/bigquery/WriteRename.java | 6 +
.../beam/sdk/io/gcp/bigquery/WriteTables.java | 6 +
.../sdk/io/gcp/firestore/FirestoreOptions.java | 11 +
.../sdk/io/gcp/firestore/FirestoreV1ReadFn.java | 9 +-
.../sdk/io/gcp/firestore/FirestoreV1WriteFn.java | 9 +-
.../sdk/io/gcp/bigquery/BigQueryIOReadTest.java | 27 +-
.../sdk/io/gcp/bigquery/BigQueryIOWriteTest.java | 34 ++-
.../gcp/firestore/it/FirestoreTestingHelper.java | 5 +-
.../org/apache/beam/sdk/io/iceberg/IcebergIO.java | 8 +
sdks/python/apache_beam/coders/observable_test.py | 2 +-
sdks/python/apache_beam/coders/row_coder.py | 3 +-
sdks/python/apache_beam/coders/slow_stream.py | 32 +-
.../apache_beam/coders/standard_coders_test.py | 2 +-
sdks/python/apache_beam/coders/typecoders.py | 20 +-
sdks/python/apache_beam/dataframe/convert.py | 38 +--
sdks/python/apache_beam/dataframe/frame_base.py | 20 +-
sdks/python/apache_beam/dataframe/partitionings.py | 9 +-
sdks/python/apache_beam/dataframe/schemas.py | 9 +-
sdks/python/apache_beam/dataframe/schemas_test.py | 85 ++++--
.../examples/cookbook/bigtableio_it_test.py | 2 +-
.../examples/cookbook/datastore_wordcount.py | 4 +-
.../apache_beam/internal/cloudpickle_pickler.py | 4 +-
sdks/python/apache_beam/internal/dill_pickler.py | 7 +-
sdks/python/apache_beam/internal/metrics/cells.py | 27 +-
sdks/python/apache_beam/internal/metrics/metric.py | 66 +++--
sdks/python/apache_beam/internal/module_test.py | 4 +-
sdks/python/apache_beam/internal/pickler.py | 3 +-
sdks/python/apache_beam/internal/util.py | 11 +-
.../apache_beam/io/azure/blobstoragefilesystem.py | 4 -
sdks/python/apache_beam/io/filebasedsource.py | 24 +-
sdks/python/apache_beam/io/fileio.py | 14 +-
sdks/python/apache_beam/io/filesystem.py | 77 ++---
.../io/flink/flink_streaming_impulse_source.py | 2 +-
.../apache_beam/io/gcp/bigquery_avro_tools.py | 26 +-
.../apache_beam/io/gcp/bigquery_schema_tools.py | 5 +-
sdks/python/apache_beam/io/gcp/bigtableio.py | 5 +-
.../apache_beam/io/gcp/datastore/v1new/helper.py | 4 +-
.../apache_beam/io/gcp/datastore/v1new/types.py | 19 +-
sdks/python/apache_beam/io/gcp/gcsfilesystem.py | 8 +-
sdks/python/apache_beam/io/gcp/pubsub.py | 58 ++--
sdks/python/apache_beam/io/gcp/pubsub_test.py | 10 +-
sdks/python/apache_beam/io/hadoopfilesystem.py | 12 +-
sdks/python/apache_beam/io/iobase.py | 79 ++---
sdks/python/apache_beam/io/jdbc.py | 18 +-
sdks/python/apache_beam/io/localfilesystem.py | 8 +-
sdks/python/apache_beam/io/restriction_trackers.py | 9 +-
sdks/python/apache_beam/io/textio.py | 66 ++---
sdks/python/apache_beam/metrics/metric.py | 94 +++---
sdks/python/apache_beam/metrics/metricbase.py | 12 +-
.../python/apache_beam/ml/gcp/naturallanguageml.py | 36 +--
sdks/python/apache_beam/options/value_provider.py | 2 +-
sdks/python/apache_beam/pvalue.py | 125 ++++----
.../apache_beam/runners/direct/bundle_factory.py | 34 +--
.../direct/consumer_tracking_pipeline_visitor.py | 15 +-
.../apache_beam/runners/direct/direct_runner.py | 5 +-
.../runners/direct/evaluation_context.py | 123 ++++----
sdks/python/apache_beam/runners/direct/executor.py | 101 +++----
.../runners/direct/sdf_direct_runner.py | 17 +-
.../apache_beam/runners/direct/test_stream_impl.py | 5 +-
.../runners/direct/transform_evaluator.py | 62 ++--
.../runners/direct/watermark_manager.py | 84 +++---
.../runners/interactive/background_caching_job.py | 4 +-
.../runners/interactive/cache_manager.py | 4 +-
.../runners/interactive/cache_manager_test.py | 2 +-
.../runners/interactive/display/pipeline_graph.py | 17 +-
.../interactive/display/pipeline_graph_renderer.py | 30 +-
.../runners/interactive/options/capture_control.py | 10 +-
.../interactive/options/capture_limiters.py | 17 +-
.../runners/interactive/recording_manager.py | 143 ++++-----
.../testing/integration/notebook_executor.py | 3 +-
.../interactive/testing/integration/screen_diff.py | 7 +-
sdks/python/apache_beam/runners/job/utils.py | 6 +-
.../python/apache_beam/runners/pipeline_context.py | 140 ++++-----
.../runners/portability/abstract_job_service.py | 149 +++++-----
.../runners/portability/artifact_service.py | 18 +-
.../runners/portability/fn_api_runner/execution.py | 307 +++++++++----------
.../runners/portability/fn_api_runner/fn_runner.py | 325 ++++++++++-----------
.../portability/fn_api_runner/fn_runner_test.py | 2 +-
.../portability/fn_api_runner/watermark_manager.py | 11 +-
.../apache_beam/runners/portability/job_server.py | 6 +-
.../runners/portability/local_job_service.py | 41 ++-
.../runners/portability/portable_runner.py | 76 +++--
.../apache_beam/runners/portability/stager_test.py | 2 +-
sdks/python/apache_beam/runners/runner.py | 42 +--
sdks/python/apache_beam/runners/sdf_utils.py | 26 +-
.../apache_beam/runners/worker/log_handler.py | 35 +--
sdks/python/apache_beam/runners/worker/logger.py | 29 +-
.../apache_beam/runners/worker/statecache.py | 76 ++---
.../apache_beam/runners/worker/statesampler.py | 50 ++--
.../runners/worker/statesampler_slow.py | 51 ++--
.../runners/worker/worker_id_interceptor.py | 3 +-
.../apache_beam/runners/worker/worker_pool_main.py | 37 +--
.../apache_beam/runners/worker/worker_status.py | 4 +-
.../testing/benchmarks/nexmark/monitor.py | 3 +-
.../testing/benchmarks/nexmark/nexmark_launcher.py | 3 +-
.../testing/benchmarks/nexmark/nexmark_perf.py | 4 +-
.../testing/benchmarks/nexmark/nexmark_util.py | 18 +-
.../testing/load_tests/load_test_metrics_utils.py | 32 +-
.../transforms/combinefn_lifecycle_pipeline.py | 2 +-
sdks/python/apache_beam/transforms/combiners.py | 2 +-
.../python/apache_beam/transforms/external_java.py | 5 +-
sdks/python/apache_beam/transforms/resources.py | 39 +--
sdks/python/apache_beam/transforms/sideinputs.py | 28 +-
sdks/python/apache_beam/transforms/trigger.py | 7 +-
sdks/python/apache_beam/transforms/userstate.py | 149 ++++------
.../apache_beam/transforms/userstate_test.py | 2 +-
sdks/python/apache_beam/transforms/window.py | 164 +++++------
sdks/python/apache_beam/typehints/decorators.py | 80 +++--
.../typehints/native_type_compatibility.py | 2 +-
sdks/python/apache_beam/typehints/typehints.py | 2 +-
sdks/python/apache_beam/utils/profiler.py | 24 +-
sdks/python/apache_beam/utils/proto_utils.py | 34 +--
sdks/python/apache_beam/utils/python_callable.py | 6 +-
sdks/python/apache_beam/utils/sharded_key.py | 5 +-
sdks/python/apache_beam/utils/shared.py | 28 +-
sdks/python/apache_beam/utils/timestamp.py | 160 ++++------
sdks/python/apache_beam/utils/urns.py | 60 ++--
sdks/python/apache_beam/version.py | 2 +-
sdks/typescript/package.json | 2 +-
137 files changed, 2268 insertions(+), 2387 deletions(-)
create mode 100644
runners/prism/java/src/main/java/org/apache/beam/runners/prism/PrismLocator.java
create mode 100644
runners/prism/java/src/test/java/org/apache/beam/runners/prism/PrismLocatorTest.java
copy
sdks/java/core/src/main/java/org/apache/beam/sdk/metrics/{NoOpCounter.java =>
Lineage.java} (53%)
