This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a change to tag nightly-master
in repository https://gitbox.apache.org/repos/asf/beam.git.
*** WARNING: tag nightly-master was modified! ***
from 0cddc44 (commit)
to 70ac565 (commit)
from 0cddc44 [BEAM-13052] Restructure pubsublite folder to move non-user
interface classes to internal folder (#15724)
add de396a6 [BEAM-13066] Produce abstract iterables from IterableCoder.
(#15662)
add fb6d554 [BEAM-12988] [Playground] Add LifeCycle; Add Java SDK
environment of LifeCycle; Fix test description into README;
add bf91bd7 Merge pull request #15645 from [BEAM-12988] [Playground] Add
FileSystem service
add 85a1227 [BEAM-11936] Fix some errorprone warnings (#15648)
add d30a4e9 [BEAM-12393] sql support for Zeta Sql
add 40d6797 [BEAM-12393] package private
add 279949d [BEAM-12393] returning more generic interface
add 8e6d8de Merge pull request #15728: [BEAM-12393] sql support for Zeta
Sql
add debbd71 [BEAM-13068] Add xlangx.DecodeStructPayload (#15741)
add a2f08e5 Revert "Merge pull request #15441 from [BEAM-8823] Make
FnApiRunner work by executing ready elements instead of stages"
add a563ddd [BEAM-13040] Add some test cases enforcing side input waiting.
add a9ec3c8 lint
add ffde2a6 Merge pull request #15719 from [BEAM-13040] Revert breaking
PR and add some test cases.
add b9edf86 [BEAM-9487] Multiple Trigger.may_lose_data fixes
add d01c152 [BEAM-9487] Remove CONDITION_NOT_GUARANTEED as potential data
loss reason and update GBK test to get around DirectRunner bug
add 0bb51d7 [BEAM-9487] Do AfterAny, AfterAll, and AfterEach checks
properly (i.e. no shortcuts)
add e2b6f56 [BEAM-9487] Remove unused import
add a9708ab [BEAM-9487] Reintroduce flag but do not use it
add 0e9260a [BEAM-9487] Add test that shows AfterCount finishing
add 9343f37 [BEAM-9487] Make _ParallelTriggerFn.may_finish clearer
add abe3aa7 Merge pull request #15603 from zhoufek/fat
add d0253ed [BEAM-13015] Implement a simplified cancellable blocking
queue with similar semantics to ArrayBlockingQueue.
add 70ac565 [BEAM-13015] Implement a simplified cancellable blocking
queue with similar semantics to ArrayBlockingQueue.
No new revisions were added by this update.
Summary of changes:
CHANGES.md | 1 +
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 10 -
.../examples/subprocess/ExampleEchoPipeline.java | 10 -
.../beam/examples/complete/AutoCompleteTest.java | 2 +-
playground/backend/README.md | 2 +-
playground/backend/go.mod | 1 +
playground/backend/internal/fs_tool/fs.go | 135 +++++++
playground/backend/internal/fs_tool/fs_test.go | 415 +++++++++++++++++++
playground/backend/internal/fs_tool/java_fs.go | 48 +++
.../backend/internal/fs_tool/java_fs_test.go | 64 +++
runners/core-construction-java/build.gradle | 1 +
.../core/construction/ParDoTranslation.java | 3 +-
.../beam/runners/core/construction/Timer.java | 4 +-
.../core/construction/TriggerTranslation.java | 26 +-
.../core/construction/graph/GreedyStageFuser.java | 4 +-
.../construction/graph/SideInputReference.java | 2 +-
.../construction/renderer/PipelineDotRenderer.java | 9 +-
.../renderer/PortablePipelineDotRenderer.java | 5 +-
.../runners/core/construction/CommonCoderTest.java | 1 +
.../core/construction/ValidateRunnerXlangTest.java | 8 -
.../beam/runners/core/metrics/GaugeData.java | 2 +-
.../core/metrics/MetricsContainerStepMap.java | 2 +-
.../core/metrics/MonitoringInfoMetricName.java | 2 +-
.../beam/runners/core/LateDataUtilsTest.java | 2 +-
.../runners/core/MergingActiveWindowSetTest.java | 2 +-
.../beam/runners/core/ReduceFnRunnerTest.java | 22 +-
.../beam/runners/core/SideInputHandlerTest.java | 12 +-
.../SimplePushbackSideInputDoFnRunnerTest.java | 13 +-
.../beam/runners/core/StatefulDoFnRunnerTest.java | 9 +-
.../direct/BoundedReadEvaluatorFactory.java | 3 +-
.../runners/direct/ImmutableListBundleFactory.java | 4 +-
.../beam/runners/direct/NanosOffsetClock.java | 4 +-
.../beam/runners/direct/ViewEvaluatorFactory.java | 3 +-
.../beam/runners/direct/WatermarkManager.java | 4 -
.../direct/GroupByKeyOnlyEvaluatorFactoryTest.java | 4 -
.../FlinkStreamingPortablePipelineTranslator.java | 2 +-
.../translation/utils/LookupPipelineVisitor.java | 2 +-
.../wrappers/streaming/DoFnOperatorTest.java | 19 +-
.../wrappers/streaming/io/TestCountingSource.java | 2 +-
.../dataflow/DataflowPipelineTranslator.java | 4 +-
...DefaultCoderCloudObjectTranslatorRegistrar.java | 3 +-
.../runners/dataflow/DataflowPipelineJobTest.java | 20 -
.../runners/dataflow/util/CloudObjectsTest.java | 3 +-
.../beam/runners/dataflow/worker/OrderedCode.java | 20 +-
.../dataflow/worker/StreamingDataflowWorker.java | 2 +-
.../dataflow/worker/WindmillComputationKey.java | 2 +-
.../control/RegisterAndProcessBundleOperation.java | 2 +-
.../beam/runners/dataflow/worker/graph/Nodes.java | 8 +-
.../worker/StreamingDataflowWorkerTest.java | 2 +-
.../worker/StreamingModeExecutionContextTest.java | 7 -
.../CreateRegisterFnOperationFunctionTest.java | 2 +-
.../graph/DeduceNodeLocationsFunctionTest.java | 6 -
.../worker/util/ScalableBloomFilterTest.java | 14 +-
.../worker/windmill/GrpcWindmillServerTest.java | 2 +-
.../control/BundleCheckpointHandlers.java | 2 +-
.../jobsubmission/PortablePipelineJarUtils.java | 5 +-
.../apache/beam/runners/jet/JetPipelineResult.java | 2 +-
runners/samza/build.gradle | 1 +
.../runners/samza/adapter/BoundedSourceSystem.java | 6 -
.../samza/adapter/UnboundedSourceSystem.java | 5 -
.../runners/samza/util/PipelineJsonRenderer.java | 9 +-
.../samza/adapter/UnboundedSourceSystemTest.java | 5 -
.../beam/runners/spark/io/SourceDStream.java | 4 +-
.../translation/helpers/EncoderHelpers.java | 4 +-
.../metrics/sink/SparkMetricsSinkTest.java | 2 +-
.../spark/metrics/SparkMetricsPusherTest.java | 2 +-
.../translation/streaming/CreateStreamTest.java | 2 +-
.../streaming/SparkCoGroupByKeyStreamingTest.java | 2 +-
sdks/go/pkg/beam/core/runtime/xlangx/payload.go | 34 ++
.../runtime/xlangx/payload_test.go} | 41 +-
.../java/org/apache/beam/sdk/coders/AvroCoder.java | 5 +-
.../org/apache/beam/sdk/coders/DelegateCoder.java | 4 -
.../apache/beam/sdk/io/TextRowCountEstimator.java | 2 +-
.../org/apache/beam/sdk/io/fs/MetadataCoderV2.java | 3 +-
.../org/apache/beam/sdk/metrics/MetricKey.java | 2 +-
.../org/apache/beam/sdk/metrics/MetricName.java | 1 +
.../beam/sdk/metrics/MetricQueryResults.java | 2 +-
.../beam/sdk/schemas/FieldAccessDescriptor.java | 2 +-
.../java/org/apache/beam/sdk/schemas/Schema.java | 10 +-
.../org/apache/beam/sdk/schemas/SchemaCoder.java | 4 -
.../beam/sdk/schemas/transforms/Convert.java | 14 -
.../apache/beam/sdk/schemas/utils/AvroUtils.java | 2 +-
.../java/org/apache/beam/sdk/testing/PAssert.java | 10 +-
.../apache/beam/sdk/testing/WindowFnTestUtils.java | 3 +-
.../org/apache/beam/sdk/transforms/Combine.java | 3 +-
.../org/apache/beam/sdk/transforms/MapKeys.java | 2 +-
.../org/apache/beam/sdk/transforms/MapValues.java | 2 +-
.../org/apache/beam/sdk/transforms/PTransform.java | 4 +-
.../java/org/apache/beam/sdk/transforms/ParDo.java | 9 +-
.../beam/sdk/transforms/display/DisplayData.java | 6 +-
.../reflect/ByteBuddyDoFnInvokerFactory.java | 18 +-
.../sdk/transforms/reflect/DoFnSignatures.java | 29 +-
.../java/org/apache/beam/sdk/values/RowUtils.java | 2 +-
.../org/apache/beam/sdk/coders/AvroCoderTest.java | 2 +-
.../beam/sdk/coders/BigDecimalCoderTest.java | 4 +-
.../org/apache/beam/sdk/coders/RowCoderTest.java | 11 +-
.../test/java/org/apache/beam/sdk/io/ReadTest.java | 2 +-
.../beam/sdk/metrics/MetricResultsMatchers.java | 2 +-
.../sdk/options/PipelineOptionsReflectorTest.java | 9 -
.../apache/beam/sdk/schemas/SchemaOptionsTest.java | 4 -
.../beam/sdk/schemas/utils/AvroUtilsTest.java | 4 +-
.../apache/beam/sdk/testing/TestStreamTest.java | 7 +-
.../apache/beam/sdk/transforms/CombineTest.java | 4 +-
.../apache/beam/sdk/transforms/FlattenTest.java | 2 +-
.../apache/beam/sdk/transforms/GroupByKeyTest.java | 2 +-
.../org/apache/beam/sdk/transforms/LatestTest.java | 5 -
.../org/apache/beam/sdk/transforms/ParDoTest.java | 11 +-
.../apache/beam/sdk/transforms/PartitionTest.java | 4 +-
.../org/apache/beam/sdk/transforms/WaitTest.java | 3 +-
.../display/DisplayDataMatchersTest.java | 5 +-
.../sdk/transforms/display/DisplayDataTest.java | 69 ++--
.../beam/sdk/transforms/join/CoGroupByKeyTest.java | 4 +-
.../reflect/DoFnSignaturesSplittableDoFnTest.java | 7 +
.../sdk/transforms/windowing/FixedWindowsTest.java | 25 +-
.../sdk/transforms/windowing/SessionsTest.java | 23 +-
.../transforms/windowing/SlidingWindowsTest.java | 34 +-
.../sdk/transforms/windowing/WindowingTest.java | 13 +-
.../java/org/apache/beam/sdk/values/RowTest.java | 19 +-
.../service/JavaClassLookupTransformProvider.java | 2 +-
.../JavaClassLookupTransformProviderTest.java | 2 +-
.../euphoria/core/translate/JoinTranslator.java | 2 +-
.../euphoria/core/testkit/ReduceByKeyTest.java | 2 +-
.../extensions/gcp/util/CustomHttpErrorsTest.java | 11 -
.../ml/RecommendationAICreateCatalogItem.java | 2 +-
.../ml/RecommendationAIImportCatalogItems.java | 2 +-
.../ml/RecommendationAIImportUserEvents.java | 2 +-
.../sdk/extensions/ml/RecommendationAIPredict.java | 2 +-
.../ml/RecommendationAIWriteUserEvent.java | 2 +-
.../beam/sdk/extensions/protobuf/ProtoDomain.java | 7 -
.../sdk/extensions/sorter/NativeFileSorter.java | 5 -
.../beam/sdk/extensions/sorter/SortValues.java | 4 +-
.../provider/datacatalog/BigQueryTableFactory.java | 3 +-
.../meta/provider/datacatalog/GcsTableFactory.java | 3 +-
.../provider/datacatalog/PubsubTableFactory.java | 3 +-
.../sdk/extensions/sql/impl/cep/CEPPattern.java | 48 ---
.../extensions/sql/impl/planner/BeamCostModel.java | 2 +-
.../sql/impl/planner/RelMdNodeStats.java | 2 +-
.../sql/impl/rel/BeamAggregationRel.java | 2 +-
.../sql/impl/rel/BeamEnumerableConverter.java | 4 +-
.../sdk/extensions/sql/impl/rel/BeamJoinRel.java | 2 +-
.../sql/impl/rel/BeamSideInputJoinRel.java | 2 +-
.../sql/impl/rel/BeamTableFunctionScanRel.java | 2 +-
.../impl/transform/agg/VarianceAccumulator.java | 2 +-
.../pubsublite/PubsubLiteTableProvider.java | 11 -
.../sql/meta/provider/pubsublite/RowHandler.java | 2 +-
.../sql/BeamSqlDslSqlStdOperatorsTest.java | 5 +-
.../transform/agg/VarianceAccumulatorTest.java | 4 +-
.../provider/kafka/BeamKafkaTableAvroTest.java | 4 +-
.../provider/kafka/BeamKafkaTableJsonTest.java | 4 +-
.../provider/kafka/BeamKafkaTableProtoTest.java | 4 +-
.../provider/kafka/BeamKafkaTableThriftTest.java | 4 +-
.../zetasketch/ApproximateCountDistinct.java | 9 +-
.../beam/sdk/extensions/zetasketch/HllCount.java | 4 +
.../org/apache/beam/sdk/fn/CancellableQueue.java | 145 +++++++
.../org/apache/beam/sdk/fn/test/TestStreams.java | 12 -
.../apache/beam/sdk/fn/CancellableQueueTest.java | 213 ++++++++++
.../java/org/apache/beam/fn/harness/FnHarness.java | 3 +-
.../fn/harness/control/ProcessBundleHandler.java | 3 +-
.../fn/harness/state/CachingBeamFnStateClient.java | 2 +-
.../fn/harness/state/StateFetchingIterators.java | 2 +-
.../beam/fn/harness/FnApiDoFnRunnerTest.java | 4 +-
.../org/apache/beam/fn/harness/FnHarnessTest.java | 5 +-
.../beam/sdk/io/aws/s3/S3FileSystemTest.java | 10 +-
.../sdk/io/aws2/dynamodb/AttributeValueCoder.java | 2 +-
.../io/aws2/dynamodb/AttributeValueCoderTest.java | 2 +-
.../beam/sdk/io/aws2/s3/S3FileSystemTest.java | 5 +-
.../sdk/io/azure/blobstore/AzfsResourceId.java | 2 +-
.../blobstore/AzureBlobStoreFileSystemTest.java | 8 +-
.../org/apache/beam/sdk/io/cassandra/ReadFn.java | 18 +-
.../beam/sdk/io/cassandra/CassandraIOTest.java | 2 +-
.../io/contextualtextio/ContextualTextIOTest.java | 5 -
.../beam/sdk/io/elasticsearch/ElasticsearchIO.java | 2 +-
.../beam/sdk/io/gcp/bigquery/CivilTimeEncoder.java | 4 -
.../beam/sdk/io/gcp/bigquery/TestBigQuery.java | 15 +-
.../sdk/io/gcp/bigquery/WriteBundlesToFiles.java | 2 +-
.../beam/sdk/io/gcp/bigtable/BigtableConfig.java | 2 +-
.../beam/sdk/io/gcp/bigtable/BigtableIO.java | 4 +-
.../beam/sdk/io/gcp/datastore/DatastoreV1.java | 8 +-
.../apache/beam/sdk/io/gcp/healthcare/FhirIO.java | 4 -
.../apache/beam/sdk/io/gcp/healthcare/HL7v2IO.java | 5 -
.../apache/beam/sdk/io/gcp/pubsub/PubsubIO.java | 9 -
.../beam/sdk/io/gcp/pubsub/PubsubJsonClient.java | 3 +-
.../sdk/io/gcp/pubsub/PubsubUnboundedSink.java | 5 +-
.../sdk/io/gcp/pubsub/PubsubUnboundedSource.java | 3 +-
.../apache/beam/sdk/io/gcp/pubsub/TestPubsub.java | 10 -
.../beam/sdk/io/gcp/pubsub/TestPubsubSignal.java | 8 -
.../io/gcp/pubsublite/internal/PublisherCache.java | 2 +-
.../beam/sdk/io/gcp/spanner/MutationUtils.java | 2 +-
.../beam/sdk/io/gcp/spanner/OrderedCode.java | 10 +-
.../sdk/io/gcp/testing/FakeDatasetService.java | 5 +-
.../beam/sdk/io/gcp/bigquery/BigQueryKmsKeyIT.java | 2 +-
.../bigquery/BigQuerySchemaUpdateOptionsIT.java | 4 +-
.../io/gcp/bigquery/BigQueryServicesImplTest.java | 2 +-
.../sdk/io/gcp/bigquery/BigQueryToTableIT.java | 2 +-
.../sdk/io/gcp/bigquery/BigQueryUtilsTest.java | 2 +-
.../beam/sdk/io/gcp/bigtable/BigtableWriteIT.java | 3 +-
.../datastore/DataStoreV1SchemaIOProviderTest.java | 3 +-
.../beam/sdk/io/gcp/datastore/DatastoreV1Test.java | 11 -
.../beam/sdk/io/gcp/healthcare/DicomIOTest.java | 9 -
.../beam/sdk/io/gcp/healthcare/FhirIOLROIT.java | 4 +-
.../beam/sdk/io/gcp/healthcare/FhirIOReadIT.java | 4 +-
.../beam/sdk/io/gcp/healthcare/FhirIOSearchIT.java | 2 +-
.../beam/sdk/io/gcp/healthcare/FhirIOTest.java | 24 --
.../beam/sdk/io/gcp/healthcare/FhirIOWriteIT.java | 2 +-
.../beam/sdk/io/gcp/healthcare/HL7v2IOReadIT.java | 2 +-
.../sdk/io/gcp/healthcare/HL7v2IOReadWriteIT.java | 2 +-
.../beam/sdk/io/gcp/healthcare/HL7v2IOWriteIT.java | 2 +-
.../sdk/io/gcp/pubsub/PubsubIOExternalTest.java | 23 --
.../sdk/io/gcp/pubsub/PubsubJsonClientTest.java | 25 +-
.../io/gcp/pubsub/PubsubUnboundedSourceTest.java | 2 +-
.../sdk/io/gcp/spanner/SpannerIOWriteTest.java | 20 -
.../sdk/io/hadoop/format/EmployeeInputFormat.java | 2 +-
.../format/ReuseObjectsEmployeeInputFormat.java | 2 +-
.../java/org/apache/beam/sdk/io/jdbc/JdbcIO.java | 2 +-
.../org/apache/beam/sdk/io/jdbc/SchemaUtil.java | 5 +-
.../java/org/apache/beam/sdk/io/jdbc/JdbcIOIT.java | 4 +-
.../org/apache/beam/sdk/io/jdbc/JdbcIOTest.java | 4 +-
.../apache/beam/sdk/io/jdbc/SchemaUtilTest.java | 1 +
.../org/apache/beam/sdk/io/kafka/KafkaIOUtils.java | 2 +-
.../CustomTimestampPolicyWithLimitedDelayTest.java | 2 +-
.../beam/sdk/io/kafka/KafkaIOExternalTest.java | 32 --
.../org/apache/beam/sdk/io/kafka/KafkaIOTest.java | 4 +-
.../beam/sdk/io/mongodb/MongoDBGridFSIOTest.java | 2 +-
.../apache/beam/sdk/io/snowflake/SnowflakeIO.java | 6 +-
.../org/apache/beam/sdk/io/splunk/SplunkIO.java | 4 +-
.../sdk/io/synthetic/SyntheticSourceOptions.java | 2 +-
.../sdk/io/synthetic/delay/SyntheticDelay.java | 4 +-
.../sdk/io/synthetic/SyntheticOptionsTest.java | 11 +-
.../apache/beam/sdk/io/thrift/ThriftSchema.java | 3 +-
.../java/org/apache/beam/sdk/io/xml/XmlSource.java | 4 +-
.../org/apache/beam/sdk/loadtests/JobFailure.java | 15 +-
sdks/java/testing/nexmark/build.gradle | 1 +
.../org/apache/beam/sdk/nexmark/NexmarkUtils.java | 8 +-
.../beam/sdk/nexmark/queries/NexmarkQueryUtil.java | 7 +-
.../apache/beam/sdk/nexmark/queries/Query3.java | 3 +-
.../beam/sdk/nexmark/queries/WinningBids.java | 3 +-
.../apache/beam/sdk/tpcds/BeamSqlEnvRunner.java | 1 +
.../beam/sdk/tpcds/TableSchemaJSONLoader.java | 2 +-
sdks/python/apache_beam/coders/coder_impl.pxd | 8 +
sdks/python/apache_beam/coders/coder_impl.py | 37 +-
sdks/python/apache_beam/coders/coders.py | 3 +
.../snippets/transforms/aggregation/groupbykey.py | 1 +
sdks/python/apache_beam/io/iobase.py | 12 +-
.../runners/portability/fn_api_runner/execution.py | 357 ++++-------------
.../runners/portability/fn_api_runner/fn_runner.py | 442 +++++++--------------
.../portability/fn_api_runner/fn_runner_test.py | 51 ++-
.../portability/fn_api_runner/translations.py | 37 +-
.../portability/fn_api_runner/watermark_manager.py | 15 +-
.../portability/fn_api_runner/worker_handlers.py | 3 -
.../apache_beam/transforms/ptransform_test.py | 7 +-
sdks/python/apache_beam/transforms/trigger.py | 80 ++--
sdks/python/apache_beam/transforms/trigger_test.py | 153 ++++---
.../apache_beam/transforms/userstate_test.py | 4 +-
sdks/python/apache_beam/transforms/util.py | 3 -
254 files changed, 2033 insertions(+), 1684 deletions(-)
create mode 100644 playground/backend/internal/fs_tool/fs.go
create mode 100644 playground/backend/internal/fs_tool/fs_test.go
create mode 100644 playground/backend/internal/fs_tool/java_fs.go
create mode 100644 playground/backend/internal/fs_tool/java_fs_test.go
copy sdks/go/pkg/beam/{util/gcsx/gcs_test.go =>
core/runtime/xlangx/payload_test.go} (62%)
create mode 100644
sdks/java/fn-execution/src/main/java/org/apache/beam/sdk/fn/CancellableQueue.java
create mode 100644
sdks/java/fn-execution/src/test/java/org/apache/beam/sdk/fn/CancellableQueueTest.java
