This closes #3479: [BEAM-79] Merge master into gearpump-runner branch Don't call .testingPipelineOptions() a second time GCP IO ITs now all use --project option Select SDK distribution based on the selected SDK name [BEAM-2373] Upgrade commons-compress dependency version to 1.14 Define the projectId in the SpannerIO Read Test (utest, not itest) Use SDK harness container for FnAPI jobs when worker_harness_container_image is not specified. Add a separate image tag to use with the SDK harness container. Ditch apache commons Add PubSub I/O support to Python DirectRunner Only use ASCII 'a' through 'z' for temporary Spanner tables ReduceFnRunner.onTrigger: add short circuit for empty pane, and move inputWM and pane after the short circuit. WindowingStrategy: add OnTimeBehavior to control whether to emit empty ON_TIME pane. Removed OnceTriggerStateMachine Visit composite nodes when checking for picklability. Upgrade beam bigtable client dependency to 0.9.7.1 Add a Combine Test for Sliding Windows without Context [BEAM-2389] moved GcpCoreApiSurfaceTest to corresponding module, adapted exposed packagees Add Experimental annotation to AMQP and refine Kind for the Experimental IOs [BEAM-2488] Elasticsearch IO should read also in replica shards Use PCollectionViews.toAdditionalInputs in Combine Use PCollectionViews.toAdditionalInputs in ParDo Use PCollectionViews.toAdditionalInputs in ParDoMultiOverrideFactory Fix getAdditionalInputs for SplittableParDo transforms Add utility to expand list of PCollectionViews Read api with naive implementation Pre read api refactoring. Extract `SpannerConfig` and `AbstractSpannerFn` Bump spanner version [BEAM-1187] Improve logging to contain the number of retries done due to IOException and unsuccessful response codes. Add WindowFn#assignsToOneWindow Use installed distribution name for sdk name [BEAM-2522] upgrading jackson to 2.8.9 (mitigating #1599) Enable grpc controller in fn_api_runner Removed uses of proto builder clone method [BEAM-2514] Improve error message on missing required value [BEAM-1237] Create AmqpIO Implement streaming GroupByKey in Python DirectRunner Bump Dataflow worker to 0623 Reintroduces DoFn.ProcessContinuation (Dataflow worker compatibility part) Remove old deprecated PubSub code Fix a typo in function args Avoid pickling the entire pipeline per-transform. Fix python fn API data plane remote grpc port access [BEAM-2745] Add Jenkins Suite for Python Performance Test [BEAM-2489] Use dynamic ES port in HIFIOWithElasticTest [BEAM-2497] Fix the reading of concat gzip files Allow output from FinishBundle in DoFnTester DataflowRunner: Reject merging windowing for stateful ParDo
Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/627ae0bc Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/627ae0bc Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/627ae0bc Branch: refs/heads/master Commit: 627ae0bc3912ace4b5acf5b8e9fd779ad4461cf7 Parents: 99f4f8b f158257 Author: Kenneth Knowles <k...@google.com> Authored: Fri Jun 30 14:30:25 2017 -0700 Committer: Kenneth Knowles <k...@google.com> Committed: Fri Jun 30 14:30:25 2017 -0700 ---------------------------------------------------------------------- .../jenkins/common_job_properties.groovy | 4 +- .../job_beam_PerformanceTests_Python.groovy | 58 ++ pom.xml | 20 +- .../apache/beam/runners/apex/ApexRunner.java | 2 +- .../core/construction/SplittableParDo.java | 66 +- .../WindowingStrategyTranslation.java | 26 +- .../core/construction/SplittableParDoTest.java | 8 +- .../beam/runners/core/ReduceFnRunner.java | 20 +- .../core/triggers/AfterAllStateMachine.java | 25 +- .../AfterDelayFromFirstElementStateMachine.java | 6 +- .../core/triggers/AfterFirstStateMachine.java | 20 +- .../core/triggers/AfterPaneStateMachine.java | 6 +- .../triggers/AfterWatermarkStateMachine.java | 7 +- .../triggers/ExecutableTriggerStateMachine.java | 23 +- .../core/triggers/NeverStateMachine.java | 5 +- .../core/triggers/TriggerStateMachine.java | 27 - .../beam/runners/core/ReduceFnRunnerTest.java | 161 +++++ .../triggers/AfterFirstStateMachineTest.java | 5 +- .../AfterWatermarkStateMachineTest.java | 7 +- .../core/triggers/StubTriggerStateMachine.java | 7 +- .../direct/ParDoMultiOverrideFactory.java | 16 +- .../flink/FlinkStreamingPipelineTranslator.java | 2 +- runners/google-cloud-dataflow-java/pom.xml | 2 +- .../dataflow/BatchStatefulParDoOverrides.java | 2 + .../dataflow/DataflowPipelineTranslator.java | 5 +- .../beam/runners/dataflow/DataflowRunner.java | 10 + .../dataflow/SplittableParDoOverrides.java | 2 +- .../runners/dataflow/DataflowRunnerTest.java | 38 ++ .../src/main/proto/beam_runner_api.proto | 14 + .../sdk/options/PipelineOptionsFactory.java | 18 +- .../sdk/options/PipelineOptionsValidator.java | 34 +- .../sdk/options/ProxyInvocationHandler.java | 4 + .../apache/beam/sdk/testing/StaticWindows.java | 5 + .../org/apache/beam/sdk/transforms/Combine.java | 13 +- .../org/apache/beam/sdk/transforms/DoFn.java | 3 + .../apache/beam/sdk/transforms/DoFnTester.java | 16 +- .../org/apache/beam/sdk/transforms/ParDo.java | 14 +- .../reflect/ByteBuddyDoFnInvokerFactory.java | 6 + .../sdk/transforms/reflect/DoFnInvoker.java | 4 +- .../sdk/transforms/windowing/GlobalWindows.java | 5 + .../windowing/PartitioningWindowFn.java | 5 + .../transforms/windowing/SlidingWindows.java | 5 + .../beam/sdk/transforms/windowing/Window.java | 32 + .../beam/sdk/transforms/windowing/WindowFn.java | 11 + .../apache/beam/sdk/util/IdentityWindowFn.java | 5 + .../beam/sdk/values/PCollectionViews.java | 14 + .../beam/sdk/values/WindowingStrategy.java | 46 +- .../options/PipelineOptionsValidatorTest.java | 44 ++ .../sdk/options/ProxyInvocationHandlerTest.java | 7 + .../apache/beam/sdk/transforms/CombineTest.java | 63 ++ .../beam/sdk/transforms/DoFnTesterTest.java | 32 + .../windowing/SlidingWindowsTest.java | 30 +- .../sdk/util/RetryHttpRequestInitializer.java | 148 +++-- .../extensions/gcp/GcpCoreApiSurfaceTest.java | 48 +- .../util/RetryHttpRequestInitializerTest.java | 31 +- sdks/java/io/amqp/pom.xml | 100 +++ .../org/apache/beam/sdk/io/amqp/AmqpIO.java | 399 ++++++++++++ .../beam/sdk/io/amqp/AmqpMessageCoder.java | 79 +++ .../amqp/AmqpMessageCoderProviderRegistrar.java | 44 ++ .../apache/beam/sdk/io/amqp/package-info.java | 22 + .../org/apache/beam/sdk/io/amqp/AmqpIOTest.java | 148 +++++ .../beam/sdk/io/amqp/AmqpMessageCoderTest.java | 89 +++ .../beam/sdk/io/cassandra/CassandraIO.java | 2 +- sdks/java/io/elasticsearch/pom.xml | 8 + .../sdk/io/elasticsearch/ElasticsearchIO.java | 13 +- sdks/java/io/google-cloud-platform/pom.xml | 13 +- .../beam/sdk/io/gcp/bigtable/BigtableIO.java | 6 +- .../io/gcp/bigtable/BigtableServiceImpl.java | 9 +- .../sdk/io/gcp/spanner/AbstractSpannerFn.java | 58 ++ .../sdk/io/gcp/spanner/CreateTransactionFn.java | 51 ++ .../sdk/io/gcp/spanner/NaiveSpannerReadFn.java | 65 ++ .../beam/sdk/io/gcp/spanner/SpannerConfig.java | 137 +++++ .../beam/sdk/io/gcp/spanner/SpannerIO.java | 604 +++++++++++++------ .../sdk/io/gcp/spanner/SpannerWriteGroupFn.java | 125 ++++ .../beam/sdk/io/gcp/spanner/Transaction.java | 33 + .../beam/sdk/io/gcp/GcpApiSurfaceTest.java | 10 + .../sdk/io/gcp/bigtable/BigtableReadIT.java | 5 +- .../io/gcp/bigtable/BigtableTestOptions.java | 5 - .../sdk/io/gcp/bigtable/BigtableWriteIT.java | 4 +- .../sdk/io/gcp/datastore/DatastoreV1Test.java | 4 +- .../beam/sdk/io/gcp/datastore/V1TestUtil.java | 2 +- .../sdk/io/gcp/spanner/FakeServiceFactory.java | 82 +++ .../beam/sdk/io/gcp/spanner/RandomUtils.java | 41 ++ .../sdk/io/gcp/spanner/SpannerIOReadTest.java | 281 +++++++++ .../beam/sdk/io/gcp/spanner/SpannerIOTest.java | 317 ---------- .../sdk/io/gcp/spanner/SpannerIOWriteTest.java | 258 ++++++++ .../beam/sdk/io/gcp/spanner/SpannerReadIT.java | 166 +++++ .../beam/sdk/io/gcp/spanner/SpannerWriteIT.java | 27 +- .../hadoop/inputformat/HadoopInputFormatIO.java | 2 +- .../inputformat/HIFIOWithElasticTest.java | 11 +- .../org/apache/beam/sdk/io/hbase/HBaseIO.java | 2 +- .../apache/beam/sdk/io/hcatalog/HCatalogIO.java | 2 +- .../org/apache/beam/sdk/io/jdbc/JdbcIO.java | 2 +- .../java/org/apache/beam/sdk/io/jms/JmsIO.java | 2 +- .../org/apache/beam/sdk/io/kafka/KafkaIO.java | 2 +- .../apache/beam/sdk/io/kinesis/KinesisIO.java | 2 +- .../beam/sdk/io/mongodb/MongoDbGridFSIO.java | 2 +- .../apache/beam/sdk/io/mongodb/MongoDbIO.java | 2 +- .../org/apache/beam/sdk/io/mqtt/MqttIO.java | 2 +- sdks/java/io/pom.xml | 1 + .../examples/snippets/snippets_test.py | 16 + .../apache_beam/examples/streaming_wordcount.py | 14 +- sdks/python/apache_beam/io/filesystem.py | 8 + sdks/python/apache_beam/io/gcp/pubsub.py | 162 +++-- sdks/python/apache_beam/io/gcp/pubsub_test.py | 89 +-- sdks/python/apache_beam/pipeline.py | 12 +- sdks/python/apache_beam/pipeline_test.py | 18 + .../runners/dataflow/dataflow_runner.py | 11 +- .../runners/dataflow/internal/apiclient.py | 6 +- .../runners/dataflow/internal/dependency.py | 69 ++- .../apache_beam/runners/direct/direct_runner.py | 83 ++- .../runners/direct/evaluation_context.py | 2 +- .../runners/direct/transform_evaluator.py | 227 ++++++- sdks/python/apache_beam/runners/direct/util.py | 25 +- .../runners/direct/watermark_manager.py | 26 +- .../runners/portability/fn_api_runner.py | 12 +- .../runners/portability/fn_api_runner_test.py | 23 +- .../apache_beam/runners/worker/data_plane.py | 10 +- .../apache_beam/runners/worker/sdk_worker.py | 2 +- .../apache_beam/testing/test_stream_test.py | 37 +- sdks/python/apache_beam/transforms/trigger.py | 16 + 121 files changed, 4273 insertions(+), 1071 deletions(-) ----------------------------------------------------------------------