Merge branch 'master' of https://github.com/apache/beam into sync-master
Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/f1582570 Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/f1582570 Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/f1582570 Branch: refs/heads/master Commit: f1582570981df349da625fad765d18366089b5dc Parents: 99f4f8b 893bf42 Author: manuzhang <owenzhang1...@gmail.com> Authored: Fri Jun 30 16:24:24 2017 +0800 Committer: manuzhang <owenzhang1...@gmail.com> Committed: Fri Jun 30 16:24:24 2017 +0800 ---------------------------------------------------------------------- .../jenkins/common_job_properties.groovy | 4 +- .../job_beam_PerformanceTests_Python.groovy | 58 ++ pom.xml | 20 +- .../apache/beam/runners/apex/ApexRunner.java | 2 +- .../core/construction/SplittableParDo.java | 66 +- .../WindowingStrategyTranslation.java | 26 +- .../core/construction/SplittableParDoTest.java | 8 +- .../beam/runners/core/ReduceFnRunner.java | 20 +- .../core/triggers/AfterAllStateMachine.java | 25 +- .../AfterDelayFromFirstElementStateMachine.java | 6 +- .../core/triggers/AfterFirstStateMachine.java | 20 +- .../core/triggers/AfterPaneStateMachine.java | 6 +- .../triggers/AfterWatermarkStateMachine.java | 7 +- .../triggers/ExecutableTriggerStateMachine.java | 23 +- .../core/triggers/NeverStateMachine.java | 5 +- .../core/triggers/TriggerStateMachine.java | 27 - .../beam/runners/core/ReduceFnRunnerTest.java | 161 +++++ .../triggers/AfterFirstStateMachineTest.java | 5 +- .../AfterWatermarkStateMachineTest.java | 7 +- .../core/triggers/StubTriggerStateMachine.java | 7 +- .../direct/ParDoMultiOverrideFactory.java | 16 +- .../flink/FlinkStreamingPipelineTranslator.java | 2 +- runners/google-cloud-dataflow-java/pom.xml | 2 +- .../dataflow/BatchStatefulParDoOverrides.java | 2 + .../dataflow/DataflowPipelineTranslator.java | 5 +- .../beam/runners/dataflow/DataflowRunner.java | 10 + .../dataflow/SplittableParDoOverrides.java | 2 +- .../runners/dataflow/DataflowRunnerTest.java | 38 ++ .../src/main/proto/beam_runner_api.proto | 14 + .../sdk/options/PipelineOptionsFactory.java | 18 +- .../sdk/options/PipelineOptionsValidator.java | 34 +- .../sdk/options/ProxyInvocationHandler.java | 4 + .../apache/beam/sdk/testing/StaticWindows.java | 5 + .../org/apache/beam/sdk/transforms/Combine.java | 13 +- .../org/apache/beam/sdk/transforms/DoFn.java | 3 + .../apache/beam/sdk/transforms/DoFnTester.java | 16 +- .../org/apache/beam/sdk/transforms/ParDo.java | 14 +- .../reflect/ByteBuddyDoFnInvokerFactory.java | 6 + .../sdk/transforms/reflect/DoFnInvoker.java | 4 +- .../sdk/transforms/windowing/GlobalWindows.java | 5 + .../windowing/PartitioningWindowFn.java | 5 + .../transforms/windowing/SlidingWindows.java | 5 + .../beam/sdk/transforms/windowing/Window.java | 32 + .../beam/sdk/transforms/windowing/WindowFn.java | 11 + .../apache/beam/sdk/util/IdentityWindowFn.java | 5 + .../beam/sdk/values/PCollectionViews.java | 14 + .../beam/sdk/values/WindowingStrategy.java | 46 +- .../options/PipelineOptionsValidatorTest.java | 44 ++ .../sdk/options/ProxyInvocationHandlerTest.java | 7 + .../apache/beam/sdk/transforms/CombineTest.java | 63 ++ .../beam/sdk/transforms/DoFnTesterTest.java | 32 + .../windowing/SlidingWindowsTest.java | 30 +- .../sdk/util/RetryHttpRequestInitializer.java | 148 +++-- .../extensions/gcp/GcpCoreApiSurfaceTest.java | 48 +- .../util/RetryHttpRequestInitializerTest.java | 31 +- sdks/java/io/amqp/pom.xml | 100 +++ .../org/apache/beam/sdk/io/amqp/AmqpIO.java | 399 ++++++++++++ .../beam/sdk/io/amqp/AmqpMessageCoder.java | 79 +++ .../amqp/AmqpMessageCoderProviderRegistrar.java | 44 ++ .../apache/beam/sdk/io/amqp/package-info.java | 22 + .../org/apache/beam/sdk/io/amqp/AmqpIOTest.java | 148 +++++ .../beam/sdk/io/amqp/AmqpMessageCoderTest.java | 89 +++ .../beam/sdk/io/cassandra/CassandraIO.java | 2 +- sdks/java/io/elasticsearch/pom.xml | 8 + .../sdk/io/elasticsearch/ElasticsearchIO.java | 13 +- sdks/java/io/google-cloud-platform/pom.xml | 13 +- .../beam/sdk/io/gcp/bigtable/BigtableIO.java | 6 +- .../io/gcp/bigtable/BigtableServiceImpl.java | 9 +- .../sdk/io/gcp/spanner/AbstractSpannerFn.java | 58 ++ .../sdk/io/gcp/spanner/CreateTransactionFn.java | 51 ++ .../sdk/io/gcp/spanner/NaiveSpannerReadFn.java | 65 ++ .../beam/sdk/io/gcp/spanner/SpannerConfig.java | 137 +++++ .../beam/sdk/io/gcp/spanner/SpannerIO.java | 604 +++++++++++++------ .../sdk/io/gcp/spanner/SpannerWriteGroupFn.java | 125 ++++ .../beam/sdk/io/gcp/spanner/Transaction.java | 33 + .../beam/sdk/io/gcp/GcpApiSurfaceTest.java | 10 + .../sdk/io/gcp/bigtable/BigtableReadIT.java | 5 +- .../io/gcp/bigtable/BigtableTestOptions.java | 5 - .../sdk/io/gcp/bigtable/BigtableWriteIT.java | 4 +- .../sdk/io/gcp/datastore/DatastoreV1Test.java | 4 +- .../beam/sdk/io/gcp/datastore/V1TestUtil.java | 2 +- .../sdk/io/gcp/spanner/FakeServiceFactory.java | 82 +++ .../beam/sdk/io/gcp/spanner/RandomUtils.java | 41 ++ .../sdk/io/gcp/spanner/SpannerIOReadTest.java | 281 +++++++++ .../beam/sdk/io/gcp/spanner/SpannerIOTest.java | 317 ---------- .../sdk/io/gcp/spanner/SpannerIOWriteTest.java | 258 ++++++++ .../beam/sdk/io/gcp/spanner/SpannerReadIT.java | 166 +++++ .../beam/sdk/io/gcp/spanner/SpannerWriteIT.java | 27 +- .../hadoop/inputformat/HadoopInputFormatIO.java | 2 +- .../inputformat/HIFIOWithElasticTest.java | 11 +- .../org/apache/beam/sdk/io/hbase/HBaseIO.java | 2 +- .../apache/beam/sdk/io/hcatalog/HCatalogIO.java | 2 +- .../org/apache/beam/sdk/io/jdbc/JdbcIO.java | 2 +- .../java/org/apache/beam/sdk/io/jms/JmsIO.java | 2 +- .../org/apache/beam/sdk/io/kafka/KafkaIO.java | 2 +- .../apache/beam/sdk/io/kinesis/KinesisIO.java | 2 +- .../beam/sdk/io/mongodb/MongoDbGridFSIO.java | 2 +- .../apache/beam/sdk/io/mongodb/MongoDbIO.java | 2 +- .../org/apache/beam/sdk/io/mqtt/MqttIO.java | 2 +- sdks/java/io/pom.xml | 1 + .../examples/snippets/snippets_test.py | 16 + .../apache_beam/examples/streaming_wordcount.py | 14 +- sdks/python/apache_beam/io/filesystem.py | 8 + sdks/python/apache_beam/io/gcp/pubsub.py | 162 +++-- sdks/python/apache_beam/io/gcp/pubsub_test.py | 89 +-- sdks/python/apache_beam/pipeline.py | 12 +- sdks/python/apache_beam/pipeline_test.py | 18 + .../runners/dataflow/dataflow_runner.py | 11 +- .../runners/dataflow/internal/apiclient.py | 6 +- .../runners/dataflow/internal/dependency.py | 69 ++- .../apache_beam/runners/direct/direct_runner.py | 83 ++- .../runners/direct/evaluation_context.py | 2 +- .../runners/direct/transform_evaluator.py | 227 ++++++- sdks/python/apache_beam/runners/direct/util.py | 25 +- .../runners/direct/watermark_manager.py | 26 +- .../runners/portability/fn_api_runner.py | 12 +- .../runners/portability/fn_api_runner_test.py | 23 +- .../apache_beam/runners/worker/data_plane.py | 10 +- .../apache_beam/runners/worker/sdk_worker.py | 2 +- .../apache_beam/testing/test_stream_test.py | 37 +- sdks/python/apache_beam/transforms/trigger.py | 16 + 121 files changed, 4273 insertions(+), 1071 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/beam/blob/f1582570/pom.xml ----------------------------------------------------------------------