This closes #2584
Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/aa07a1d4 Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/aa07a1d4 Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/aa07a1d4 Branch: refs/heads/DSL_SQL Commit: aa07a1d41b71c810c4968bd35e5ca69dc56b8c30 Parents: ca87603 19ae877 Author: Dan Halperin <[email protected]> Authored: Wed Apr 19 12:14:05 2017 -0700 Committer: Dan Halperin <[email protected]> Committed: Wed Apr 19 12:14:05 2017 -0700 ---------------------------------------------------------------------- .jenkins/common_job_properties.groovy | 261 ---- .../job_beam_PerformanceTests_Dataflow.groovy | 43 - .jenkins/job_beam_PerformanceTests_JDBC.groovy | 60 - .jenkins/job_beam_PerformanceTests_Spark.groovy | 44 - ...job_beam_PostCommit_Java_MavenInstall.groovy | 42 - ..._PostCommit_Java_ValidatesRunner_Apex.groovy | 48 - ...tCommit_Java_ValidatesRunner_Dataflow.groovy | 45 - ...PostCommit_Java_ValidatesRunner_Flink.groovy | 43 - ...tCommit_Java_ValidatesRunner_Gearpump.groovy | 49 - ...PostCommit_Java_ValidatesRunner_Spark.groovy | 44 - .../job_beam_PostCommit_Python_Verify.groovy | 55 - .../job_beam_PreCommit_Java_MavenInstall.groovy | 42 - .../job_beam_PreCommit_Website_Stage.groovy | 80 - .jenkins/job_beam_PreCommit_Website_Test.groovy | 65 - .../job_beam_Release_NightlySnapshot.groovy | 45 - .jenkins/job_seed.groovy | 53 - .../jenkins/common_job_properties.groovy | 261 ++++ .../job_beam_PerformanceTests_Dataflow.groovy | 43 + .../job_beam_PerformanceTests_JDBC.groovy | 60 + .../job_beam_PerformanceTests_Spark.groovy | 44 + ...job_beam_PostCommit_Java_MavenInstall.groovy | 42 + ..._PostCommit_Java_ValidatesRunner_Apex.groovy | 48 + ...tCommit_Java_ValidatesRunner_Dataflow.groovy | 45 + ...PostCommit_Java_ValidatesRunner_Flink.groovy | 43 + ...tCommit_Java_ValidatesRunner_Gearpump.groovy | 49 + ...PostCommit_Java_ValidatesRunner_Spark.groovy | 44 + .../job_beam_PostCommit_Python_Verify.groovy | 55 + .../job_beam_PreCommit_Java_MavenInstall.groovy | 42 + .../job_beam_PreCommit_Website_Stage.groovy | 80 + .../job_beam_PreCommit_Website_Test.groovy | 65 + .../job_beam_Release_NightlySnapshot.groovy | 45 + .test-infra/jenkins/job_seed.groovy | 53 + .../cassandra-service-for-local-dev.yaml | 28 + .../cassandra-svc-statefulset.yaml | 114 ++ .../LargeITCluster/cassandra-svc-temp.yaml | 74 + .../cassandra/LargeITCluster/data-load.sh | 122 ++ .../cassandra/LargeITCluster/show_health.sh | 47 + .../cassandra/LargeITCluster/start-up.sh | 22 + .../cassandra/LargeITCluster/teardown.sh | 25 + .../cassandra-service-for-local-dev.yaml | 30 + .../SmallITCluster/cassandra-svc-rc.yaml | 74 + .../cassandra/SmallITCluster/data-load.sh | 86 ++ .../cassandra/SmallITCluster/show_health.sh | 47 + .../cassandra/SmallITCluster/start-up.sh | 23 + .../cassandra/SmallITCluster/teardown.sh | 22 + .../kubernetes/cassandra/data-load-setup.sh | 29 + .../elasticsearch-service-for-local-dev.yaml | 33 + .../es-services-deployments.yaml | 258 ++++ .../LargeProductionCluster/start-up.sh | 22 + .../LargeProductionCluster/teardown.sh | 21 + .../elasticsearch-service-for-local-dev.yaml | 34 + .../SmallITCluster/elasticsearch-svc-rc.yaml | 96 ++ .../elasticsearch/SmallITCluster/start-up.sh | 23 + .../elasticsearch/SmallITCluster/teardown.sh | 21 + .../kubernetes/elasticsearch/data-load-setup.sh | 26 + .../kubernetes/elasticsearch/data-load.sh | 33 + .../kubernetes/elasticsearch/es_test_data.py | 299 ++++ .../kubernetes/elasticsearch/show-health.sh | 33 + .../postgres/postgres-service-for-local-dev.yml | 28 + .test-infra/kubernetes/postgres/postgres.yml | 56 + .test-infra/travis/README.md | 23 + .test-infra/travis/settings.xml | 33 + .test-infra/travis/test_wordcount.sh | 125 ++ .travis.yml | 4 +- .travis/README.md | 23 - .travis/settings.xml | 33 - .travis/test_wordcount.sh | 125 -- examples/java/pom.xml | 5 + examples/java8/pom.xml | 39 + pom.xml | 13 + runners/apex/pom.xml | 1 + .../apache/beam/runners/apex/ApexRunner.java | 32 +- .../beam/runners/apex/ApexYarnLauncher.java | 2 - .../apex/translation/ParDoTranslator.java | 6 +- .../operators/ApexGroupByKeyOperator.java | 226 +-- .../operators/ApexParDoOperator.java | 51 +- .../apex/translation/utils/NoOpStepContext.java | 2 +- .../utils/SerializablePipelineOptions.java | 13 +- .../apex/translation/utils/ValuesSource.java | 2 +- .../apex/examples/UnboundedTextSource.java | 2 +- .../translation/GroupByKeyTranslatorTest.java | 2 +- .../apex/translation/ParDoTranslatorTest.java | 18 +- .../translation/utils/CollectionSource.java | 2 +- runners/core-construction-java/pom.xml | 2 +- .../beam/runners/core/construction/Coders.java | 26 +- .../DeduplicatedFlattenFactory.java | 63 +- .../EmptyFlattenAsCreateFactory.java | 20 +- .../runners/core/construction/PCollections.java | 97 ++ .../core/construction/PTransformMatchers.java | 2 - .../construction/PTransformReplacements.java | 69 + .../runners/core/construction/PTransforms.java | 107 ++ .../core/construction/PrimitiveCreate.java | 13 +- .../core/construction/SdkComponents.java | 49 +- .../SingleInputOutputOverrideFactory.java | 9 +- .../runners/core/construction/Triggers.java | 336 +++++ .../UnboundedReadFromBoundedSource.java | 7 +- .../core/construction/UnconsumedReads.java | 72 + .../UnsupportedOverrideFactory.java | 14 +- .../core/construction/WindowingStrategies.java | 1 - .../runners/core/construction/CodersTest.java | 144 +- .../DeduplicatedFlattenFactoryTest.java | 18 +- .../EmptyFlattenAsCreateFactoryTest.java | 36 +- .../core/construction/PCollectionsTest.java | 188 +++ .../PTransformReplacementsTest.java | 131 ++ .../core/construction/PTransformsTest.java | 189 +++ .../core/construction/SdkComponentsTest.java | 85 +- .../SingleInputOutputOverrideFactoryTest.java | 31 +- .../runners/core/construction/TriggersTest.java | 111 ++ .../core/construction/UnconsumedReadsTest.java | 105 ++ .../UnsupportedOverrideFactoryTest.java | 11 +- runners/core-java/pom.xml | 5 + .../beam/runners/core/BaseExecutionContext.java | 13 +- .../apache/beam/runners/core/DoFnAdapters.java | 16 +- .../apache/beam/runners/core/DoFnRunners.java | 40 +- .../beam/runners/core/ExecutionContext.java | 13 +- .../GroupAlsoByWindowViaOutputBufferDoFn.java | 2 +- .../core/GroupAlsoByWindowViaWindowSetDoFn.java | 2 +- .../GroupAlsoByWindowViaWindowSetNewDoFn.java | 8 +- .../org/apache/beam/runners/core/OldDoFn.java | 41 +- ...eBoundedSplittableProcessElementInvoker.java | 8 +- .../beam/runners/core/OutputWindowedValue.java | 10 +- .../beam/runners/core/ProcessFnRunner.java | 127 ++ .../core/PushbackSideInputDoFnRunner.java | 106 +- .../beam/runners/core/SimpleDoFnRunner.java | 54 +- .../beam/runners/core/SimpleOldDoFnRunner.java | 63 +- .../core/SimplePushbackSideInputDoFnRunner.java | 115 ++ .../beam/runners/core/SplittableParDo.java | 131 +- .../beam/runners/core/WindowingInternals.java | 8 +- .../core/WindowingInternalsAdapters.java | 8 +- .../core/GroupAlsoByWindowsProperties.java | 10 +- .../apache/beam/runners/core/NoOpOldDoFn.java | 4 +- .../apache/beam/runners/core/OldDoFnTest.java | 4 +- ...ndedSplittableProcessElementInvokerTest.java | 6 +- .../core/PushbackSideInputDoFnRunnerTest.java | 282 ---- .../beam/runners/core/ReduceFnTester.java | 10 +- .../runners/core/SimpleOldDoFnRunnerTest.java | 4 +- .../SimplePushbackSideInputDoFnRunnerTest.java | 282 ++++ .../beam/runners/core/SplittableParDoTest.java | 98 +- runners/direct-java/pom.xml | 1 + .../direct/BoundedReadEvaluatorFactory.java | 2 +- ...ectGBKIntoKeyedWorkItemsOverrideFactory.java | 16 +- .../direct/DirectGroupByKeyOverrideFactory.java | 14 +- ...ecycleManagerRemovingTransformEvaluator.java | 6 +- .../GroupAlsoByWindowEvaluatorFactory.java | 10 +- .../beam/runners/direct/ModelEnforcement.java | 13 +- .../beam/runners/direct/ParDoEvaluator.java | 129 +- .../runners/direct/ParDoEvaluatorFactory.java | 23 +- .../direct/ParDoMultiOverrideFactory.java | 24 +- ...littableProcessElementsEvaluatorFactory.java | 108 +- .../direct/StatefulParDoEvaluatorFactory.java | 6 +- .../direct/TestStreamEvaluatorFactory.java | 14 +- .../direct/TransformEvaluatorFactory.java | 10 +- .../direct/TransformEvaluatorRegistry.java | 4 +- .../direct/UnboundedReadEvaluatorFactory.java | 2 +- .../runners/direct/ViewOverrideFactory.java | 18 +- .../direct/WriteWithShardingFactory.java | 16 +- .../direct/BoundedReadEvaluatorFactoryTest.java | 4 +- .../DirectGroupByKeyOverrideFactoryTest.java | 12 +- .../beam/runners/direct/DirectRunnerTest.java | 4 +- ...leManagerRemovingTransformEvaluatorTest.java | 8 +- .../beam/runners/direct/ParDoEvaluatorTest.java | 13 +- .../direct/ParDoMultiOverrideFactoryTest.java | 45 - .../direct/TestStreamEvaluatorFactoryTest.java | 12 - .../UnboundedReadEvaluatorFactoryTest.java | 2 +- .../runners/direct/ViewOverrideFactoryTest.java | 42 +- .../direct/WriteWithShardingFactoryTest.java | 23 +- runners/flink/examples/pom.xml | 126 -- .../beam/runners/flink/examples/TFIDF.java | 455 ------ .../beam/runners/flink/examples/WordCount.java | 129 -- .../runners/flink/examples/package-info.java | 22 - .../flink/examples/streaming/AutoComplete.java | 400 ----- .../flink/examples/streaming/JoinExamples.java | 154 -- .../examples/streaming/KafkaIOExamples.java | 338 ----- .../KafkaWindowedWordCountExample.java | 164 -- .../examples/streaming/WindowedWordCount.java | 141 -- .../flink/examples/streaming/package-info.java | 22 - runners/flink/pom.xml | 320 +++- runners/flink/runner/pom.xml | 311 ---- .../flink/DefaultParallelismFactory.java | 39 - .../flink/FlinkBatchPipelineTranslator.java | 139 -- .../flink/FlinkBatchTransformTranslators.java | 723 --------- .../flink/FlinkBatchTranslationContext.java | 153 -- .../flink/FlinkDetachedRunnerResult.java | 76 - .../FlinkPipelineExecutionEnvironment.java | 241 --- .../runners/flink/FlinkPipelineOptions.java | 101 -- .../runners/flink/FlinkPipelineTranslator.java | 53 - .../apache/beam/runners/flink/FlinkRunner.java | 232 --- .../runners/flink/FlinkRunnerRegistrar.java | 62 - .../beam/runners/flink/FlinkRunnerResult.java | 98 -- .../flink/FlinkStreamingPipelineTranslator.java | 272 ---- .../FlinkStreamingTransformTranslators.java | 1107 -------------- .../flink/FlinkStreamingTranslationContext.java | 130 -- .../flink/FlinkStreamingViewOverrides.java | 372 ----- .../flink/PipelineTranslationOptimizer.java | 72 - .../beam/runners/flink/TestFlinkRunner.java | 84 -- .../beam/runners/flink/TranslationMode.java | 31 - .../apache/beam/runners/flink/package-info.java | 22 - .../functions/FlinkAggregatorFactory.java | 53 - .../functions/FlinkAssignContext.java | 63 - .../functions/FlinkAssignWindows.java | 49 - .../functions/FlinkDoFnFunction.java | 161 -- .../FlinkMergingNonShuffleReduceFunction.java | 228 --- .../FlinkMergingPartialReduceFunction.java | 201 --- .../functions/FlinkMergingReduceFunction.java | 199 --- .../FlinkMultiOutputPruningFunction.java | 50 - .../functions/FlinkNoOpStepContext.java | 73 - .../functions/FlinkPartialReduceFunction.java | 172 --- .../functions/FlinkReduceFunction.java | 173 --- .../functions/FlinkSideInputReader.java | 80 - .../functions/FlinkStatefulDoFnFunction.java | 198 --- .../functions/SideInputInitializer.java | 73 - .../translation/functions/package-info.java | 22 - .../runners/flink/translation/package-info.java | 22 - .../translation/types/CoderTypeInformation.java | 120 -- .../translation/types/CoderTypeSerializer.java | 132 -- .../types/EncodedValueComparator.java | 195 --- .../types/EncodedValueSerializer.java | 113 -- .../types/EncodedValueTypeInformation.java | 107 -- .../flink/translation/types/FlinkCoder.java | 63 - .../types/InspectableByteArrayOutputStream.java | 34 - .../flink/translation/types/KvKeySelector.java | 50 - .../flink/translation/types/package-info.java | 22 - .../utils/SerializedPipelineOptions.java | 65 - .../flink/translation/utils/package-info.java | 22 - .../wrappers/DataInputViewWrapper.java | 58 - .../wrappers/DataOutputViewWrapper.java | 51 - .../SerializableFnAggregatorWrapper.java | 98 -- .../translation/wrappers/SourceInputFormat.java | 149 -- .../translation/wrappers/SourceInputSplit.java | 52 - .../translation/wrappers/package-info.java | 22 - .../wrappers/streaming/DoFnOperator.java | 772 ---------- .../streaming/KvToByteBufferKeySelector.java | 56 - .../streaming/SingletonKeyedWorkItem.java | 58 - .../streaming/SingletonKeyedWorkItemCoder.java | 128 -- .../streaming/SplittableDoFnOperator.java | 150 -- .../wrappers/streaming/WindowDoFnOperator.java | 120 -- .../wrappers/streaming/WorkItemKeySelector.java | 56 - .../streaming/io/BoundedSourceWrapper.java | 218 --- .../streaming/io/UnboundedFlinkSink.java | 200 --- .../streaming/io/UnboundedFlinkSource.java | 120 -- .../streaming/io/UnboundedSocketSource.java | 249 --- .../streaming/io/UnboundedSourceWrapper.java | 476 ------ .../wrappers/streaming/io/package-info.java | 22 - .../wrappers/streaming/package-info.java | 22 - .../state/FlinkBroadcastStateInternals.java | 865 ----------- .../state/FlinkKeyGroupStateInternals.java | 487 ------ .../state/FlinkSplitStateInternals.java | 260 ---- .../streaming/state/FlinkStateInternals.java | 1053 ------------- .../state/KeyGroupCheckpointedOperator.java | 35 - .../state/KeyGroupRestoringOperator.java | 32 - .../wrappers/streaming/state/package-info.java | 22 - .../runner/src/main/resources/log4j.properties | 23 - .../flink/EncodedValueComparatorTest.java | 70 - .../runners/flink/FlinkRunnerRegistrarTest.java | 48 - .../beam/runners/flink/FlinkTestPipeline.java | 72 - .../beam/runners/flink/PipelineOptionsTest.java | 184 --- .../beam/runners/flink/ReadSourceITCase.java | 85 -- .../flink/ReadSourceStreamingITCase.java | 74 - .../beam/runners/flink/WriteSinkITCase.java | 192 --- .../flink/streaming/DoFnOperatorTest.java | 600 -------- .../FlinkBroadcastStateInternalsTest.java | 245 --- .../FlinkKeyGroupStateInternalsTest.java | 262 ---- .../streaming/FlinkSplitStateInternalsTest.java | 101 -- .../streaming/FlinkStateInternalsTest.java | 395 ----- .../flink/streaming/GroupByNullKeyTest.java | 124 -- .../flink/streaming/TestCountingSource.java | 254 ---- .../streaming/TopWikipediaSessionsITCase.java | 133 -- .../streaming/UnboundedSourceWrapperTest.java | 464 ------ .../runners/flink/streaming/package-info.java | 22 - .../src/test/resources/log4j-test.properties | 27 - .../flink/DefaultParallelismFactory.java | 39 + .../flink/FlinkBatchPipelineTranslator.java | 139 ++ .../flink/FlinkBatchTransformTranslators.java | 723 +++++++++ .../flink/FlinkBatchTranslationContext.java | 153 ++ .../flink/FlinkDetachedRunnerResult.java | 75 + .../FlinkPipelineExecutionEnvironment.java | 241 +++ .../runners/flink/FlinkPipelineOptions.java | 101 ++ .../runners/flink/FlinkPipelineTranslator.java | 53 + .../apache/beam/runners/flink/FlinkRunner.java | 232 +++ .../runners/flink/FlinkRunnerRegistrar.java | 62 + .../beam/runners/flink/FlinkRunnerResult.java | 98 ++ .../flink/FlinkStreamingPipelineTranslator.java | 276 ++++ .../FlinkStreamingTransformTranslators.java | 1044 +++++++++++++ .../flink/FlinkStreamingTranslationContext.java | 130 ++ .../flink/FlinkStreamingViewOverrides.java | 372 +++++ .../flink/PipelineTranslationOptimizer.java | 72 + .../beam/runners/flink/TestFlinkRunner.java | 84 ++ .../beam/runners/flink/TranslationMode.java | 31 + .../apache/beam/runners/flink/package-info.java | 22 + .../functions/FlinkAggregatorFactory.java | 53 + .../functions/FlinkAssignContext.java | 63 + .../functions/FlinkAssignWindows.java | 49 + .../functions/FlinkDoFnFunction.java | 161 ++ .../FlinkMergingNonShuffleReduceFunction.java | 228 +++ .../FlinkMergingPartialReduceFunction.java | 201 +++ .../functions/FlinkMergingReduceFunction.java | 199 +++ .../FlinkMultiOutputPruningFunction.java | 50 + .../functions/FlinkNoOpStepContext.java | 73 + .../functions/FlinkPartialReduceFunction.java | 172 +++ .../functions/FlinkReduceFunction.java | 173 +++ .../functions/FlinkSideInputReader.java | 80 + .../functions/FlinkStatefulDoFnFunction.java | 198 +++ .../functions/SideInputInitializer.java | 73 + .../translation/functions/package-info.java | 22 + .../runners/flink/translation/package-info.java | 22 + .../translation/types/CoderTypeInformation.java | 120 ++ .../translation/types/CoderTypeSerializer.java | 132 ++ .../types/EncodedValueComparator.java | 195 +++ .../types/EncodedValueSerializer.java | 113 ++ .../types/EncodedValueTypeInformation.java | 98 ++ .../types/InspectableByteArrayOutputStream.java | 34 + .../flink/translation/types/KvKeySelector.java | 50 + .../flink/translation/types/package-info.java | 22 + .../utils/SerializedPipelineOptions.java | 67 + .../flink/translation/utils/package-info.java | 22 + .../wrappers/DataInputViewWrapper.java | 58 + .../wrappers/DataOutputViewWrapper.java | 51 + .../SerializableFnAggregatorWrapper.java | 98 ++ .../translation/wrappers/SourceInputFormat.java | 150 ++ .../translation/wrappers/SourceInputSplit.java | 52 + .../translation/wrappers/package-info.java | 22 + .../wrappers/streaming/DoFnOperator.java | 774 ++++++++++ .../streaming/KvToByteBufferKeySelector.java | 56 + .../streaming/SingletonKeyedWorkItem.java | 56 + .../streaming/SingletonKeyedWorkItemCoder.java | 126 ++ .../streaming/SplittableDoFnOperator.java | 150 ++ .../wrappers/streaming/WindowDoFnOperator.java | 117 ++ .../wrappers/streaming/WorkItemKeySelector.java | 56 + .../streaming/io/BoundedSourceWrapper.java | 218 +++ .../streaming/io/UnboundedSocketSource.java | 249 +++ .../streaming/io/UnboundedSourceWrapper.java | 476 ++++++ .../wrappers/streaming/io/package-info.java | 22 + .../wrappers/streaming/package-info.java | 22 + .../state/FlinkBroadcastStateInternals.java | 865 +++++++++++ .../state/FlinkKeyGroupStateInternals.java | 487 ++++++ .../state/FlinkSplitStateInternals.java | 260 ++++ .../streaming/state/FlinkStateInternals.java | 1053 +++++++++++++ .../state/KeyGroupCheckpointedOperator.java | 35 + .../state/KeyGroupRestoringOperator.java | 32 + .../wrappers/streaming/state/package-info.java | 22 + .../flink/src/main/resources/log4j.properties | 23 + .../flink/EncodedValueComparatorTest.java | 70 + .../runners/flink/FlinkRunnerRegistrarTest.java | 48 + .../beam/runners/flink/FlinkTestPipeline.java | 72 + .../beam/runners/flink/PipelineOptionsTest.java | 184 +++ .../beam/runners/flink/ReadSourceITCase.java | 85 ++ .../flink/ReadSourceStreamingITCase.java | 74 + .../beam/runners/flink/WriteSinkITCase.java | 192 +++ .../flink/streaming/DoFnOperatorTest.java | 600 ++++++++ .../FlinkBroadcastStateInternalsTest.java | 245 +++ .../FlinkKeyGroupStateInternalsTest.java | 262 ++++ .../streaming/FlinkSplitStateInternalsTest.java | 101 ++ .../streaming/FlinkStateInternalsTest.java | 395 +++++ .../flink/streaming/GroupByNullKeyTest.java | 124 ++ .../flink/streaming/TestCountingSource.java | 254 ++++ .../streaming/TopWikipediaSessionsITCase.java | 133 ++ .../streaming/UnboundedSourceWrapperTest.java | 464 ++++++ .../runners/flink/streaming/package-info.java | 22 + .../src/test/resources/log4j-test.properties | 27 + runners/google-cloud-dataflow-java/pom.xml | 62 +- .../dataflow/BatchStatefulParDoOverrides.java | 44 +- .../runners/dataflow/BatchViewOverrides.java | 23 +- .../beam/runners/dataflow/DataflowRunner.java | 96 +- .../dataflow/PrimitiveParDoSingleFactory.java | 15 +- .../dataflow/ReshuffleOverrideFactory.java | 12 +- .../dataflow/StreamingViewOverrides.java | 14 +- .../dataflow/internal/CustomSources.java | 2 +- .../dataflow/BatchViewOverridesTest.java | 4 +- .../dataflow/DataflowPipelineJobTest.java | 2 +- .../runners/dataflow/DataflowRunnerTest.java | 24 + .../PrimitiveParDoSingleFactoryTest.java | 59 +- runners/pom.xml | 40 - runners/spark/pom.xml | 2 +- .../beam/runners/spark/TestSparkRunner.java | 14 +- .../beam/runners/spark/io/MicrobatchSource.java | 7 +- .../beam/runners/spark/io/SourceDStream.java | 2 +- .../apache/beam/runners/spark/io/SourceRDD.java | 4 +- .../SparkGroupAlsoByWindowViaWindowSet.java | 13 +- .../spark/translation/MultiDoFnFunction.java | 4 +- ...SparkGroupAlsoByWindowViaOutputBufferFn.java | 10 +- .../spark/translation/SparkProcessContext.java | 2 +- .../spark/translation/SparkRuntimeContext.java | 2 + .../streaming/StreamingTransformTranslator.java | 2 +- .../spark/util/SparkSideInputReader.java | 3 +- sdks/common/fn-api/pom.xml | 5 - .../fn-api/src/main/proto/beam_fn_api.proto | 174 +-- .../src/main/proto/beam_runner_api.proto | 4 +- .../src/main/resources/beam/findbugs-filter.xml | 7 - sdks/java/core/pom.xml | 49 +- .../main/java/org/apache/beam/sdk/Pipeline.java | 58 +- .../java/org/apache/beam/sdk/io/AvroIO.java | 13 +- .../sdk/io/BoundedReadFromUnboundedSource.java | 4 +- .../org/apache/beam/sdk/io/BoundedSource.java | 13 +- .../apache/beam/sdk/io/CompressedSource.java | 4 +- .../org/apache/beam/sdk/io/CountingSource.java | 2 +- .../org/apache/beam/sdk/io/FileBasedSource.java | 9 +- .../apache/beam/sdk/io/OffsetBasedSource.java | 4 +- .../main/java/org/apache/beam/sdk/io/Sink.java | 2 +- .../java/org/apache/beam/sdk/io/TFRecordIO.java | 639 +++----- .../org/apache/beam/sdk/io/UnboundedSource.java | 15 +- .../main/java/org/apache/beam/sdk/io/XmlIO.java | 477 ++++++ .../java/org/apache/beam/sdk/io/XmlSink.java | 226 +-- .../java/org/apache/beam/sdk/io/XmlSource.java | 191 +-- .../beam/sdk/options/BigQueryOptions.java | 32 - .../options/CloudResourceManagerOptions.java | 40 - .../DefaultPipelineOptionsRegistrar.java | 5 - .../org/apache/beam/sdk/options/GcpOptions.java | 227 --- .../org/apache/beam/sdk/options/GcsOptions.java | 158 -- .../beam/sdk/options/GoogleApiDebugOptions.java | 87 -- .../beam/sdk/options/PipelineOptions.java | 16 +- .../apache/beam/sdk/options/PubsubOptions.java | 36 - .../sdk/runners/PTransformOverrideFactory.java | 33 +- .../apache/beam/sdk/runners/PipelineRunner.java | 7 +- .../beam/sdk/testing/BigqueryMatcher.java | 256 ---- .../beam/sdk/testing/SourceTestUtils.java | 6 +- .../apache/beam/sdk/testing/TestPipeline.java | 5 +- ...esSplittableParDoWithWindowedSideInputs.java | 26 + .../beam/sdk/transforms/AppliedPTransform.java | 5 + .../org/apache/beam/sdk/transforms/Combine.java | 8 +- .../org/apache/beam/sdk/transforms/Create.java | 7 +- .../org/apache/beam/sdk/transforms/DoFn.java | 14 +- .../apache/beam/sdk/transforms/DoFnTester.java | 41 +- .../beam/sdk/transforms/FlatMapElements.java | 4 +- .../apache/beam/sdk/transforms/MapElements.java | 4 +- .../org/apache/beam/sdk/transforms/ParDo.java | 86 +- .../apache/beam/sdk/transforms/Partition.java | 2 +- .../sdk/transforms/display/DisplayData.java | 6 + .../beam/sdk/transforms/windowing/Triggers.java | 322 ---- .../beam/sdk/transforms/windowing/Window.java | 1 + .../beam/sdk/transforms/windowing/WindowFn.java | 13 - .../beam/sdk/util/AppEngineEnvironment.java | 62 - ...AttemptAndTimeBoundedExponentialBackOff.java | 170 --- .../util/AttemptBoundedExponentialBackOff.java | 86 -- .../apache/beam/sdk/util/CredentialFactory.java | 29 - .../org/apache/beam/sdk/util/DefaultBucket.java | 105 -- .../beam/sdk/util/GcpCredentialFactory.java | 67 - .../apache/beam/sdk/util/GcpProjectUtil.java | 106 -- .../beam/sdk/util/GcsIOChannelFactory.java | 111 -- .../sdk/util/GcsIOChannelFactoryRegistrar.java | 38 - .../apache/beam/sdk/util/GcsPathValidator.java | 95 -- .../java/org/apache/beam/sdk/util/GcsUtil.java | 798 ---------- .../apache/beam/sdk/util/IOChannelUtils.java | 22 +- .../util/IntervalBoundedExponentialBackOff.java | 89 -- .../beam/sdk/util/NoopCredentialFactory.java | 68 - .../sdk/util/NullCredentialInitializer.java | 62 - .../org/apache/beam/sdk/util/Transport.java | 178 --- .../beam/sdk/values/PCollectionTuple.java | 3 +- .../org/apache/beam/sdk/values/TupleTag.java | 26 +- .../apache/beam/sdk/values/TupleTagList.java | 2 +- .../apache/beam/sdk/values/TypeDescriptors.java | 25 +- .../org/apache/beam/sdk/values/TypedPValue.java | 4 +- .../org/apache/beam/SdkCoreApiSurfaceTest.java | 2 - .../java/org/apache/beam/sdk/PipelineTest.java | 92 +- .../org/apache/beam/sdk/io/AvroSourceTest.java | 10 +- .../apache/beam/sdk/io/CountingSourceTest.java | 6 +- .../apache/beam/sdk/io/FileBasedSourceTest.java | 8 +- .../beam/sdk/io/OffsetBasedSourceTest.java | 8 +- .../java/org/apache/beam/sdk/io/ReadTest.java | 4 +- .../org/apache/beam/sdk/io/TFRecordIOTest.java | 20 +- .../java/org/apache/beam/sdk/io/TextIOTest.java | 113 +- .../org/apache/beam/sdk/io/XmlSinkTest.java | 89 +- .../org/apache/beam/sdk/io/XmlSourceTest.java | 250 +-- .../apache/beam/sdk/metrics/MetricsTest.java | 2 +- .../apache/beam/sdk/options/GcpOptionsTest.java | 171 --- .../sdk/options/GoogleApiDebugOptionsTest.java | 145 -- .../sdk/options/PipelineOptionsFactoryTest.java | 4 +- .../beam/sdk/runners/PipelineRunnerTest.java | 46 +- .../runners/dataflow/TestCountingSource.java | 2 +- .../beam/sdk/testing/BigqueryMatcherTest.java | 176 --- .../beam/sdk/testing/SourceTestUtilsTest.java | 2 +- .../beam/sdk/testing/StaticWindowsTest.java | 10 +- .../beam/sdk/testing/TestPipelineTest.java | 6 +- .../apache/beam/sdk/transforms/CreateTest.java | 20 +- .../apache/beam/sdk/transforms/ParDoTest.java | 293 ++-- .../beam/sdk/transforms/SplittableDoFnTest.java | 107 +- .../sdk/transforms/display/DisplayDataTest.java | 17 + .../sdk/transforms/windowing/TriggersTest.java | 100 -- ...mptAndTimeBoundedExponentialBackOffTest.java | 213 --- .../AttemptBoundedExponentialBackOffTest.java | 85 -- .../apache/beam/sdk/util/DefaultBucketTest.java | 112 -- .../beam/sdk/util/GcpProjectUtilTest.java | 76 - .../util/GcsIOChannelFactoryRegistrarTest.java | 44 - .../beam/sdk/util/GcsIOChannelFactoryTest.java | 43 - .../beam/sdk/util/GcsPathValidatorTest.java | 87 -- .../org/apache/beam/sdk/util/GcsUtilTest.java | 798 ---------- .../IntervalBoundedExponentialBackOffTest.java | 100 -- .../util/RetryHttpRequestInitializerTest.java | 290 ---- .../beam/sdk/values/PCollectionTupleTest.java | 8 +- .../apache/beam/sdk/values/TypedPValueTest.java | 46 +- sdks/java/extensions/gcp-core/pom.xml | 217 +++ .../beam/sdk/options/BigQueryOptions.java | 32 + .../options/CloudResourceManagerOptions.java | 40 + .../org/apache/beam/sdk/options/GcpOptions.java | 227 +++ .../options/GcpPipelineOptionsRegistrar.java | 39 + .../org/apache/beam/sdk/options/GcsOptions.java | 154 ++ .../beam/sdk/options/GoogleApiDebugOptions.java | 87 ++ .../apache/beam/sdk/options/PubsubOptions.java | 36 + .../apache/beam/sdk/options/package-info.java | 22 + .../beam/sdk/testing/BigqueryMatcher.java | 256 ++++ .../apache/beam/sdk/testing/package-info.java | 21 + .../apache/beam/sdk/util/CredentialFactory.java | 29 + .../org/apache/beam/sdk/util/DefaultBucket.java | 105 ++ .../beam/sdk/util/GcpCredentialFactory.java | 67 + .../apache/beam/sdk/util/GcpProjectUtil.java | 106 ++ .../beam/sdk/util/GcsIOChannelFactory.java | 111 ++ .../sdk/util/GcsIOChannelFactoryRegistrar.java | 38 + .../apache/beam/sdk/util/GcsPathValidator.java | 95 ++ .../java/org/apache/beam/sdk/util/GcsUtil.java | 798 ++++++++++ .../beam/sdk/util/NoopCredentialFactory.java | 68 + .../sdk/util/NullCredentialInitializer.java | 62 + .../org/apache/beam/sdk/util/Transport.java | 178 +++ .../org/apache/beam/sdk/util/package-info.java | 20 + .../org/apache/beam/GcpCoreApiSurfaceTest.java | 62 + .../apache/beam/sdk/options/GcpOptionsTest.java | 171 +++ .../sdk/options/GoogleApiDebugOptionsTest.java | 145 ++ .../beam/sdk/testing/BigqueryMatcherTest.java | 176 +++ .../apache/beam/sdk/util/DefaultBucketTest.java | 112 ++ .../beam/sdk/util/GcpProjectUtilTest.java | 76 + .../util/GcsIOChannelFactoryRegistrarTest.java | 44 + .../beam/sdk/util/GcsIOChannelFactoryTest.java | 43 + .../beam/sdk/util/GcsPathValidatorTest.java | 87 ++ .../org/apache/beam/sdk/util/GcsUtilTest.java | 798 ++++++++++ .../util/RetryHttpRequestInitializerTest.java | 290 ++++ sdks/java/extensions/pom.xml | 1 + sdks/java/harness/pom.xml | 5 + .../beam/fn/harness/fake/FakeStepContext.java | 2 +- .../control/ProcessBundleHandlerTest.java | 30 +- .../sdk/io/elasticsearch/ElasticsearchIO.java | 2 +- .../sdk/io/elasticsearch/ElasticsearchIOIT.java | 2 +- .../io/elasticsearch/ElasticsearchIOTest.java | 4 +- sdks/java/io/google-cloud-platform/pom.xml | 20 +- .../sdk/io/gcp/bigquery/BatchLoadBigQuery.java | 180 --- .../beam/sdk/io/gcp/bigquery/BatchLoads.java | 225 +++ .../sdk/io/gcp/bigquery/BigQueryHelpers.java | 13 + .../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 172 +-- .../sdk/io/gcp/bigquery/BigQuerySourceBase.java | 33 +- .../io/gcp/bigquery/BigQueryTableSource.java | 34 +- .../beam/sdk/io/gcp/bigquery/CreateTables.java | 127 ++ .../io/gcp/bigquery/GenerateShardedTable.java | 47 + .../beam/sdk/io/gcp/bigquery/PrepareWrite.java | 81 + .../beam/sdk/io/gcp/bigquery/ShardedKey.java | 25 +- .../sdk/io/gcp/bigquery/StreamWithDeDup.java | 90 -- .../sdk/io/gcp/bigquery/StreamingInserts.java | 79 + .../sdk/io/gcp/bigquery/StreamingWriteFn.java | 81 +- .../io/gcp/bigquery/StreamingWriteTables.java | 86 ++ .../sdk/io/gcp/bigquery/TableDestination.java | 76 + .../io/gcp/bigquery/TableDestinationCoder.java | 60 + .../sdk/io/gcp/bigquery/TableRowWriter.java | 19 +- .../sdk/io/gcp/bigquery/TagWithUniqueIds.java | 62 + .../gcp/bigquery/TagWithUniqueIdsAndTable.java | 135 -- .../sdk/io/gcp/bigquery/TransformingSource.java | 4 +- .../beam/sdk/io/gcp/bigquery/WriteBundles.java | 82 - .../io/gcp/bigquery/WriteBundlesToFiles.java | 157 ++ .../sdk/io/gcp/bigquery/WritePartition.java | 163 +- .../beam/sdk/io/gcp/bigquery/WriteRename.java | 71 +- .../beam/sdk/io/gcp/bigquery/WriteTables.java | 58 +- .../beam/sdk/io/gcp/bigtable/BigtableIO.java | 6 +- .../io/gcp/pubsub/PubsubUnboundedSource.java | 6 +- .../sdk/io/gcp/bigquery/BigQueryIOTest.java | 1419 +++++++----------- .../io/gcp/bigquery/FakeBigQueryServices.java | 166 ++ .../sdk/io/gcp/bigquery/FakeDatasetService.java | 208 +++ .../sdk/io/gcp/bigquery/FakeJobService.java | 404 +++++ .../sdk/io/gcp/bigquery/TableContainer.java | 61 + .../sdk/io/gcp/bigtable/BigtableIOTest.java | 8 +- .../gcp/pubsub/PubsubUnboundedSourceTest.java | 4 +- .../hadoop/inputformat/HadoopInputFormatIO.java | 4 +- .../inputformat/HadoopInputFormatIOTest.java | 19 +- .../integration/tests/HIFIOCassandraIT.java | 6 +- .../integration/tests/HIFIOElasticIT.java | 4 +- .../SmallITCluster/cassandra-svc-rc.yaml | 88 -- .../cassandra/SmallITCluster/start-up.sh | 21 - .../cassandra/SmallITCluster/teardown.sh | 21 - .../kubernetes/cassandra/data-load-setup.sh | 29 - .../resources/kubernetes/cassandra/data-load.sh | 67 - .../LargeProductionCluster/es-services.yaml | 277 ---- .../LargeProductionCluster/start-up.sh | 21 - .../LargeProductionCluster/teardown.sh | 20 - .../SmallITCluster/elasticsearch-svc-rc.yaml | 84 -- .../elasticsearch/SmallITCluster/start-up.sh | 22 - .../elasticsearch/SmallITCluster/teardown.sh | 20 - .../kubernetes/elasticsearch/data-load-setup.sh | 26 - .../kubernetes/elasticsearch/data-load.sh | 33 - .../kubernetes/elasticsearch/es_test_data.py | 299 ---- .../kubernetes/elasticsearch/show-health.sh | 25 - sdks/java/io/hadoop/pom.xml | 15 +- .../org/apache/beam/sdk/io/hbase/HBaseIO.java | 7 +- .../apache/beam/sdk/io/hbase/HBaseIOTest.java | 2 +- .../apache/beam/sdk/io/hdfs/HDFSFileSource.java | 6 +- .../beam/sdk/io/hdfs/HDFSFileSourceTest.java | 4 +- .../org/apache/beam/sdk/io/jdbc/JdbcIO.java | 42 +- .../org/apache/beam/sdk/io/jdbc/JdbcIOTest.java | 10 +- .../postgres-service-for-local-dev.yml | 28 - .../src/test/resources/kubernetes/postgres.yml | 56 - .../jdbc/src/test/resources/kubernetes/setup.sh | 19 - .../src/test/resources/kubernetes/teardown.sh | 19 - .../java/org/apache/beam/sdk/io/jms/JmsIO.java | 2 +- .../org/apache/beam/sdk/io/jms/JmsIOTest.java | 4 +- .../org/apache/beam/sdk/io/kafka/KafkaIO.java | 12 +- .../apache/beam/sdk/io/kafka/KafkaIOTest.java | 8 +- .../beam/sdk/io/kinesis/KinesisSource.java | 2 +- .../beam/sdk/io/mongodb/MongoDbGridFSIO.java | 11 +- .../apache/beam/sdk/io/mongodb/MongoDbIO.java | 2 +- .../sdk/io/mongodb/MongoDBGridFSIOTest.java | 2 +- .../org/apache/beam/sdk/io/mqtt/MqttIO.java | 2 +- sdks/python/.pylintrc | 6 +- sdks/python/apache_beam/coders/coder_impl.py | 50 +- sdks/python/apache_beam/coders/coders.py | 6 +- sdks/python/apache_beam/error.py | 4 + .../examples/complete/game/user_score.py | 8 +- .../examples/complete/top_wikipedia_sessions.py | 8 - .../examples/cookbook/datastore_wordcount.py | 15 +- .../examples/cookbook/group_with_coder.py | 6 +- .../examples/cookbook/multiple_output_pardo.py | 47 +- .../examples/snippets/snippets_test.py | 33 +- sdks/python/apache_beam/examples/wordcount.py | 32 +- .../apache_beam/examples/wordcount_debugging.py | 4 - sdks/python/apache_beam/internal/gcp/auth.py | 80 +- .../apache_beam/internal/gcp/auth_test.py | 44 - .../apache_beam/internal/gcp/json_value.py | 6 + sdks/python/apache_beam/internal/pickler.py | 8 +- sdks/python/apache_beam/io/avroio_test.py | 14 +- sdks/python/apache_beam/io/concat_source.py | 74 +- .../python/apache_beam/io/concat_source_test.py | 12 +- sdks/python/apache_beam/io/filebasedsource.py | 53 +- .../apache_beam/io/filebasedsource_test.py | 25 +- sdks/python/apache_beam/io/fileio.py | 158 +- sdks/python/apache_beam/io/fileio_test.py | 45 +- sdks/python/apache_beam/io/filesystem.py | 1 + sdks/python/apache_beam/io/filesystems_util.py | 10 +- sdks/python/apache_beam/io/gcp/bigquery.py | 32 +- .../io/gcp/datastore/v1/datastoreio.py | 24 +- .../io/gcp/datastore/v1/datastoreio_test.py | 4 +- .../apache_beam/io/gcp/datastore/v1/helper.py | 16 +- .../io/gcp/datastore/v1/query_splitter.py | 2 +- sdks/python/apache_beam/io/gcp/gcsfilesystem.py | 7 +- .../io/gcp/tests/bigquery_matcher.py | 3 +- sdks/python/apache_beam/io/iobase.py | 7 +- sdks/python/apache_beam/io/localfilesystem.py | 3 +- sdks/python/apache_beam/io/range_trackers.py | 19 +- sdks/python/apache_beam/io/source_test_utils.py | 79 +- .../apache_beam/io/source_test_utils_test.py | 20 +- sdks/python/apache_beam/io/textio.py | 15 +- sdks/python/apache_beam/io/textio_test.py | 18 +- sdks/python/apache_beam/metrics/cells.py | 28 +- sdks/python/apache_beam/metrics/execution.py | 3 +- sdks/python/apache_beam/metrics/metric.py | 9 +- sdks/python/apache_beam/pvalue.py | 26 +- sdks/python/apache_beam/runners/common.pxd | 2 +- sdks/python/apache_beam/runners/common.py | 15 +- .../runners/dataflow/dataflow_metrics_test.py | 3 +- .../runners/dataflow/dataflow_runner.py | 29 +- .../runners/dataflow/internal/apiclient.py | 9 +- .../runners/dataflow/internal/dependency.py | 6 +- .../runners/dataflow/test_dataflow_runner.py | 4 - .../runners/direct/bundle_factory.py | 14 +- sdks/python/apache_beam/runners/direct/clock.py | 9 +- .../consumer_tracking_pipeline_visitor_test.py | 2 +- .../apache_beam/runners/direct/direct_runner.py | 6 + .../runners/direct/evaluation_context.py | 21 +- .../apache_beam/runners/direct/executor.py | 21 +- .../runners/direct/transform_evaluator.py | 19 +- .../runners/direct/transform_result.py | 45 +- .../runners/direct/watermark_manager.py | 4 +- sdks/python/apache_beam/runners/runner.py | 17 +- .../apache_beam/tests/pipeline_verifiers.py | 7 +- sdks/python/apache_beam/transforms/combiners.py | 62 +- .../apache_beam/transforms/combiners_test.py | 4 +- sdks/python/apache_beam/transforms/core.py | 112 +- .../apache_beam/transforms/create_test.py | 18 +- sdks/python/apache_beam/transforms/display.py | 4 +- .../apache_beam/transforms/display_test.py | 36 + .../python/apache_beam/transforms/ptransform.py | 38 +- .../apache_beam/transforms/ptransform_test.py | 19 +- .../python/apache_beam/transforms/sideinputs.py | 11 +- sdks/python/apache_beam/transforms/trigger.py | 26 +- .../apache_beam/transforms/trigger_test.py | 6 +- sdks/python/apache_beam/typehints/decorators.py | 26 +- .../apache_beam/typehints/trivial_inference.py | 26 +- .../typehints/trivial_inference_test.py | 3 +- sdks/python/apache_beam/typehints/typecheck.py | 11 +- sdks/python/apache_beam/typehints/typehints.py | 66 +- .../apache_beam/typehints/typehints_test.py | 7 +- .../apache_beam/utils/annotations_test.py | 2 +- sdks/python/apache_beam/utils/path.py | 3 +- .../apache_beam/utils/pipeline_options.py | 95 +- .../apache_beam/utils/pipeline_options_test.py | 52 +- sdks/python/apache_beam/utils/proto_utils.py | 15 +- sdks/python/apache_beam/utils/retry.py | 14 +- sdks/python/apache_beam/utils/timestamp.py | 6 +- sdks/python/apache_beam/utils/value_provider.py | 103 ++ .../apache_beam/utils/value_provider_test.py | 145 ++ sdks/python/apache_beam/utils/windowed_value.py | 17 +- sdks/python/run_pylint.sh | 2 +- sdks/python/tox.ini | 18 +- 694 files changed, 32745 insertions(+), 32281 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/beam/blob/aa07a1d4/pom.xml ----------------------------------------------------------------------
