Merge remote-tracking branch 'origin/master' into python-sdk
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/77742979 Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/77742979 Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/77742979 Branch: refs/heads/python-sdk Commit: 77742979463d0590969b186cee2af017a2bb1037 Parents: 3b4fd5c 57d9bbd Author: Ahmet Altay <al...@google.com> Authored: Wed Dec 21 14:22:06 2016 -0800 Committer: Ahmet Altay <al...@google.com> Committed: Wed Dec 21 14:22:06 2016 -0800 ---------------------------------------------------------------------- .gitignore | 4 + .jenkins/common_job_properties.groovy | 173 ++++++ ...job_beam_PostCommit_Java_MavenInstall.groovy | 42 ++ ...ostCommit_Java_RunnableOnService_Apex.groovy | 41 ++ ...ommit_Java_RunnableOnService_Dataflow.groovy | 39 ++ ...stCommit_Java_RunnableOnService_Flink.groovy | 38 ++ ...ommit_Java_RunnableOnService_Gearpump.groovy | 41 ++ ...stCommit_Java_RunnableOnService_Spark.groovy | 38 ++ .../job_beam_PostCommit_Python_Verify.groovy | 37 ++ .../job_beam_PreCommit_Java_MavenInstall.groovy | 42 ++ .../job_beam_Release_NightlySnapshot.groovy | 46 ++ .jenkins/job_seed.groovy | 47 ++ .travis.yml | 2 +- .travis/README.md | 2 +- README.md | 2 +- examples/java/pom.xml | 20 +- .../apache/beam/examples/WindowedWordCount.java | 177 +++--- .../org/apache/beam/examples/WordCount.java | 2 +- .../examples/common/WriteWindowedFilesDoFn.java | 77 +++ .../beam/examples/complete/AutoComplete.java | 6 +- .../org/apache/beam/examples/complete/README.md | 14 +- .../apache/beam/examples/complete/TfIdf.java | 6 +- .../examples/complete/TopWikipediaSessions.java | 15 +- .../examples/complete/TrafficMaxLaneFlow.java | 4 +- .../beam/examples/complete/TrafficRoutes.java | 4 +- .../examples/cookbook/BigQueryTornadoes.java | 2 +- .../cookbook/CombinePerKeyExamples.java | 2 +- .../beam/examples/cookbook/DeDupExample.java | 96 --- .../beam/examples/cookbook/DistinctExample.java | 96 +++ .../beam/examples/cookbook/FilterExamples.java | 2 +- .../examples/cookbook/MaxPerKeyExamples.java | 2 +- .../org/apache/beam/examples/cookbook/README.md | 16 +- .../beam/examples/cookbook/TriggerExample.java | 4 +- .../beam/examples/WindowedWordCountIT.java | 182 +++++- .../org/apache/beam/examples/WordCountIT.java | 2 +- .../org/apache/beam/examples/WordCountTest.java | 7 +- .../examples/complete/AutoCompleteTest.java | 13 +- .../beam/examples/complete/TfIdfTest.java | 6 +- .../complete/TopWikipediaSessionsTest.java | 7 +- .../examples/cookbook/DistinctExampleTest.java | 9 +- .../examples/cookbook/JoinExamplesTest.java | 6 +- .../examples/cookbook/TriggerExampleTest.java | 6 +- examples/java8/pom.xml | 2 +- .../beam/examples/complete/game/GameStats.java | 2 +- .../examples/complete/game/LeaderBoard.java | 4 +- .../beam/examples/complete/game/UserScore.java | 2 +- .../complete/game/utils/WriteToBigQuery.java | 2 +- .../game/utils/WriteWindowedToBigQuery.java | 7 +- .../examples/MinimalWordCountJava8Test.java | 6 +- .../examples/complete/game/GameStatsTest.java | 7 +- .../complete/game/HourlyTeamScoreTest.java | 5 +- .../examples/complete/game/LeaderBoardTest.java | 11 +- .../examples/complete/game/UserScoreTest.java | 10 +- examples/pom.xml | 2 +- pom.xml | 15 +- runners/apex/pom.xml | 69 ++- .../apache/beam/runners/apex/ApexRunner.java | 111 ++-- .../beam/runners/apex/ApexRunnerResult.java | 50 +- .../beam/runners/apex/ApexYarnLauncher.java | 395 ++++++++++++ .../translation/ApexPipelineTranslator.java | 14 +- .../translation/ParDoBoundMultiTranslator.java | 22 +- .../apex/translation/ParDoBoundTranslator.java | 22 +- .../apex/translation/TranslationContext.java | 10 +- .../apex/translation/WindowBoundTranslator.java | 78 +++ .../operators/ApexGroupByKeyOperator.java | 26 +- .../operators/ApexParDoOperator.java | 29 +- .../apex/translation/utils/NoOpStepContext.java | 3 +- .../beam/runners/apex/ApexYarnLauncherTest.java | 138 +++++ .../FlattenPCollectionTranslatorTest.java | 15 +- .../translation/GroupByKeyTranslatorTest.java | 21 +- .../translation/ParDoBoundTranslatorTest.java | 38 +- .../translation/ReadUnboundTranslatorTest.java | 15 +- runners/core-java/pom.xml | 6 +- .../beam/runners/core/ActiveWindowSet.java | 173 ++++++ .../beam/runners/core/AggregatorFactory.java | 1 - .../apache/beam/runners/core/AssignWindows.java | 46 -- .../beam/runners/core/BaseExecutionContext.java | 176 ++++++ .../apache/beam/runners/core/DoFnRunner.java | 24 +- .../apache/beam/runners/core/DoFnRunners.java | 4 +- .../core/ElementAndRestrictionCoder.java | 8 + .../beam/runners/core/ExecutionContext.java | 102 +++ .../runners/core/GBKIntoKeyedWorkItems.java | 55 -- .../core/GroupAlsoByWindowViaWindowSetDoFn.java | 12 +- .../GroupAlsoByWindowsViaOutputBufferDoFn.java | 55 +- .../core/GroupByKeyViaGroupByKeyOnly.java | 30 +- .../runners/core/InMemoryTimerInternals.java | 302 +++++++++ .../apache/beam/runners/core/KeyedWorkItem.java | 44 ++ .../beam/runners/core/KeyedWorkItemCoder.java | 130 ++++ .../beam/runners/core/KeyedWorkItems.java | 122 ++++ .../core/LateDataDroppingDoFnRunner.java | 34 +- .../runners/core/MergingActiveWindowSet.java | 428 +++++++++++++ .../runners/core/NonMergingActiveWindowSet.java | 91 +++ .../beam/runners/core/PaneInfoTracker.java | 4 + .../runners/core/PerKeyCombineFnRunner.java | 149 +++++ .../runners/core/PerKeyCombineFnRunners.java | 262 ++++++++ .../core/PushbackSideInputDoFnRunner.java | 11 + .../runners/core/ReduceFnContextFactory.java | 26 +- .../beam/runners/core/ReduceFnRunner.java | 496 +++++++++------ .../beam/runners/core/SimpleDoFnRunner.java | 328 +++++++++- .../beam/runners/core/SimpleOldDoFnRunner.java | 27 +- .../beam/runners/core/SplittableParDo.java | 392 +++++++++--- .../core/UnboundedReadFromBoundedSource.java | 2 +- .../apache/beam/runners/core/WatermarkHold.java | 9 +- .../TriggerStateMachineContextFactory.java | 2 +- .../triggers/TriggerStateMachineRunner.java | 14 +- .../core/GroupAlsoByWindowsProperties.java | 590 ++++++++++-------- .../core/InMemoryTimerInternalsTest.java | 197 ++++++ .../runners/core/KeyedWorkItemCoderTest.java | 64 ++ .../core/LateDataDroppingDoFnRunnerTest.java | 3 + .../core/MergingActiveWindowSetTest.java | 262 ++++++++ .../core/PushbackSideInputDoFnRunnerTest.java | 60 +- .../beam/runners/core/ReduceFnRunnerTest.java | 138 +++-- .../beam/runners/core/ReduceFnTester.java | 78 ++- .../beam/runners/core/SimpleDoFnRunnerTest.java | 301 +++++++++ .../runners/core/SimpleOldDoFnRunnerTest.java | 3 +- .../beam/runners/core/SplittableParDoTest.java | 158 ++++- .../UnboundedReadFromBoundedSourceTest.java | 6 +- .../beam/runners/core/WindowMatchers.java | 204 ++++++ .../beam/runners/core/WindowMatchersTest.java | 82 +++ .../triggers/TriggerStateMachineTester.java | 27 +- runners/direct-java/pom.xml | 3 +- .../direct/AbstractModelEnforcement.java | 2 +- .../runners/direct/AggregatorContainer.java | 4 +- .../direct/BoundedReadEvaluatorFactory.java | 8 +- .../beam/runners/direct/CommittedResult.java | 2 +- .../beam/runners/direct/CompletionCallback.java | 2 +- .../direct/ConsumerTrackingPipelineVisitor.java | 173 ------ .../CopyOnAccessInMemoryStateInternals.java | 467 ++++++++++++++ .../runners/direct/DirectExecutionContext.java | 7 +- ...ectGBKIntoKeyedWorkItemsOverrideFactory.java | 42 +- .../apache/beam/runners/direct/DirectGraph.java | 89 +++ .../beam/runners/direct/DirectGraphVisitor.java | 142 +++++ .../beam/runners/direct/DirectGroupByKey.java | 46 +- .../direct/DirectGroupByKeyOverrideFactory.java | 3 +- .../beam/runners/direct/DirectRunner.java | 62 +- .../runners/direct/DirectTimerInternals.java | 14 +- .../runners/direct/DisplayDataValidator.java | 6 +- .../runners/direct/DoFnLifecycleManager.java | 4 +- ...ecycleManagerRemovingTransformEvaluator.java | 2 +- .../runners/direct/EmptyTransformEvaluator.java | 4 +- .../beam/runners/direct/EvaluationContext.java | 105 ++-- .../direct/ExecutorServiceParallelExecutor.java | 29 +- .../runners/direct/FlattenEvaluatorFactory.java | 10 +- .../runners/direct/ForwardingPTransform.java | 6 +- .../GroupAlsoByWindowEvaluatorFactory.java | 13 +- .../direct/GroupByKeyOnlyEvaluatorFactory.java | 6 +- .../ImmutabilityCheckingBundleFactory.java | 21 +- .../direct/ImmutabilityEnforcementFactory.java | 2 +- .../direct/KeyedPValueTrackingVisitor.java | 64 +- .../beam/runners/direct/ModelEnforcement.java | 2 +- .../direct/PTransformOverrideFactory.java | 35 -- .../beam/runners/direct/ParDoEvaluator.java | 45 +- .../runners/direct/ParDoEvaluatorFactory.java | 97 ++- .../direct/ParDoMultiOverrideFactory.java | 193 +++++- .../ParDoSingleViaMultiOverrideFactory.java | 21 +- .../direct/PassthroughTransformEvaluator.java | 4 +- ...littableProcessElementsEvaluatorFactory.java | 145 +++++ .../direct/StatefulParDoEvaluatorFactory.java | 257 ++++++++ .../runners/direct/StepTransformResult.java | 39 +- .../direct/TestStreamEvaluatorFactory.java | 9 +- .../beam/runners/direct/TransformEvaluator.java | 2 +- .../direct/TransformEvaluatorRegistry.java | 7 + .../beam/runners/direct/TransformExecutor.java | 4 +- .../beam/runners/direct/TransformResult.java | 17 +- .../direct/UnboundedReadEvaluatorFactory.java | 3 +- .../runners/direct/ViewEvaluatorFactory.java | 10 +- .../direct/WatermarkCallbackExecutor.java | 34 + .../beam/runners/direct/WatermarkManager.java | 139 +++-- .../runners/direct/WindowEvaluatorFactory.java | 6 +- .../direct/WriteWithShardingFactory.java | 10 +- .../runners/direct/AggregatorContainerTest.java | 2 +- .../direct/BoundedReadEvaluatorFactoryTest.java | 47 +- .../direct/CloningBundleFactoryTest.java | 8 +- .../runners/direct/CommittedResultTest.java | 8 +- .../ConsumerTrackingPipelineVisitorTest.java | 287 --------- .../CopyOnAccessInMemoryStateInternalsTest.java | 565 +++++++++++++++++ .../runners/direct/DirectGraphVisitorTest.java | 265 ++++++++ .../beam/runners/direct/DirectGraphs.java | 35 ++ .../beam/runners/direct/DirectRunnerTest.java | 36 -- ...leManagerRemovingTransformEvaluatorTest.java | 4 +- .../runners/direct/EvaluationContextTest.java | 132 ++-- .../direct/FlattenEvaluatorFactoryTest.java | 27 +- .../direct/ForwardingPTransformTest.java | 4 +- .../direct/GroupByKeyEvaluatorFactoryTest.java | 11 +- .../GroupByKeyOnlyEvaluatorFactoryTest.java | 12 +- .../ImmutabilityCheckingBundleFactoryTest.java | 10 +- .../ImmutabilityEnforcementFactoryTest.java | 11 +- .../direct/ImmutableListBundleFactoryTest.java | 14 +- .../direct/KeyedPValueTrackingVisitorTest.java | 137 ++-- .../beam/runners/direct/ParDoEvaluatorTest.java | 11 +- .../runners/direct/SideInputContainerTest.java | 5 +- .../beam/runners/direct/SplittableDoFnTest.java | 231 ------- .../StatefulParDoEvaluatorFactoryTest.java | 323 ++++++++++ .../runners/direct/StepTransformResultTest.java | 32 +- .../direct/TestStreamEvaluatorFactoryTest.java | 29 +- .../runners/direct/TransformExecutorTest.java | 228 +++---- .../UnboundedReadEvaluatorFactoryTest.java | 53 +- .../direct/ViewEvaluatorFactoryTest.java | 9 +- .../direct/WatermarkCallbackExecutorTest.java | 11 +- .../runners/direct/WatermarkManagerTest.java | 260 ++++---- .../direct/WindowEvaluatorFactoryTest.java | 21 +- .../direct/WriteWithShardingFactoryTest.java | 18 +- runners/flink/examples/pom.xml | 14 +- .../beam/runners/flink/examples/TFIDF.java | 6 +- .../beam/runners/flink/examples/WordCount.java | 2 +- .../flink/examples/streaming/AutoComplete.java | 6 +- runners/flink/pom.xml | 2 +- runners/flink/runner/pom.xml | 15 +- .../FlinkPipelineExecutionEnvironment.java | 12 + .../runners/flink/FlinkPipelineOptions.java | 5 + .../apache/beam/runners/flink/FlinkRunner.java | 36 +- .../FlinkBatchPipelineTranslator.java | 18 +- .../FlinkBatchTransformTranslators.java | 55 +- .../FlinkStreamingPipelineTranslator.java | 23 +- .../FlinkStreamingTransformTranslators.java | 140 ++--- .../PipelineTranslationOptimizer.java | 10 +- .../functions/FlinkDoFnFunction.java | 10 +- .../FlinkMergingNonShuffleReduceFunction.java | 4 +- .../FlinkMergingPartialReduceFunction.java | 4 +- .../functions/FlinkMergingReduceFunction.java | 4 +- .../functions/FlinkMultiOutputDoFnFunction.java | 10 +- .../functions/FlinkPartialReduceFunction.java | 4 +- .../functions/FlinkProcessContextBase.java | 29 +- .../functions/FlinkReduceFunction.java | 4 +- .../wrappers/streaming/DoFnOperator.java | 48 +- .../streaming/SingletonKeyedWorkItem.java | 2 +- .../streaming/SingletonKeyedWorkItemCoder.java | 4 +- .../wrappers/streaming/WindowDoFnOperator.java | 25 +- .../wrappers/streaming/WorkItemKeySelector.java | 2 +- .../beam/runners/flink/PipelineOptionsTest.java | 6 +- .../flink/streaming/DoFnOperatorTest.java | 13 +- .../flink/streaming/GroupByNullKeyTest.java | 18 +- .../streaming/TopWikipediaSessionsITCase.java | 10 +- runners/google-cloud-dataflow-java/pom.xml | 14 +- .../beam/runners/dataflow/DataflowClient.java | 140 +++++ .../runners/dataflow/DataflowPipelineJob.java | 31 +- .../dataflow/DataflowPipelineTranslator.java | 36 +- .../beam/runners/dataflow/DataflowRunner.java | 210 ++++--- .../runners/dataflow/DataflowRunnerInfo.java | 92 +++ .../dataflow/internal/AssignWindows.java | 10 +- .../DataflowUnboundedReadFromBoundedSource.java | 2 +- .../options/DataflowPipelineOptions.java | 19 +- .../DataflowPipelineWorkerPoolOptions.java | 57 +- .../options/DataflowProfilingOptions.java | 8 +- .../dataflow/testing/TestDataflowRunner.java | 35 +- .../dataflow/util/DataflowTemplateJob.java | 2 +- .../runners/dataflow/util/MonitoringUtil.java | 22 +- .../beam/runners/dataflow/dataflow.properties | 23 + .../dataflow/DataflowPipelineJobTest.java | 49 +- .../DataflowPipelineTranslatorTest.java | 63 +- .../dataflow/DataflowRunnerInfoTest.java | 51 ++ .../runners/dataflow/DataflowRunnerTest.java | 333 ++++++---- .../dataflow/RecordingPipelineVisitor.java | 6 +- .../options/DataflowPipelineOptionsTest.java | 20 +- .../options/DataflowProfilingOptionsTest.java | 6 +- .../testing/TestDataflowRunnerTest.java | 52 +- .../transforms/DataflowGroupByKeyTest.java | 18 +- .../dataflow/transforms/DataflowViewTest.java | 20 +- .../dataflow/util/MonitoringUtilTest.java | 21 +- runners/pom.xml | 2 +- runners/spark/README.md | 59 +- runners/spark/pom.xml | 60 +- .../beam/runners/spark/EvaluationResult.java | 67 -- .../runners/spark/SparkPipelineOptions.java | 15 + .../beam/runners/spark/SparkPipelineResult.java | 193 ++++++ .../apache/beam/runners/spark/SparkRunner.java | 213 +++++-- .../beam/runners/spark/TestSparkRunner.java | 91 ++- .../spark/aggregators/AccumulatorSingleton.java | 8 +- .../spark/aggregators/SparkAggregators.java | 126 ++++ .../metrics/WithNamedAggregatorsSupport.java | 4 +- .../coders/BeamSparkRunnerRegistrator.java | 60 +- .../beam/runners/spark/examples/WordCount.java | 15 +- .../apache/beam/runners/spark/io/ConsoleIO.java | 2 +- .../beam/runners/spark/io/CreateStream.java | 2 +- .../beam/runners/spark/io/MicrobatchSource.java | 20 +- .../beam/runners/spark/io/SourceDStream.java | 22 +- .../runners/spark/io/SparkUnboundedSource.java | 17 +- .../beam/runners/spark/io/hadoop/HadoopIO.java | 4 +- .../spark/stateful/StateSpecFunctions.java | 2 +- .../spark/translation/BoundedDataset.java | 5 +- .../beam/runners/spark/translation/Dataset.java | 2 +- .../runners/spark/translation/DoFnFunction.java | 110 ++-- .../spark/translation/EvaluationContext.java | 137 +--- .../translation/GroupCombineFunctions.java | 23 +- .../spark/translation/MultiDoFnFunction.java | 135 ++-- .../spark/translation/SparkAssignWindowFn.java | 69 +++ .../spark/translation/SparkContextFactory.java | 2 +- .../translation/SparkGroupAlsoByWindowFn.java | 214 +++++++ .../spark/translation/SparkProcessContext.java | 390 +++--------- .../spark/translation/SparkRuntimeContext.java | 62 +- .../translation/StorageLevelPTransform.java | 43 ++ .../spark/translation/TransformTranslator.java | 91 ++- .../spark/translation/TranslationUtils.java | 33 + .../spark/translation/WindowingHelpers.java | 3 +- .../SparkRunnerStreamingContextFactory.java | 2 +- .../streaming/StreamingTransformTranslator.java | 47 +- .../translation/streaming/UnboundedDataset.java | 13 +- .../util/SinglePrimitiveOutputPTransform.java | 2 +- .../beam/runners/spark/ForceStreamingTest.java | 123 ++++ .../runners/spark/ProvidedSparkContextTest.java | 6 +- .../runners/spark/SparkPipelineStateTest.java | 217 +++++++ .../spark/aggregators/ClearAggregatorsRule.java | 37 ++ .../metrics/sink/ClearAggregatorsRule.java | 33 - .../metrics/sink/NamedAggregatorsTest.java | 3 +- .../coders/BeamSparkRunnerRegistratorTest.java | 57 ++ .../beam/runners/spark/io/AvroPipelineTest.java | 2 +- .../beam/runners/spark/io/NumShardsTest.java | 2 +- .../io/hadoop/HadoopFileFormatPipelineTest.java | 2 +- .../spark/translation/SideEffectsTest.java | 59 -- .../spark/translation/StorageLevelTest.java | 56 ++ .../streaming/EmptyStreamAssertionTest.java | 6 + .../streaming/FlattenStreamingTest.java | 2 + .../streaming/KafkaStreamingTest.java | 34 +- .../ResumeFromCheckpointStreamingTest.java | 32 +- .../streaming/SimpleStreamingWordCountTest.java | 1 + .../streaming/utils/PAssertStreaming.java | 31 +- .../SparkTestPipelineOptionsForStreaming.java | 6 - sdks/java/build-tools/pom.xml | 2 +- .../src/main/resources/beam/findbugs-filter.xml | 68 +- sdks/java/core/pom.xml | 31 +- .../beam/sdk/AggregatorPipelineExtractor.java | 9 +- .../main/java/org/apache/beam/sdk/Pipeline.java | 151 +---- .../beam/sdk/annotations/Experimental.java | 5 +- .../apache/beam/sdk/coders/CoderRegistry.java | 2 + .../java/org/apache/beam/sdk/io/AvroIO.java | 4 +- .../sdk/io/BoundedReadFromUnboundedSource.java | 16 +- .../org/apache/beam/sdk/io/CountingInput.java | 52 +- .../org/apache/beam/sdk/io/CountingSource.java | 17 +- .../org/apache/beam/sdk/io/FileBasedSink.java | 95 ++- .../org/apache/beam/sdk/io/FileBasedSource.java | 12 +- .../java/org/apache/beam/sdk/io/FileSystem.java | 29 + .../apache/beam/sdk/io/FileSystemRegistrar.java | 49 ++ .../org/apache/beam/sdk/io/FileSystems.java | 155 +++++ .../org/apache/beam/sdk/io/LocalFileSystem.java | 27 + .../beam/sdk/io/LocalFileSystemRegistrar.java | 41 ++ .../java/org/apache/beam/sdk/io/PubsubIO.java | 180 +++++- .../apache/beam/sdk/io/PubsubUnboundedSink.java | 27 +- .../beam/sdk/io/PubsubUnboundedSource.java | 48 +- .../main/java/org/apache/beam/sdk/io/Read.java | 4 +- .../java/org/apache/beam/sdk/io/TextIO.java | 32 +- .../main/java/org/apache/beam/sdk/io/Write.java | 2 +- .../java/org/apache/beam/sdk/io/XmlSink.java | 4 +- .../apache/beam/sdk/io/range/ByteKeyRange.java | 4 +- .../beam/sdk/io/range/ByteKeyRangeTracker.java | 8 +- .../apache/beam/sdk/metrics/CounterCell.java | 10 +- .../beam/sdk/metrics/DistributionCell.java | 14 +- .../org/apache/beam/sdk/metrics/MetricCell.java | 2 +- .../org/apache/beam/sdk/metrics/MetricName.java | 3 +- .../org/apache/beam/sdk/metrics/Metrics.java | 5 +- .../beam/sdk/metrics/MetricsEnvironment.java | 8 +- .../org/apache/beam/sdk/options/GcpOptions.java | 19 +- .../beam/sdk/options/PipelineOptions.java | 7 + .../sdk/options/PipelineOptionsFactory.java | 111 +++- .../sdk/options/ProxyInvocationHandler.java | 26 + .../beam/sdk/options/StreamingOptions.java | 2 +- .../apache/beam/sdk/options/ValueProvider.java | 29 +- .../sdk/runners/PTransformOverrideFactory.java | 41 ++ .../apache/beam/sdk/runners/PipelineRunner.java | 2 +- .../beam/sdk/runners/TransformHierarchy.java | 345 +++++++++-- .../beam/sdk/runners/TransformTreeNode.java | 257 -------- .../beam/sdk/testing/BigqueryMatcher.java | 48 +- .../beam/sdk/testing/FileChecksumMatcher.java | 124 ++-- .../apache/beam/sdk/testing/GatherAllPanes.java | 88 +++ .../org/apache/beam/sdk/testing/PAssert.java | 87 +-- .../apache/beam/sdk/testing/PaneExtractors.java | 55 +- .../apache/beam/sdk/testing/TestPipeline.java | 225 ++++++- .../org/apache/beam/sdk/testing/TestStream.java | 2 +- .../apache/beam/sdk/testing/UsesMetrics.java | 24 + .../beam/sdk/testing/UsesSplittableParDo.java | 25 + .../apache/beam/sdk/testing/UsesTestStream.java | 24 + .../beam/sdk/testing/UsesTimersInParDo.java | 25 + .../beam/sdk/testing/ValueInSingleWindow.java | 134 ++++ .../apache/beam/sdk/transforms/Aggregator.java | 19 - .../sdk/transforms/AggregatorRetriever.java | 2 +- .../beam/sdk/transforms/AppliedPTransform.java | 75 +-- .../beam/sdk/transforms/ApproximateUnique.java | 4 +- .../org/apache/beam/sdk/transforms/Combine.java | 51 +- .../beam/sdk/transforms/CombineWithContext.java | 2 +- .../org/apache/beam/sdk/transforms/Count.java | 2 +- .../org/apache/beam/sdk/transforms/Create.java | 4 +- .../apache/beam/sdk/transforms/Distinct.java | 4 +- .../org/apache/beam/sdk/transforms/DoFn.java | 38 +- .../beam/sdk/transforms/DoFnAdapters.java | 14 + .../apache/beam/sdk/transforms/DoFnTester.java | 387 ++++++------ .../org/apache/beam/sdk/transforms/Filter.java | 2 +- .../beam/sdk/transforms/FlatMapElements.java | 2 +- .../org/apache/beam/sdk/transforms/Flatten.java | 4 +- .../apache/beam/sdk/transforms/GroupByKey.java | 2 +- .../org/apache/beam/sdk/transforms/Keys.java | 2 +- .../org/apache/beam/sdk/transforms/KvSwap.java | 2 +- .../org/apache/beam/sdk/transforms/Latest.java | 4 +- .../apache/beam/sdk/transforms/MapElements.java | 2 +- .../org/apache/beam/sdk/transforms/OldDoFn.java | 8 +- .../apache/beam/sdk/transforms/PTransform.java | 4 +- .../org/apache/beam/sdk/transforms/ParDo.java | 201 +----- .../apache/beam/sdk/transforms/Partition.java | 2 +- .../org/apache/beam/sdk/transforms/Regex.java | 14 +- .../org/apache/beam/sdk/transforms/Sample.java | 2 +- .../org/apache/beam/sdk/transforms/Values.java | 2 +- .../org/apache/beam/sdk/transforms/View.java | 16 +- .../apache/beam/sdk/transforms/WithKeys.java | 2 +- .../beam/sdk/transforms/WithTimestamps.java | 2 +- .../sdk/transforms/display/DisplayData.java | 22 + .../beam/sdk/transforms/join/CoGroupByKey.java | 2 +- .../transforms/join/KeyedPCollectionTuple.java | 9 +- .../reflect/ByteBuddyDoFnInvokerFactory.java | 39 +- .../reflect/ByteBuddyOnTimerInvokerFactory.java | 10 +- .../sdk/transforms/reflect/DoFnInvoker.java | 11 + .../sdk/transforms/reflect/DoFnInvokers.java | 16 +- .../sdk/transforms/reflect/DoFnSignature.java | 34 +- .../sdk/transforms/reflect/DoFnSignatures.java | 96 ++- .../sdk/transforms/windowing/BoundedWindow.java | 31 + .../beam/sdk/transforms/windowing/Window.java | 13 +- .../apache/beam/sdk/util/ActiveWindowSet.java | 173 ------ .../org/apache/beam/sdk/util/ApiSurface.java | 52 +- .../beam/sdk/util/BaseExecutionContext.java | 174 ------ .../apache/beam/sdk/util/ExecutionContext.java | 100 --- .../beam/sdk/util/ExplicitShardedFile.java | 120 ++++ .../apache/beam/sdk/util/GatherAllPanes.java | 86 --- .../apache/beam/sdk/util/GcsPathValidator.java | 3 +- .../java/org/apache/beam/sdk/util/GcsUtil.java | 107 +++- .../apache/beam/sdk/util/IdentityWindowFn.java | 2 +- .../org/apache/beam/sdk/util/KeyedWorkItem.java | 43 -- .../beam/sdk/util/KeyedWorkItemCoder.java | 128 ---- .../apache/beam/sdk/util/KeyedWorkItems.java | 121 ---- .../beam/sdk/util/MergingActiveWindowSet.java | 428 ------------- .../sdk/util/NonMergingActiveWindowSet.java | 91 --- .../beam/sdk/util/NumberedShardedFile.java | 220 +++++++ .../beam/sdk/util/PerKeyCombineFnRunner.java | 148 ----- .../beam/sdk/util/PerKeyCombineFnRunners.java | 258 -------- .../org/apache/beam/sdk/util/PropertyNames.java | 2 + .../org/apache/beam/sdk/util/ReleaseInfo.java | 4 - .../org/apache/beam/sdk/util/Reshuffle.java | 2 +- .../org/apache/beam/sdk/util/ShardedFile.java | 42 ++ .../apache/beam/sdk/util/TimerInternals.java | 120 ++-- .../beam/sdk/util/WindowingInternals.java | 10 - .../CopyOnAccessInMemoryStateInternals.java | 453 -------------- .../sdk/util/state/InMemoryStateInternals.java | 33 +- .../sdk/util/state/InMemoryTimerInternals.java | 247 -------- .../beam/sdk/util/state/StateInternals.java | 2 +- .../beam/sdk/util/state/TimerCallback.java | 35 -- .../sdk/util/state/TimerInternalsFactory.java | 36 ++ .../java/org/apache/beam/sdk/values/PBegin.java | 4 +- .../org/apache/beam/sdk/values/PCollection.java | 4 +- .../apache/beam/sdk/values/PCollectionList.java | 65 +- .../beam/sdk/values/PCollectionTuple.java | 30 +- .../java/org/apache/beam/sdk/values/PDone.java | 4 +- .../java/org/apache/beam/sdk/values/PInput.java | 4 +- .../org/apache/beam/sdk/values/POutput.java | 4 +- .../java/org/apache/beam/sdk/values/PValue.java | 10 + .../org/apache/beam/sdk/values/PValueBase.java | 11 +- .../apache/beam/sdk/values/TaggedPValue.java | 42 ++ .../org/apache/beam/sdk/values/TypedPValue.java | 2 +- .../sdk/AggregatorPipelineExtractorTest.java | 28 +- .../java/org/apache/beam/sdk/PipelineTest.java | 43 +- .../org/apache/beam/sdk/WindowMatchers.java | 204 ------ .../org/apache/beam/sdk/WindowMatchersTest.java | 82 --- .../apache/beam/sdk/coders/AvroCoderTest.java | 11 +- .../beam/sdk/coders/CoderRegistryTest.java | 10 +- .../beam/sdk/coders/SerializableCoderTest.java | 7 +- .../java/org/apache/beam/sdk/io/AvroIOTest.java | 16 +- .../io/BoundedReadFromUnboundedSourceTest.java | 6 +- .../beam/sdk/io/CompressedSourceTest.java | 12 +- .../apache/beam/sdk/io/CountingInputTest.java | 69 ++- .../apache/beam/sdk/io/CountingSourceTest.java | 21 +- .../apache/beam/sdk/io/FileBasedSinkTest.java | 11 +- .../apache/beam/sdk/io/FileBasedSourceTest.java | 30 +- .../org/apache/beam/sdk/io/FileSystemsTest.java | 104 ++++ .../sdk/io/LocalFileSystemRegistrarTest.java | 44 ++ .../org/apache/beam/sdk/io/PubsubIOTest.java | 43 +- .../beam/sdk/io/PubsubUnboundedSinkTest.java | 30 +- .../beam/sdk/io/PubsubUnboundedSourceTest.java | 26 +- .../java/org/apache/beam/sdk/io/TextIOTest.java | 44 +- .../java/org/apache/beam/sdk/io/WriteTest.java | 4 +- .../org/apache/beam/sdk/io/XmlSinkTest.java | 10 +- .../org/apache/beam/sdk/io/XmlSourceTest.java | 10 +- .../apache/beam/sdk/metrics/MetricMatchers.java | 47 +- .../apache/beam/sdk/metrics/MetricsTest.java | 63 +- .../apache/beam/sdk/options/GcpOptionsTest.java | 32 +- .../sdk/options/PipelineOptionsFactoryTest.java | 222 ++++++- .../beam/sdk/options/PipelineOptionsTest.java | 24 + .../sdk/options/ProxyInvocationHandlerTest.java | 5 +- .../beam/sdk/options/ValueProviderTest.java | 39 ++ .../sdk/runners/TransformHierarchyTest.java | 208 +++++-- .../beam/sdk/runners/TransformTreeTest.java | 27 +- .../beam/sdk/testing/BigqueryMatcherTest.java | 82 +-- .../sdk/testing/FileChecksumMatcherTest.java | 66 +- .../beam/sdk/testing/GatherAllPanesTest.java | 141 +++++ .../apache/beam/sdk/testing/PAssertTest.java | 32 +- .../beam/sdk/testing/PaneExtractorsTest.java | 133 ++-- .../beam/sdk/testing/TestPipelineTest.java | 186 +++++- .../apache/beam/sdk/testing/TestStreamTest.java | 19 +- .../testing/ValueInSingleWindowCoderTest.java | 51 ++ .../transforms/ApproximateQuantilesTest.java | 12 +- .../sdk/transforms/ApproximateUniqueTest.java | 6 +- .../beam/sdk/transforms/CombineFnsTest.java | 5 +- .../apache/beam/sdk/transforms/CombineTest.java | 25 +- .../apache/beam/sdk/transforms/CountTest.java | 13 +- .../apache/beam/sdk/transforms/CreateTest.java | 27 +- .../beam/sdk/transforms/DistinctTest.java | 12 +- .../apache/beam/sdk/transforms/DoFnTest.java | 4 +- .../beam/sdk/transforms/DoFnTesterTest.java | 46 +- .../apache/beam/sdk/transforms/FilterTest.java | 18 +- .../sdk/transforms/FlatMapElementsTest.java | 12 +- .../apache/beam/sdk/transforms/FlattenTest.java | 35 +- .../beam/sdk/transforms/GroupByKeyTest.java | 34 +- .../apache/beam/sdk/transforms/KeysTest.java | 9 +- .../apache/beam/sdk/transforms/KvSwapTest.java | 9 +- .../apache/beam/sdk/transforms/LatestTest.java | 12 +- .../beam/sdk/transforms/MapElementsTest.java | 16 +- .../apache/beam/sdk/transforms/NoOpOldDoFn.java | 72 --- .../apache/beam/sdk/transforms/OldDoFnTest.java | 125 ++-- .../beam/sdk/transforms/PTransformTest.java | 2 +- .../beam/sdk/transforms/ParDoLifecycleTest.java | 45 +- .../apache/beam/sdk/transforms/ParDoTest.java | 617 ++++++++++++------- .../beam/sdk/transforms/PartitionTest.java | 8 +- .../apache/beam/sdk/transforms/RegexTest.java | 25 +- .../apache/beam/sdk/transforms/SampleTest.java | 34 +- .../beam/sdk/transforms/SplittableDoFnTest.java | 453 ++++++++++++++ .../org/apache/beam/sdk/transforms/TopTest.java | 15 +- .../apache/beam/sdk/transforms/ValuesTest.java | 7 +- .../apache/beam/sdk/transforms/ViewTest.java | 88 +-- .../beam/sdk/transforms/WithKeysTest.java | 8 +- .../beam/sdk/transforms/WithTimestampsTest.java | 9 +- .../display/DisplayDataEvaluator.java | 8 +- .../display/DisplayDataEvaluatorTest.java | 2 +- .../display/DisplayDataMatchersTest.java | 6 +- .../sdk/transforms/display/DisplayDataTest.java | 90 ++- .../sdk/transforms/join/CoGroupByKeyTest.java | 11 +- .../DoFnSignaturesSplittableDoFnTest.java | 3 +- .../transforms/reflect/DoFnSignaturesTest.java | 47 ++ .../sdk/transforms/windowing/WindowTest.java | 22 +- .../sdk/transforms/windowing/WindowingTest.java | 13 +- .../beam/sdk/util/GatherAllPanesTest.java | 143 ----- .../beam/sdk/util/GcsPathValidatorTest.java | 15 +- .../org/apache/beam/sdk/util/GcsUtilTest.java | 85 ++- .../beam/sdk/util/KeyedWorkItemCoderTest.java | 62 -- .../sdk/util/MergingActiveWindowSetTest.java | 262 -------- .../beam/sdk/util/NumberedShardedFileTest.java | 181 ++++++ .../apache/beam/sdk/util/ReleaseInfoTest.java | 45 ++ .../org/apache/beam/sdk/util/ReshuffleTest.java | 11 +- .../apache/beam/sdk/util/StringUtilsTest.java | 6 +- .../beam/sdk/util/TimerInternalsTest.java | 4 +- .../CopyOnAccessInMemoryStateInternalsTest.java | 552 ----------------- .../util/state/InMemoryTimerInternalsTest.java | 116 ---- .../beam/sdk/values/PCollectionListTest.java | 117 ++++ .../beam/sdk/values/PCollectionTupleTest.java | 80 ++- .../org/apache/beam/sdk/values/PDoneTest.java | 13 +- .../apache/beam/sdk/values/TypedPValueTest.java | 10 +- sdks/java/extensions/join-library/pom.xml | 2 +- .../extensions/joinlibrary/InnerJoinTest.java | 9 +- .../joinlibrary/OuterLeftJoinTest.java | 10 +- .../joinlibrary/OuterRightJoinTest.java | 10 +- sdks/java/extensions/pom.xml | 2 +- sdks/java/extensions/sorter/README.md | 2 +- sdks/java/extensions/sorter/pom.xml | 10 +- .../sorter/BufferedExternalSorter.java | 12 +- .../sdk/extensions/sorter/ExternalSorter.java | 21 +- .../sdk/extensions/sorter/InMemorySorter.java | 26 +- .../beam/sdk/extensions/sorter/SortValues.java | 2 +- .../sorter/BufferedExternalSorterTest.java | 74 ++- .../extensions/sorter/ExternalSorterTest.java | 69 ++- .../extensions/sorter/InMemorySorterTest.java | 8 + .../sdk/extensions/sorter/SortValuesTest.java | 7 +- sdks/java/io/google-cloud-platform/pom.xml | 8 +- .../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 453 ++++++++++---- .../sdk/io/gcp/bigquery/BigQueryServices.java | 7 +- .../io/gcp/bigquery/BigQueryServicesImpl.java | 121 +++- .../io/gcp/bigquery/BigQueryTableInserter.java | 217 ------- .../beam/sdk/io/gcp/bigtable/BigtableIO.java | 28 +- .../io/gcp/bigtable/BigtableServiceImpl.java | 6 +- .../beam/sdk/io/gcp/datastore/DatastoreV1.java | 4 +- .../beam/sdk/io/gcp/storage/GcsFileSystem.java | 34 + .../io/gcp/storage/GcsFileSystemRegistrar.java | 42 ++ .../beam/sdk/io/gcp/storage/package-info.java | 21 + .../apache/beam/sdk/io/gcp/ApiSurfaceTest.java | 2 + .../sdk/io/gcp/bigquery/BigQueryIOTest.java | 568 +++++++++++++---- .../gcp/bigquery/BigQueryServicesImplTest.java | 139 ++++- .../gcp/bigquery/BigQueryTableInserterTest.java | 245 -------- .../sdk/io/gcp/bigquery/BigQueryUtilTest.java | 50 +- .../sdk/io/gcp/bigtable/BigtableIOTest.java | 13 +- .../gcp/storage/GcsFileSystemRegistrarTest.java | 51 ++ sdks/java/io/hdfs/pom.xml | 8 +- .../beam/sdk/io/hdfs/HadoopFileSystem.java | 29 + .../sdk/io/hdfs/HadoopFileSystemRegistrar.java | 42 ++ .../io/hdfs/HadoopFileSystemRegistrarTest.java | 52 ++ sdks/java/io/jdbc/pom.xml | 2 +- .../org/apache/beam/sdk/io/jdbc/JdbcIO.java | 56 +- .../org/apache/beam/sdk/io/jdbc/JdbcIOTest.java | 38 +- sdks/java/io/jms/pom.xml | 22 +- .../java/org/apache/beam/sdk/io/jms/JmsIO.java | 342 ++++++---- .../org/apache/beam/sdk/io/jms/JmsRecord.java | 8 +- .../org/apache/beam/sdk/io/jms/JmsIOTest.java | 9 +- sdks/java/io/kafka/pom.xml | 2 +- .../org/apache/beam/sdk/io/kafka/KafkaIO.java | 20 +- .../apache/beam/sdk/io/kafka/KafkaIOTest.java | 25 +- sdks/java/io/kinesis/pom.xml | 2 +- .../sdk/io/kinesis/KinesisMockReadTest.java | 7 +- .../beam/sdk/io/kinesis/KinesisReaderIT.java | 6 +- sdks/java/io/mongodb/pom.xml | 2 +- .../beam/sdk/io/mongodb/MongoDbGridFSIO.java | 4 +- .../apache/beam/sdk/io/mongodb/MongoDbIO.java | 22 +- .../sdk/io/mongodb/MongoDBGridFSIOTest.java | 17 +- .../beam/sdk/io/mongodb/MongoDbIOTest.java | 15 +- sdks/java/io/pom.xml | 2 +- sdks/java/java8tests/pom.xml | 2 +- .../beam/sdk/transforms/CombineJava8Test.java | 8 +- .../beam/sdk/transforms/DistinctJava8Test.java | 5 +- .../beam/sdk/transforms/FilterJava8Test.java | 9 +- .../transforms/FlatMapElementsJava8Test.java | 7 +- .../sdk/transforms/MapElementsJava8Test.java | 9 +- .../beam/sdk/transforms/PartitionJava8Test.java | 7 +- .../beam/sdk/transforms/WithKeysJava8Test.java | 6 +- .../sdk/transforms/WithTimestampsJava8Test.java | 14 +- .../examples-java8/generate-sources.sh | 82 +++ .../maven-archetypes/examples-java8/pom.xml | 182 ++++++ .../META-INF/maven/archetype-metadata.xml | 39 ++ .../main/resources/archetype-resources/pom.xml | 295 +++++++++ .../projects/basic/archetype.properties | 21 + .../src/test/resources/projects/basic/goal.txt | 1 + .../examples/generate-sources.sh | 68 ++ sdks/java/maven-archetypes/examples/pom.xml | 32 +- .../main/resources/archetype-resources/pom.xml | 168 ++++- .../src/main/java/DebuggingWordCount.java | 164 ----- .../src/main/java/MinimalWordCount.java | 118 ---- .../src/main/java/WindowedWordCount.java | 229 ------- .../src/main/java/WordCount.java | 186 ------ .../common/ExampleBigQueryTableOptions.java | 55 -- .../src/main/java/common/ExampleOptions.java | 37 -- ...xamplePubsubTopicAndSubscriptionOptions.java | 45 -- .../java/common/ExamplePubsubTopicOptions.java | 45 -- .../src/main/java/common/ExampleUtils.java | 352 ----------- .../src/test/java/DebuggingWordCountTest.java | 52 -- .../src/test/java/WordCountTest.java | 85 --- sdks/java/maven-archetypes/pom.xml | 40 +- sdks/java/maven-archetypes/starter/pom.xml | 9 +- .../main/resources/archetype-resources/pom.xml | 20 +- .../resources/projects/basic/reference/pom.xml | 20 +- .../update-examples-archetype.sh | 59 -- sdks/java/pom.xml | 2 +- sdks/pom.xml | 2 +- 641 files changed, 23199 insertions(+), 14088 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/77742979/.travis.yml ---------------------------------------------------------------------- diff --cc .travis.yml index 470d2fc,a806477..cb6f790 --- a/.travis.yml +++ b/.travis.yml @@@ -28,13 -28,9 +28,13 @@@ notifications on_success: change on_failure: always +addons: + apt: + packages: + - python2.7 env: global: - - MAVEN_OVERRIDE="--settings=.travis/settings.xml -Dmaven.javadoc.skip=true -Dcheckstyle.skip=true -Dfindbugs.skip=true" + - MAVEN_OVERRIDE="--settings=.travis/settings.xml" - MAVEN_CONTAINER_OVERRIDE="-DbeamSurefireArgline='-Xmx512m'" matrix: http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/77742979/pom.xml ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/77742979/runners/flink/runner/src/main/java/org/apache/beam/runners/flink/translation/FlinkBatchTransformTranslators.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/77742979/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java ---------------------------------------------------------------------- diff --cc runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java index 60d668e,5dd6beb..8430bbf --- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java +++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/TransformTranslator.java @@@ -54,11 -56,8 +56,9 @@@ import org.apache.beam.sdk.transforms.G import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.ParDo; import org.apache.beam.sdk.transforms.View; +import org.apache.beam.sdk.transforms.reflect.DoFnSignatures; import org.apache.beam.sdk.transforms.windowing.BoundedWindow; import org.apache.beam.sdk.transforms.windowing.Window; - import org.apache.beam.sdk.transforms.windowing.WindowFn; import org.apache.beam.sdk.util.CombineFnUtil; import org.apache.beam.sdk.util.WindowedValue; import org.apache.beam.sdk.util.WindowingStrategy; http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/77742979/sdks/pom.xml ----------------------------------------------------------------------
