Closes #758
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/9a329aad Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/9a329aad Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/9a329aad Branch: refs/heads/python-sdk Commit: 9a329aada82d3ca7f619b88eddea04bdd329d992 Parents: 388816a 3466a0e Author: Dan Halperin <[email protected]> Authored: Wed Aug 3 18:25:53 2016 -0700 Committer: Dan Halperin <[email protected]> Committed: Wed Aug 3 18:25:53 2016 -0700 ---------------------------------------------------------------------- .../beam/examples/DebuggingWordCount.java | 2 +- .../apache/beam/examples/MinimalWordCount.java | 2 +- .../apache/beam/examples/WindowedWordCount.java | 4 +- .../org/apache/beam/examples/WordCount.java | 2 +- .../examples/common/PubsubFileInjector.java | 6 +- .../beam/examples/complete/AutoComplete.java | 14 +- .../examples/complete/StreamingWordExtract.java | 12 +- .../apache/beam/examples/complete/TfIdf.java | 16 +- .../examples/complete/TopWikipediaSessions.java | 12 +- .../examples/complete/TrafficMaxLaneFlow.java | 10 +- .../beam/examples/complete/TrafficRoutes.java | 12 +- .../examples/cookbook/BigQueryTornadoes.java | 6 +- .../cookbook/CombinePerKeyExamples.java | 6 +- .../examples/cookbook/DatastoreWordCount.java | 11 +- .../beam/examples/cookbook/FilterExamples.java | 12 +- .../beam/examples/cookbook/JoinExamples.java | 10 +- .../examples/cookbook/MaxPerKeyExamples.java | 6 +- .../beam/examples/cookbook/TriggerExample.java | 12 +- .../org/apache/beam/examples/WordCountTest.java | 2 +- .../examples/complete/AutoCompleteTest.java | 4 +- .../examples/cookbook/TriggerExampleTest.java | 4 +- .../beam/examples/complete/game/GameStats.java | 10 +- .../beam/examples/complete/game/UserScore.java | 4 +- .../complete/game/utils/WriteToBigQuery.java | 12 +- .../game/utils/WriteWindowedToBigQuery.java | 8 +- .../examples/complete/game/UserScoreTest.java | 2 +- .../core/GroupAlsoByWindowViaWindowSetDoFn.java | 12 +- .../core/UnboundedReadFromBoundedSource.java | 2 +- .../apache/beam/sdk/util/AssignWindowsDoFn.java | 10 +- .../org/apache/beam/sdk/util/DoFnRunner.java | 21 +- .../apache/beam/sdk/util/DoFnRunnerBase.java | 54 +- .../org/apache/beam/sdk/util/DoFnRunners.java | 24 +- .../beam/sdk/util/GroupAlsoByWindowsDoFn.java | 6 +- .../GroupAlsoByWindowsViaOutputBufferDoFn.java | 4 +- .../sdk/util/GroupByKeyViaGroupByKeyOnly.java | 6 +- .../sdk/util/LateDataDroppingDoFnRunner.java | 4 +- .../apache/beam/sdk/util/PaneInfoTracker.java | 1 - .../apache/beam/sdk/util/ReduceFnRunner.java | 4 +- .../apache/beam/sdk/util/SimpleDoFnRunner.java | 12 +- .../org/apache/beam/sdk/util/WatermarkHold.java | 1 - .../beam/sdk/util/ReduceFnRunnerTest.java | 1 + .../apache/beam/sdk/util/ReduceFnTester.java | 1 + .../beam/sdk/util/SimpleDoFnRunnerTest.java | 6 +- .../GroupAlsoByWindowEvaluatorFactory.java | 4 +- .../ImmutabilityCheckingBundleFactory.java | 4 +- .../beam/runners/direct/ParDoEvaluator.java | 4 +- .../direct/ParDoMultiEvaluatorFactory.java | 11 +- .../direct/ParDoSingleEvaluatorFactory.java | 11 +- .../direct/TransformEvaluatorFactory.java | 6 +- .../direct/WriteWithShardingFactory.java | 4 +- .../ConsumerTrackingPipelineVisitorTest.java | 22 +- .../beam/runners/direct/DirectRunnerTest.java | 24 +- .../ImmutabilityCheckingBundleFactoryTest.java | 6 +- .../ImmutabilityEnforcementFactoryTest.java | 6 +- .../direct/KeyedPValueTrackingVisitorTest.java | 6 +- .../beam/runners/direct/ParDoEvaluatorTest.java | 6 +- .../direct/ParDoMultiEvaluatorFactoryTest.java | 10 +- .../direct/ParDoSingleEvaluatorFactoryTest.java | 10 +- .../runners/direct/WatermarkManagerTest.java | 7 +- .../beam/runners/flink/examples/TFIDF.java | 16 +- .../beam/runners/flink/examples/WordCount.java | 4 +- .../flink/examples/streaming/AutoComplete.java | 16 +- .../flink/examples/streaming/JoinExamples.java | 8 +- .../examples/streaming/KafkaIOExamples.java | 4 +- .../KafkaWindowedWordCountExample.java | 6 +- .../examples/streaming/WindowedWordCount.java | 6 +- .../FlinkBatchTransformTranslators.java | 12 +- .../FlinkStreamingTransformTranslators.java | 9 +- .../functions/FlinkDoFnFunction.java | 10 +- .../FlinkMergingNonShuffleReduceFunction.java | 8 +- .../functions/FlinkMultiOutputDoFnFunction.java | 10 +- .../FlinkMultiOutputProcessContext.java | 6 +- .../functions/FlinkNoElementAssignContext.java | 8 +- .../functions/FlinkPartialReduceFunction.java | 8 +- .../functions/FlinkProcessContext.java | 16 +- .../functions/FlinkReduceFunction.java | 8 +- .../streaming/FlinkAbstractParDoWrapper.java | 18 +- .../FlinkGroupAlsoByWindowWrapper.java | 10 +- .../streaming/FlinkParDoBoundMultiWrapper.java | 4 +- .../streaming/FlinkParDoBoundWrapper.java | 4 +- .../state/AbstractFlinkTimerInternals.java | 4 +- .../beam/runners/flink/PipelineOptionsTest.java | 6 +- .../beam/runners/flink/ReadSourceITCase.java | 4 +- .../flink/ReadSourceStreamingITCase.java | 4 +- .../flink/streaming/GroupByNullKeyTest.java | 8 +- .../streaming/TopWikipediaSessionsITCase.java | 6 +- .../dataflow/DataflowPipelineTranslator.java | 6 +- .../beam/runners/dataflow/DataflowRunner.java | 87 ++- .../dataflow/internal/AssignWindows.java | 6 +- .../beam/runners/dataflow/util/DoFnInfo.java | 16 +- .../DataflowPipelineTranslatorTest.java | 10 +- .../beam/runners/spark/examples/WordCount.java | 4 +- .../runners/spark/translation/DoFnFunction.java | 8 +- .../spark/translation/MultiDoFnFunction.java | 8 +- .../spark/translation/SparkProcessContext.java | 18 +- .../spark/translation/TransformTranslator.java | 7 +- .../streaming/StreamingTransformTranslator.java | 4 +- .../apache/beam/runners/spark/TfIdfTest.java | 12 +- .../spark/translation/CombinePerKeyTest.java | 4 +- .../spark/translation/DoFnOutputTest.java | 4 +- .../translation/MultiOutputWordCountTest.java | 8 +- .../spark/translation/SerializationTest.java | 10 +- .../spark/translation/SideEffectsTest.java | 4 +- .../streaming/KafkaStreamingTest.java | 4 +- .../org/apache/beam/sdk/coders/AvroCoder.java | 1 - .../apache/beam/sdk/coders/DurationCoder.java | 1 - .../apache/beam/sdk/coders/InstantCoder.java | 1 - .../java/org/apache/beam/sdk/io/PubsubIO.java | 6 +- .../apache/beam/sdk/io/PubsubUnboundedSink.java | 8 +- .../beam/sdk/io/PubsubUnboundedSource.java | 4 +- .../java/org/apache/beam/sdk/io/Source.java | 2 +- .../main/java/org/apache/beam/sdk/io/Write.java | 21 +- .../org/apache/beam/sdk/options/GcpOptions.java | 1 - .../beam/sdk/options/PipelineOptions.java | 8 +- .../sdk/options/PipelineOptionsFactory.java | 1 - .../sdk/options/PipelineOptionsReflector.java | 1 + .../beam/sdk/runners/AggregatorValues.java | 4 +- .../org/apache/beam/sdk/testing/PAssert.java | 24 +- .../beam/sdk/testing/SerializableMatchers.java | 1 - .../apache/beam/sdk/testing/TestPipeline.java | 1 - .../beam/sdk/testing/TestPipelineOptions.java | 1 + .../apache/beam/sdk/transforms/Aggregator.java | 14 +- .../sdk/transforms/AggregatorRetriever.java | 6 +- .../org/apache/beam/sdk/transforms/Combine.java | 14 +- .../apache/beam/sdk/transforms/CombineFns.java | 4 +- .../org/apache/beam/sdk/transforms/Count.java | 2 +- .../org/apache/beam/sdk/transforms/Create.java | 2 +- .../org/apache/beam/sdk/transforms/DoFn.java | 418 +++++--------- .../beam/sdk/transforms/DoFnReflector.java | 116 ++-- .../apache/beam/sdk/transforms/DoFnTester.java | 88 +-- .../beam/sdk/transforms/DoFnWithContext.java | 429 -------------- .../org/apache/beam/sdk/transforms/Filter.java | 2 +- .../beam/sdk/transforms/FlatMapElements.java | 2 +- .../org/apache/beam/sdk/transforms/Flatten.java | 2 +- .../apache/beam/sdk/transforms/GroupByKey.java | 2 +- .../transforms/IntraBundleParallelization.java | 40 +- .../org/apache/beam/sdk/transforms/Keys.java | 2 +- .../org/apache/beam/sdk/transforms/KvSwap.java | 2 +- .../apache/beam/sdk/transforms/MapElements.java | 2 +- .../org/apache/beam/sdk/transforms/OldDoFn.java | 565 +++++++++++++++++++ .../apache/beam/sdk/transforms/PTransform.java | 2 +- .../org/apache/beam/sdk/transforms/ParDo.java | 219 +++---- .../apache/beam/sdk/transforms/Partition.java | 2 +- .../beam/sdk/transforms/RemoveDuplicates.java | 2 +- .../org/apache/beam/sdk/transforms/Sample.java | 4 +- .../beam/sdk/transforms/SimpleFunction.java | 6 +- .../org/apache/beam/sdk/transforms/Values.java | 2 +- .../org/apache/beam/sdk/transforms/View.java | 8 +- .../apache/beam/sdk/transforms/WithKeys.java | 2 +- .../beam/sdk/transforms/WithTimestamps.java | 4 +- .../sdk/transforms/display/DisplayData.java | 1 - .../beam/sdk/transforms/join/CoGbkResult.java | 1 - .../beam/sdk/transforms/join/CoGroupByKey.java | 14 +- .../sdk/transforms/windowing/AfterEach.java | 1 + .../windowing/AfterProcessingTime.java | 1 + .../transforms/windowing/IntervalWindow.java | 1 - .../beam/sdk/transforms/windowing/Never.java | 1 + .../beam/sdk/transforms/windowing/PaneInfo.java | 10 +- .../beam/sdk/transforms/windowing/Window.java | 4 +- .../beam/sdk/util/BaseExecutionContext.java | 4 +- .../apache/beam/sdk/util/BucketingFunction.java | 1 + .../beam/sdk/util/CombineContextFactory.java | 6 +- .../apache/beam/sdk/util/ExecutionContext.java | 8 +- .../apache/beam/sdk/util/MovingFunction.java | 1 + .../beam/sdk/util/PerKeyCombineFnRunner.java | 44 +- .../beam/sdk/util/PerKeyCombineFnRunners.java | 30 +- .../org/apache/beam/sdk/util/PubsubClient.java | 1 + .../apache/beam/sdk/util/PubsubTestClient.java | 1 + .../sdk/util/ReifyTimestampAndWindowsDoFn.java | 6 +- .../org/apache/beam/sdk/util/Reshuffle.java | 4 +- .../apache/beam/sdk/util/SerializableUtils.java | 2 +- .../org/apache/beam/sdk/util/StringUtils.java | 2 +- .../beam/sdk/util/SystemDoFnInternal.java | 6 +- .../apache/beam/sdk/util/TimerInternals.java | 1 - .../apache/beam/sdk/util/ValueWithRecordId.java | 6 +- .../org/apache/beam/sdk/util/WindowedValue.java | 1 - .../beam/sdk/util/WindowingInternals.java | 4 +- .../beam/sdk/util/common/ReflectHelpers.java | 1 + .../beam/sdk/values/TimestampedValue.java | 1 - .../java/org/apache/beam/sdk/PipelineTest.java | 6 +- .../apache/beam/sdk/coders/AvroCoderTest.java | 4 +- .../beam/sdk/coders/CoderRegistryTest.java | 6 +- .../beam/sdk/coders/SerializableCoderTest.java | 6 +- .../org/apache/beam/sdk/io/AvroSourceTest.java | 1 + .../io/BoundedReadFromUnboundedSourceTest.java | 1 + .../beam/sdk/io/CompressedSourceTest.java | 1 + .../apache/beam/sdk/io/CountingInputTest.java | 5 +- .../apache/beam/sdk/io/CountingSourceTest.java | 4 +- .../beam/sdk/io/OffsetBasedSourceTest.java | 1 + .../beam/sdk/io/PubsubUnboundedSinkTest.java | 4 +- .../java/org/apache/beam/sdk/io/ReadTest.java | 1 + .../java/org/apache/beam/sdk/io/TextIOTest.java | 1 + .../java/org/apache/beam/sdk/io/WriteTest.java | 7 +- .../org/apache/beam/sdk/io/XmlSinkTest.java | 1 + .../apache/beam/sdk/options/GcpOptionsTest.java | 1 + .../sdk/options/GoogleApiDebugOptionsTest.java | 1 - .../sdk/options/PipelineOptionsFactoryTest.java | 1 - .../beam/sdk/options/PipelineOptionsTest.java | 1 - .../sdk/options/ProxyInvocationHandlerTest.java | 2 +- .../AggregatorPipelineExtractorTest.java | 6 +- .../apache/beam/sdk/testing/PAssertTest.java | 1 - .../beam/sdk/testing/TestPipelineTest.java | 1 - .../transforms/ApproximateQuantilesTest.java | 1 + .../sdk/transforms/ApproximateUniqueTest.java | 5 +- .../beam/sdk/transforms/CombineFnsTest.java | 2 +- .../apache/beam/sdk/transforms/CombineTest.java | 12 +- .../apache/beam/sdk/transforms/CreateTest.java | 2 +- .../beam/sdk/transforms/DoFnContextTest.java | 69 --- .../DoFnDelegatingAggregatorTest.java | 16 +- .../beam/sdk/transforms/DoFnReflectorTest.java | 88 +-- .../apache/beam/sdk/transforms/DoFnTest.java | 131 +++-- .../beam/sdk/transforms/DoFnTesterTest.java | 10 +- .../sdk/transforms/DoFnWithContextTest.java | 237 -------- .../apache/beam/sdk/transforms/FlattenTest.java | 4 +- .../beam/sdk/transforms/GroupByKeyTest.java | 6 +- .../IntraBundleParallelizationTest.java | 23 +- .../beam/sdk/transforms/MapElementsTest.java | 1 + .../org/apache/beam/sdk/transforms/MaxTest.java | 1 + .../org/apache/beam/sdk/transforms/MinTest.java | 2 + .../apache/beam/sdk/transforms/NoOpDoFn.java | 144 ----- .../apache/beam/sdk/transforms/NoOpOldDoFn.java | 144 +++++ .../beam/sdk/transforms/OldDoFnContextTest.java | 69 +++ .../apache/beam/sdk/transforms/OldDoFnTest.java | 242 ++++++++ .../apache/beam/sdk/transforms/ParDoTest.java | 108 ++-- .../beam/sdk/transforms/PartitionTest.java | 1 + .../apache/beam/sdk/transforms/SampleTest.java | 1 + .../org/apache/beam/sdk/transforms/TopTest.java | 1 + .../apache/beam/sdk/transforms/ViewTest.java | 398 ++++++------- .../beam/sdk/transforms/WithTimestampsTest.java | 8 +- .../display/DisplayDataEvaluatorTest.java | 6 +- .../display/DisplayDataMatchersTest.java | 1 + .../sdk/transforms/display/DisplayDataTest.java | 6 +- .../dofnreflector/DoFnReflectorTestHelper.java | 26 +- .../sdk/transforms/join/CoGroupByKeyTest.java | 18 +- .../sdk/transforms/windowing/NeverTest.java | 1 + .../sdk/transforms/windowing/WindowTest.java | 6 +- .../sdk/transforms/windowing/WindowingTest.java | 10 +- .../beam/sdk/util/BucketingFunctionTest.java | 4 +- .../beam/sdk/util/MovingFunctionTest.java | 4 +- .../beam/sdk/util/SerializableUtilsTest.java | 1 - .../apache/beam/sdk/util/SerializerTest.java | 1 - .../apache/beam/sdk/util/StringUtilsTest.java | 16 +- .../org/apache/beam/sdk/util/TriggerTester.java | 1 + .../beam/sdk/util/common/CounterTest.java | 1 + .../beam/sdk/values/PCollectionTupleTest.java | 4 +- .../apache/beam/sdk/values/TypedPValueTest.java | 6 +- .../beam/sdk/extensions/joinlibrary/Join.java | 8 +- .../beam/sdk/io/gcp/bigquery/BigQueryIO.java | 18 +- .../beam/sdk/io/gcp/bigtable/BigtableIO.java | 4 +- .../beam/sdk/io/gcp/datastore/V1Beta3.java | 13 +- .../sdk/io/gcp/bigquery/BigQueryIOTest.java | 6 +- .../sdk/io/gcp/bigtable/BigtableWriteIT.java | 4 +- .../sdk/io/gcp/datastore/V1Beta3TestUtil.java | 6 +- .../java/org/apache/beam/sdk/io/jms/JmsIO.java | 4 +- .../org/apache/beam/sdk/io/kafka/KafkaIO.java | 8 +- .../apache/beam/sdk/io/kafka/KafkaIOTest.java | 7 +- .../sdk/transforms/WithTimestampsJava8Test.java | 4 +- .../src/main/java/DebuggingWordCount.java | 4 +- .../src/main/java/MinimalWordCount.java | 6 +- .../src/main/java/WindowedWordCount.java | 6 +- .../src/main/java/WordCount.java | 6 +- .../main/java/common/PubsubFileInjector.java | 4 +- .../src/main/java/StarterPipeline.java | 6 +- .../src/main/java/it/pkg/StarterPipeline.java | 6 +- .../transforms/DoFnReflectorBenchmark.java | 49 +- 265 files changed, 2641 insertions(+), 2596 deletions(-) ----------------------------------------------------------------------
