This closes #498
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/f166b16b Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/f166b16b Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/f166b16b Branch: refs/heads/python-sdk Commit: f166b16b811624aa66d54dfe3811a80306d12e9c Parents: 0e5c662 4b41768 Author: Davor Bonaci <[email protected]> Authored: Mon Jun 20 15:14:49 2016 -0700 Committer: Davor Bonaci <[email protected]> Committed: Mon Jun 20 15:14:49 2016 -0700 ---------------------------------------------------------------------- README.md | 4 +- examples/java/README.md | 42 +- examples/java/pom.xml | 22 +- .../beam/examples/DebuggingWordCount.java | 2 +- .../apache/beam/examples/MinimalWordCount.java | 4 +- .../apache/beam/examples/WindowedWordCount.java | 13 +- .../org/apache/beam/examples/WordCount.java | 21 +- .../examples/common/DataflowExampleUtils.java | 31 +- .../beam/examples/complete/AutoComplete.java | 12 +- .../apache/beam/examples/complete/TfIdf.java | 2 +- .../examples/complete/TopWikipediaSessions.java | 2 +- .../examples/cookbook/BigQueryTornadoes.java | 2 +- .../cookbook/CombinePerKeyExamples.java | 2 +- .../examples/cookbook/DatastoreWordCount.java | 8 +- .../beam/examples/cookbook/DeDupExample.java | 2 +- .../beam/examples/cookbook/FilterExamples.java | 2 +- .../beam/examples/cookbook/JoinExamples.java | 2 +- .../examples/cookbook/MaxPerKeyExamples.java | 2 +- .../org/apache/beam/examples/WordCountTest.java | 6 +- .../examples/complete/AutoCompleteTest.java | 14 +- .../cookbook/BigQueryTornadoesTest.java | 8 +- .../cookbook/CombinePerKeyExamplesTest.java | 4 +- .../examples/cookbook/FilterExamplesTest.java | 4 +- .../examples/cookbook/JoinExamplesTest.java | 4 +- .../cookbook/MaxPerKeyExamplesTest.java | 4 +- .../examples/cookbook/TriggerExampleTest.java | 4 +- .../beam/examples/MinimalWordCountJava8.java | 12 +- .../beam/examples/complete/game/GameStats.java | 7 +- .../examples/complete/game/HourlyTeamScore.java | 8 +- .../examples/complete/game/LeaderBoard.java | 2 +- .../beam/examples/complete/game/UserScore.java | 7 +- .../examples/MinimalWordCountJava8Test.java | 8 +- .../complete/game/HourlyTeamScoreTest.java | 7 +- .../examples/complete/game/UserScoreTest.java | 7 +- runners/core-java/pom.xml | 13 + runners/direct-java/pom.xml | 34 +- .../direct/AbstractModelEnforcement.java | 4 +- .../direct/BoundedReadEvaluatorFactory.java | 16 +- .../beam/runners/direct/BundleFactory.java | 8 +- .../beam/runners/direct/CommittedResult.java | 6 +- .../beam/runners/direct/CompletionCallback.java | 4 +- .../runners/direct/DirectExecutionContext.java | 106 + .../beam/runners/direct/DirectGroupByKey.java | 132 + .../direct/DirectGroupByKeyOverrideFactory.java | 41 + .../beam/runners/direct/DirectOptions.java | 101 + .../beam/runners/direct/DirectRegistrar.java | 57 + .../beam/runners/direct/DirectRunner.java | 372 ++ .../runners/direct/DirectTimerInternals.java | 84 + .../runners/direct/EmptyTransformEvaluator.java | 4 +- .../direct/EncodabilityEnforcementFactory.java | 2 +- .../beam/runners/direct/EvaluationContext.java | 429 +++ .../beam/runners/direct/EvaluatorKey.java | 4 +- .../direct/ExecutorServiceParallelExecutor.java | 32 +- .../runners/direct/FlattenEvaluatorFactory.java | 18 +- .../GroupAlsoByWindowEvaluatorFactory.java | 127 + .../direct/GroupByKeyOnlyEvaluatorFactory.java | 186 + .../ImmutabilityCheckingBundleFactory.java | 4 +- .../direct/ImmutabilityEnforcementFactory.java | 4 +- .../direct/ImmutableListBundleFactory.java | 163 + .../direct/InMemoryWatermarkManager.java | 1420 -------- .../runners/direct/InProcessBundleFactory.java | 161 - .../direct/InProcessBundleOutputManager.java | 51 - .../direct/InProcessEvaluationContext.java | 428 --- .../direct/InProcessExecutionContext.java | 105 - .../beam/runners/direct/InProcessExecutor.java | 48 - ...rocessGroupAlsoByWindowEvaluatorFactory.java | 127 - .../runners/direct/InProcessGroupByKey.java | 132 - ...InProcessGroupByKeyOnlyEvaluatorFactory.java | 185 - .../InProcessGroupByKeyOverrideFactory.java | 41 - .../direct/InProcessPipelineOptions.java | 101 - .../runners/direct/InProcessPipelineRunner.java | 370 -- .../beam/runners/direct/InProcessRegistrar.java | 55 - .../direct/InProcessSideInputContainer.java | 277 -- .../runners/direct/InProcessTimerInternals.java | 84 - .../direct/InProcessTransformResult.java | 84 - .../beam/runners/direct/ModelEnforcement.java | 8 +- .../runners/direct/ModelEnforcementFactory.java | 2 +- .../beam/runners/direct/ParDoEvaluator.java | 186 + .../runners/direct/ParDoInProcessEvaluator.java | 186 - .../direct/ParDoMultiEvaluatorFactory.java | 10 +- .../direct/ParDoSingleEvaluatorFactory.java | 10 +- .../direct/PassthroughTransformEvaluator.java | 4 +- .../beam/runners/direct/PipelineExecutor.java | 48 + .../beam/runners/direct/SideInputContainer.java | 277 ++ .../runners/direct/StepTransformResult.java | 8 +- ...readLocalInvalidatingTransformEvaluator.java | 2 +- .../beam/runners/direct/TransformEvaluator.java | 6 +- .../direct/TransformEvaluatorFactory.java | 4 +- .../direct/TransformEvaluatorRegistry.java | 12 +- .../beam/runners/direct/TransformExecutor.java | 16 +- .../beam/runners/direct/TransformResult.java | 84 + .../direct/UnboundedReadEvaluatorFactory.java | 161 +- .../direct/UncommittedBundleOutputManager.java | 51 + .../runners/direct/ViewEvaluatorFactory.java | 22 +- .../direct/WatermarkCallbackExecutor.java | 10 +- .../beam/runners/direct/WatermarkManager.java | 1420 ++++++++ .../runners/direct/WindowEvaluatorFactory.java | 18 +- .../direct/AvroIOShardedWriteFactoryTest.java | 12 +- .../direct/BoundedReadEvaluatorFactoryTest.java | 14 +- .../runners/direct/CommittedResultTest.java | 14 +- .../runners/direct/DirectRegistrarTest.java | 74 + .../beam/runners/direct/DirectRunnerTest.java | 339 ++ .../direct/DirectTimerInternalsTest.java | 134 + .../EncodabilityEnforcementFactoryTest.java | 4 +- .../runners/direct/EvaluationContextTest.java | 545 +++ .../direct/FlattenEvaluatorFactoryTest.java | 16 +- .../direct/GroupByKeyEvaluatorFactoryTest.java | 13 +- .../GroupByKeyOnlyEvaluatorFactoryTest.java | 197 ++ .../ImmutabilityCheckingBundleFactoryTest.java | 6 +- .../ImmutabilityEnforcementFactoryTest.java | 4 +- .../direct/ImmutableListBundleFactoryTest.java | 231 ++ .../direct/InMemoryWatermarkManagerTest.java | 1428 -------- .../direct/InProcessBundleFactoryTest.java | 231 -- .../direct/InProcessEvaluationContextTest.java | 545 --- ...ocessGroupByKeyOnlyEvaluatorFactoryTest.java | 196 -- .../direct/InProcessPipelineRegistrarTest.java | 74 - .../direct/InProcessPipelineRunnerTest.java | 330 -- .../direct/InProcessSideInputContainerTest.java | 520 --- .../direct/InProcessTimerInternalsTest.java | 134 - .../direct/KeyedPValueTrackingVisitorTest.java | 7 +- .../beam/runners/direct/ParDoEvaluatorTest.java | 214 ++ .../direct/ParDoInProcessEvaluatorTest.java | 214 -- .../direct/ParDoMultiEvaluatorFactoryTest.java | 36 +- .../direct/ParDoSingleEvaluatorFactoryTest.java | 36 +- .../runners/direct/SideInputContainerTest.java | 520 +++ .../direct/TextIOShardedWriteFactoryTest.java | 12 +- ...LocalInvalidatingTransformEvaluatorTest.java | 4 +- .../runners/direct/TransformExecutorTest.java | 44 +- .../UnboundedReadEvaluatorFactoryTest.java | 72 +- .../direct/ViewEvaluatorFactoryTest.java | 8 +- .../runners/direct/WatermarkManagerTest.java | 1428 ++++++++ .../direct/WindowEvaluatorFactoryTest.java | 20 +- runners/flink/README.md | 4 +- .../beam/runners/flink/examples/TFIDF.java | 6 +- .../beam/runners/flink/examples/WordCount.java | 4 +- .../flink/examples/streaming/AutoComplete.java | 16 +- .../flink/examples/streaming/JoinExamples.java | 4 +- .../examples/streaming/KafkaIOExamples.java | 4 +- .../KafkaWindowedWordCountExample.java | 4 +- .../examples/streaming/WindowedWordCount.java | 4 +- runners/flink/runner/pom.xml | 26 +- .../beam/runners/flink/FlinkPipelineRunner.java | 180 - .../apache/beam/runners/flink/FlinkRunner.java | 179 + .../runners/flink/FlinkRunnerRegistrar.java | 4 +- .../beam/runners/flink/FlinkRunnerResult.java | 8 + .../runners/flink/TestFlinkPipelineRunner.java | 80 - .../beam/runners/flink/TestFlinkRunner.java | 81 + .../FlinkBatchTransformTranslators.java | 49 +- .../FlinkBatchTranslationContext.java | 13 +- .../translation/types/CoderComparator.java | 217 -- .../translation/types/CoderTypeInformation.java | 14 +- .../translation/types/CoderTypeSerializer.java | 37 +- .../types/EncodedValueComparator.java | 197 ++ .../types/EncodedValueSerializer.java | 113 + .../types/EncodedValueTypeInformation.java | 111 + .../translation/types/KvCoderComperator.java | 259 -- .../types/KvCoderTypeInformation.java | 207 -- .../flink/translation/types/KvKeySelector.java | 51 + .../streaming/FlinkAbstractParDoWrapper.java | 9 +- .../streaming/io/UnboundedFlinkSource.java | 6 +- .../streaming/state/FlinkStateInternals.java | 2 +- .../flink/EncodedValueComparatorTest.java | 71 + .../runners/flink/FlinkRunnerRegistrarTest.java | 8 +- .../beam/runners/flink/FlinkTestPipeline.java | 4 +- .../beam/runners/flink/PipelineOptionsTest.java | 11 +- runners/google-cloud-dataflow-java/pom.xml | 11 + .../BlockingDataflowPipelineRunner.java | 186 - .../dataflow/BlockingDataflowRunner.java | 186 + .../dataflow/DataflowJobCancelledException.java | 2 +- .../dataflow/DataflowJobExecutionException.java | 2 +- .../dataflow/DataflowJobUpdatedException.java | 2 +- .../runners/dataflow/DataflowPipelineJob.java | 2 +- .../dataflow/DataflowPipelineRegistrar.java | 8 +- .../dataflow/DataflowPipelineRunner.java | 3229 ------------------ .../dataflow/DataflowPipelineRunnerHooks.java | 39 - .../dataflow/DataflowPipelineTranslator.java | 10 +- .../beam/runners/dataflow/DataflowRunner.java | 3229 ++++++++++++++++++ .../runners/dataflow/DataflowRunnerHooks.java | 39 + .../runners/dataflow/internal/package-info.java | 2 +- .../BlockingDataflowPipelineOptions.java | 6 +- .../options/DataflowPipelineOptions.java | 4 +- .../DataflowPipelineWorkerPoolOptions.java | 6 +- .../options/DataflowWorkerLoggingOptions.java | 79 +- .../testing/TestDataflowPipelineRunner.java | 271 -- .../dataflow/testing/TestDataflowRunner.java | 271 ++ .../dataflow/util/DataflowTransport.java | 11 - .../BlockingDataflowPipelineRunnerTest.java | 302 -- .../dataflow/BlockingDataflowRunnerTest.java | 304 ++ .../dataflow/DataflowPipelineRegistrarTest.java | 4 +- .../dataflow/DataflowPipelineRunnerTest.java | 1415 -------- .../DataflowPipelineTranslatorTest.java | 50 +- .../runners/dataflow/DataflowRunnerTest.java | 1417 ++++++++ .../runners/dataflow/io/DataflowAvroIOTest.java | 4 +- .../dataflow/io/DataflowPubsubIOTest.java | 4 +- .../runners/dataflow/io/DataflowTextIOTest.java | 85 +- .../testing/TestDataflowPipelineRunnerTest.java | 601 ---- .../testing/TestDataflowRunnerTest.java | 601 ++++ .../DataflowDisplayDataEvaluator.java | 10 +- .../transforms/DataflowGroupByKeyTest.java | 8 +- .../dataflow/transforms/DataflowViewTest.java | 8 +- .../util/DataflowPathValidatorTest.java | 8 +- runners/spark/pom.xml | 118 +- .../runners/spark/SparkRunnerRegistrar.java | 3 +- .../runners/spark/TestSparkPipelineRunner.java | 77 + .../spark/translation/TransformTranslator.java | 17 +- .../beam/runners/spark/SimpleWordCountTest.java | 2 +- .../runners/spark/SparkRunnerRegistrarTest.java | 2 +- .../spark/translation/SerializationTest.java | 2 +- .../translation/TransformTranslatorTest.java | 6 +- .../translation/WindowedWordCountTest.java | 19 +- sdks/java/core/pom.xml | 3 + .../apache/beam/sdk/coders/BigDecimalCoder.java | 102 + .../apache/beam/sdk/coders/BigIntegerCoder.java | 91 + .../java/org/apache/beam/sdk/io/BigQueryIO.java | 6 +- .../java/org/apache/beam/sdk/io/PubsubIO.java | 3 +- .../main/java/org/apache/beam/sdk/io/Read.java | 44 - .../java/org/apache/beam/sdk/io/TextIO.java | 3 +- .../beam/sdk/io/range/ByteKeyRangeTracker.java | 8 + .../beam/sdk/options/DirectPipelineOptions.java | 75 - .../beam/sdk/options/PipelineOptions.java | 38 +- .../sdk/runners/DirectPipelineRegistrar.java | 55 - .../beam/sdk/runners/DirectPipelineRunner.java | 1298 ------- .../apache/beam/sdk/runners/package-info.java | 12 +- .../apache/beam/sdk/testing/CrashingRunner.java | 33 +- .../org/apache/beam/sdk/testing/PAssert.java | 206 +- .../apache/beam/sdk/testing/TestPipeline.java | 5 +- .../apache/beam/sdk/transforms/Aggregator.java | 2 +- .../apache/beam/sdk/transforms/DoFnTester.java | 159 +- .../org/apache/beam/sdk/transforms/Filter.java | 128 +- .../org/apache/beam/sdk/transforms/Flatten.java | 32 - .../org/apache/beam/sdk/transforms/ParDo.java | 302 +- .../org/apache/beam/sdk/transforms/View.java | 24 - .../sdk/transforms/display/package-info.java | 26 + .../transforms/windowing/AfterWatermark.java | 14 +- .../apache/beam/sdk/util/AssignWindowsDoFn.java | 9 +- .../beam/sdk/util/BigQueryTableRowIterator.java | 4 +- .../sdk/util/DirectModeExecutionContext.java | 130 - .../apache/beam/sdk/util/DoFnRunnerBase.java | 3 +- .../apache/beam/sdk/util/ReduceFnRunner.java | 19 +- .../java/org/apache/beam/sdk/PipelineTest.java | 4 +- .../org/apache/beam/sdk/WindowMatchers.java | 5 + .../beam/sdk/coders/BigDecimalCoderTest.java | 144 + .../beam/sdk/coders/BigIntegerCoderTest.java | 119 + .../org/apache/beam/sdk/io/BigQueryIOTest.java | 4 +- .../io/BoundedReadFromUnboundedSourceTest.java | 6 - .../beam/sdk/io/CompressedSourceTest.java | 4 +- .../java/org/apache/beam/sdk/io/TextIOTest.java | 75 + .../java/org/apache/beam/sdk/io/WriteTest.java | 9 +- .../sdk/io/range/ByteKeyRangeTrackerTest.java | 84 +- .../sdk/options/PipelineOptionsFactoryTest.java | 79 +- .../beam/sdk/options/PipelineOptionsTest.java | 8 - .../options/PipelineOptionsValidatorTest.java | 15 + .../runners/DirectPipelineRegistrarTest.java | 71 - .../sdk/runners/DirectPipelineRunnerTest.java | 221 -- .../beam/sdk/runners/PipelineRunnerTest.java | 35 +- .../beam/sdk/testing/TestPipelineTest.java | 5 +- .../apache/beam/sdk/transforms/CombineTest.java | 21 - .../beam/sdk/transforms/DoFnTesterTest.java | 54 +- .../apache/beam/sdk/transforms/FilterTest.java | 63 +- .../beam/sdk/transforms/GroupByKeyTest.java | 219 +- .../apache/beam/sdk/transforms/ViewTest.java | 29 +- .../sdk/transforms/join/CoGroupByKeyTest.java | 2 +- .../windowing/AfterWatermarkTest.java | 19 + .../sdk/util/GroupAlsoByWindowsProperties.java | 293 +- .../beam/sdk/util/ReduceFnRunnerTest.java | 68 + sdks/java/extensions/join-library/pom.xml | 7 + sdks/java/io/google-cloud-platform/pom.xml | 7 + .../beam/sdk/io/gcp/bigtable/BigtableIO.java | 5 +- .../sdk/io/gcp/bigtable/BigtableIOTest.java | 2 - sdks/java/io/hdfs/pom.xml | 7 + sdks/java/io/kafka/pom.xml | 7 + sdks/java/java8tests/pom.xml | 7 + .../beam/sdk/transforms/FilterJava8Test.java | 8 +- .../beam/sdk/transforms/WithKeysJava8Test.java | 3 +- .../main/resources/archetype-resources/pom.xml | 19 + .../src/main/java/DebuggingWordCount.java | 2 +- .../src/main/java/MinimalWordCount.java | 4 +- .../src/main/java/WindowedWordCount.java | 2 +- .../src/main/java/WordCount.java | 2 +- .../main/java/common/DataflowExampleUtils.java | 25 +- .../src/test/java/WordCountTest.java | 6 +- .../main/resources/archetype-resources/pom.xml | 13 + .../src/main/java/StarterPipeline.java | 4 +- .../resources/projects/basic/reference/pom.xml | 13 + .../src/main/java/it/pkg/StarterPipeline.java | 4 +- sdks/java/pom.xml | 2 +- testing/travis/test_wordcount.sh | 4 +- 287 files changed, 17190 insertions(+), 18220 deletions(-) ----------------------------------------------------------------------
