This closes #909
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/1ceb12ae Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/1ceb12ae Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/1ceb12ae Branch: refs/heads/master Commit: 1ceb12aebd0ffa63bd28d31cbe830230713705ec Parents: 5c23f49 0feb649 Author: Sela <ans...@paypal.com> Authored: Wed Sep 21 20:17:38 2016 +0300 Committer: Sela <ans...@paypal.com> Committed: Wed Sep 21 20:17:38 2016 +0300 ---------------------------------------------------------------------- .../runners/spark/SparkPipelineOptions.java | 28 +- .../apache/beam/runners/spark/SparkRunner.java | 121 ++-- .../spark/aggregators/AccumulatorSingleton.java | 53 ++ .../runners/spark/translation/DoFnFunction.java | 35 +- .../spark/translation/EvaluationContext.java | 17 +- .../translation/GroupCombineFunctions.java | 262 +++++++++ .../spark/translation/MultiDoFnFunction.java | 44 +- .../spark/translation/SparkContextFactory.java | 48 +- .../translation/SparkPipelineEvaluator.java | 57 -- .../translation/SparkPipelineTranslator.java | 5 +- .../spark/translation/SparkProcessContext.java | 10 +- .../spark/translation/SparkRuntimeContext.java | 44 +- .../spark/translation/TransformTranslator.java | 473 +++------------- .../spark/translation/TranslationUtils.java | 195 +++++++ .../SparkRunnerStreamingContextFactory.java | 98 ++++ .../streaming/StreamingEvaluationContext.java | 44 +- .../streaming/StreamingTransformTranslator.java | 549 ++++++++++++------- .../runners/spark/util/BroadcastHelper.java | 26 + .../runners/spark/ClearAggregatorsRule.java | 33 ++ .../beam/runners/spark/SimpleWordCountTest.java | 4 + .../spark/translation/SideEffectsTest.java | 3 +- .../streaming/FlattenStreamingTest.java | 54 +- .../streaming/KafkaStreamingTest.java | 26 +- .../RecoverFromCheckpointStreamingTest.java | 179 ++++++ .../streaming/SimpleStreamingWordCountTest.java | 25 +- .../utils/TestOptionsForStreaming.java | 55 ++ .../org/apache/beam/sdk/transforms/Combine.java | 7 + 27 files changed, 1682 insertions(+), 813 deletions(-) ----------------------------------------------------------------------
