This closes #42
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/a91e1150 Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/a91e1150 Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/a91e1150 Branch: refs/heads/master Commit: a91e1150084808202e382f169eb273e5b52bbc97 Parents: 46412e5 b49e3c9 Author: Sela <[email protected]> Authored: Tue Mar 15 20:41:13 2016 +0200 Committer: Sela <[email protected]> Committed: Tue Mar 15 20:41:13 2016 +0200 ---------------------------------------------------------------------- README.md | 7 +- runners/pom.xml | 1 + runners/spark/.gitignore | 10 - runners/spark/.travis.yml | 22 - runners/spark/build-resources/checkstyle.xml | 27 +- runners/spark/build-resources/header-file.txt | 23 +- runners/spark/pom.xml | 246 +++--- .../com/cloudera/dataflow/hadoop/HadoopIO.java | 202 ----- .../dataflow/hadoop/NullWritableCoder.java | 71 -- .../cloudera/dataflow/hadoop/WritableCoder.java | 120 --- .../com/cloudera/dataflow/io/ConsoleIO.java | 60 -- .../com/cloudera/dataflow/io/CreateStream.java | 66 -- .../java/com/cloudera/dataflow/io/KafkaIO.java | 128 --- .../dataflow/spark/BroadcastHelper.java | 121 --- .../com/cloudera/dataflow/spark/ByteArray.java | 52 -- .../cloudera/dataflow/spark/CoderHelpers.java | 185 ----- .../cloudera/dataflow/spark/DoFnFunction.java | 93 --- .../dataflow/spark/EvaluationContext.java | 283 ------- .../dataflow/spark/EvaluationResult.java | 62 -- .../dataflow/spark/MultiDoFnFunction.java | 115 --- .../dataflow/spark/ShardNameBuilder.java | 106 --- .../dataflow/spark/ShardNameTemplateAware.java | 28 - .../dataflow/spark/ShardNameTemplateHelper.java | 58 -- .../dataflow/spark/SparkContextFactory.java | 66 -- .../dataflow/spark/SparkPipelineEvaluator.java | 52 -- .../dataflow/spark/SparkPipelineOptions.java | 39 - .../spark/SparkPipelineOptionsFactory.java | 27 - .../spark/SparkPipelineOptionsRegistrar.java | 27 - .../dataflow/spark/SparkPipelineRunner.java | 252 ------ .../spark/SparkPipelineRunnerRegistrar.java | 27 - .../dataflow/spark/SparkPipelineTranslator.java | 27 - .../dataflow/spark/SparkProcessContext.java | 250 ------ .../dataflow/spark/SparkRuntimeContext.java | 212 ----- .../spark/TemplatedAvroKeyOutputFormat.java | 40 - .../TemplatedSequenceFileOutputFormat.java | 40 - .../spark/TemplatedTextOutputFormat.java | 40 - .../dataflow/spark/TransformEvaluator.java | 24 - .../dataflow/spark/TransformTranslator.java | 800 ------------------ .../dataflow/spark/WindowingHelpers.java | 59 -- .../spark/aggregators/AggAccumParam.java | 35 - .../spark/aggregators/NamedAggregators.java | 202 ----- .../SparkStreamingPipelineOptions.java | 40 - .../SparkStreamingPipelineOptionsFactory.java | 27 - .../SparkStreamingPipelineOptionsRegistrar.java | 28 - .../streaming/StreamingEvaluationContext.java | 226 ------ .../streaming/StreamingTransformTranslator.java | 414 ---------- .../StreamingWindowPipelineDetector.java | 100 --- .../beam/runners/spark/EvaluationResult.java | 65 ++ .../runners/spark/SparkPipelineOptions.java | 42 + .../beam/runners/spark/SparkPipelineRunner.java | 255 ++++++ .../spark/SparkStreamingPipelineOptions.java | 41 + .../spark/aggregators/AggAccumParam.java | 38 + .../spark/aggregators/NamedAggregators.java | 205 +++++ .../beam/runners/spark/coders/CoderHelpers.java | 189 +++++ .../runners/spark/coders/NullWritableCoder.java | 74 ++ .../runners/spark/coders/WritableCoder.java | 123 +++ .../apache/beam/runners/spark/io/ConsoleIO.java | 63 ++ .../beam/runners/spark/io/CreateStream.java | 69 ++ .../apache/beam/runners/spark/io/KafkaIO.java | 131 +++ .../beam/runners/spark/io/hadoop/HadoopIO.java | 203 +++++ .../spark/io/hadoop/ShardNameBuilder.java | 109 +++ .../spark/io/hadoop/ShardNameTemplateAware.java | 31 + .../io/hadoop/ShardNameTemplateHelper.java | 61 ++ .../io/hadoop/TemplatedAvroKeyOutputFormat.java | 43 + .../TemplatedSequenceFileOutputFormat.java | 43 + .../io/hadoop/TemplatedTextOutputFormat.java | 43 + .../runners/spark/translation/DoFnFunction.java | 97 +++ .../spark/translation/EvaluationContext.java | 288 +++++++ .../spark/translation/MultiDoFnFunction.java | 119 +++ .../spark/translation/SparkContextFactory.java | 69 ++ .../translation/SparkPipelineEvaluator.java | 56 ++ .../SparkPipelineOptionsFactory.java | 31 + .../SparkPipelineOptionsRegistrar.java | 31 + .../SparkPipelineRunnerRegistrar.java | 31 + .../translation/SparkPipelineTranslator.java | 30 + .../spark/translation/SparkProcessContext.java | 262 ++++++ .../spark/translation/SparkRuntimeContext.java | 217 +++++ .../spark/translation/TransformEvaluator.java | 27 + .../spark/translation/TransformTranslator.java | 808 +++++++++++++++++++ .../spark/translation/WindowingHelpers.java | 62 ++ .../SparkStreamingPipelineOptionsFactory.java | 31 + .../SparkStreamingPipelineOptionsRegistrar.java | 32 + .../streaming/StreamingEvaluationContext.java | 229 ++++++ .../streaming/StreamingTransformTranslator.java | 418 ++++++++++ .../StreamingWindowPipelineDetector.java | 104 +++ .../runners/spark/util/BroadcastHelper.java | 125 +++ .../beam/runners/spark/util/ByteArray.java | 55 ++ ...ataflow.sdk.options.PipelineOptionsRegistrar | 4 +- ...dataflow.sdk.runners.PipelineRunnerRegistrar | 2 +- .../dataflow/hadoop/WritableCoderTest.java | 42 - .../dataflow/spark/AvroPipelineTest.java | 103 --- .../dataflow/spark/CombineGloballyTest.java | 87 -- .../dataflow/spark/CombinePerKeyTest.java | 69 -- .../com/cloudera/dataflow/spark/DeDupTest.java | 55 -- .../cloudera/dataflow/spark/DoFnOutputTest.java | 57 -- .../cloudera/dataflow/spark/EmptyInputTest.java | 64 -- .../spark/HadoopFileFormatPipelineTest.java | 105 --- .../spark/MultiOutputWordCountTest.java | 148 ---- .../cloudera/dataflow/spark/NumShardsTest.java | 89 -- .../dataflow/spark/SerializationTest.java | 183 ----- .../dataflow/spark/ShardNameBuilderTest.java | 82 -- .../dataflow/spark/SideEffectsTest.java | 77 -- .../dataflow/spark/SimpleWordCountTest.java | 117 --- .../spark/TestSparkPipelineOptionsFactory.java | 34 - .../com/cloudera/dataflow/spark/TfIdfTest.java | 60 -- .../dataflow/spark/TransformTranslatorTest.java | 95 --- .../dataflow/spark/WindowedWordCountTest.java | 63 -- .../spark/streaming/FlattenStreamingTest.java | 84 -- .../spark/streaming/KafkaStreamingTest.java | 133 --- .../streaming/SimpleStreamingWordCountTest.java | 73 -- .../utils/DataflowAssertStreaming.java | 39 - .../streaming/utils/EmbeddedKafkaCluster.java | 314 ------- .../apache/beam/runners/spark/DeDupTest.java | 60 ++ .../beam/runners/spark/EmptyInputTest.java | 69 ++ .../beam/runners/spark/SimpleWordCountTest.java | 115 +++ .../apache/beam/runners/spark/TfIdfTest.java | 64 ++ .../runners/spark/coders/WritableCoderTest.java | 45 ++ .../beam/runners/spark/io/AvroPipelineTest.java | 108 +++ .../beam/runners/spark/io/NumShardsTest.java | 96 +++ .../io/hadoop/HadoopFileFormatPipelineTest.java | 113 +++ .../spark/io/hadoop/ShardNameBuilderTest.java | 85 ++ .../spark/translation/CombineGloballyTest.java | 94 +++ .../spark/translation/CombinePerKeyTest.java | 70 ++ .../spark/translation/DoFnOutputTest.java | 64 ++ .../translation/MultiOutputWordCountTest.java | 137 ++++ .../spark/translation/SerializationTest.java | 183 +++++ .../spark/translation/SideEffectsTest.java | 81 ++ .../TestSparkPipelineOptionsFactory.java | 38 + .../translation/TransformTranslatorTest.java | 99 +++ .../translation/WindowedWordCountTest.java | 71 ++ .../streaming/FlattenStreamingTest.java | 88 ++ .../streaming/KafkaStreamingTest.java | 140 ++++ .../streaming/SimpleStreamingWordCountTest.java | 77 ++ .../utils/DataflowAssertStreaming.java | 42 + .../streaming/utils/EmbeddedKafkaCluster.java | 317 ++++++++ 135 files changed, 7362 insertions(+), 7163 deletions(-) ----------------------------------------------------------------------
