This closes #3677: [BEAM-2709] Add TezRunner
Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/da4687a6 Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/da4687a6 Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/da4687a6 Branch: refs/heads/tez-runner Commit: da4687a6eaa8b0cc5914c5762c2325766872287a Parents: f10399d 8994f07 Author: Kenneth Knowles <[email protected]> Authored: Fri Nov 17 12:34:17 2017 -0800 Committer: Kenneth Knowles <[email protected]> Committed: Fri Nov 17 12:34:17 2017 -0800 ---------------------------------------------------------------------- runners/pom.xml | 1 + runners/tez/pom.xml | 135 +++++++++++++++ .../beam/runners/tez/TezPipelineOptions.java | 27 +++ .../org/apache/beam/runners/tez/TezRunner.java | 79 +++++++++ .../beam/runners/tez/TezRunnerResult.java | 78 +++++++++ .../FlattenPCollectionTranslator.java | 34 ++++ .../tez/translation/GroupByKeyTranslator.java | 43 +++++ .../tez/translation/ParDoTranslator.java | 92 ++++++++++ .../tez/translation/ReadBoundedTranslator.java | 43 +++++ .../tez/translation/TezDoFnProcessor.java | 131 +++++++++++++++ .../tez/translation/TezPipelineTranslator.java | 139 ++++++++++++++++ .../tez/translation/TransformTranslator.java | 28 ++++ .../tez/translation/TranslationContext.java | 166 +++++++++++++++++++ .../runners/tez/translation/TranslatorUtil.java | 147 ++++++++++++++++ .../ViewCreatePCollectionViewTranslator.java | 35 ++++ .../tez/translation/WindowAssignTranslator.java | 35 ++++ .../tez/translation/WriteFilesTranslator.java | 45 +++++ .../tez/translation/io/MROutputManager.java | 67 ++++++++ .../tez/translation/io/NoOpOutputManager.java | 35 ++++ .../io/OrderedPartitionedKVOutputManager.java | 62 +++++++ .../translation/io/OutputManagerFactory.java | 39 +++++ .../tez/translation/io/TezOutputManager.java | 62 +++++++ .../io/UnorderedKVEdgeOutputManager.java | 57 +++++++ .../apache/beam/runners/tez/TezRunnerTest.java | 155 +++++++++++++++++ .../tez/translation/ParDoTranslatorTest.java | 120 ++++++++++++++ .../tez/translation/TezDoFnProcessorTest.java | 112 +++++++++++++ .../tez/translation/TranslationContextTest.java | 110 ++++++++++++ .../tez/translation/TranslatorUtilTest.java | 43 +++++ runners/tez/src/test/resources/test_input.txt | 2 + 29 files changed, 2122 insertions(+) ----------------------------------------------------------------------
