Repository: beam Updated Branches: refs/heads/mr-runner 2fa4fdecd -> 5fa0b14d2
Initial commit for MapReduceRunner. Project: http://git-wip-us.apache.org/repos/asf/beam/repo Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/9fffd554 Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/9fffd554 Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/9fffd554 Branch: refs/heads/mr-runner Commit: 9fffd554f1e5fd6465989bb3568dfb6f2d854eeb Parents: f54072a Author: Pei He <p...@apache.org> Authored: Thu Jul 6 10:22:27 2017 +0800 Committer: Pei He <p...@apache.org> Committed: Mon Jul 24 20:15:42 2017 +0800 ---------------------------------------------------------------------- runners/map-reduce/pom.xml | 165 +++++++++++++++++++ .../mapreduce/MapReducePipelineOptions.java | 9 + .../runners/mapreduce/MapReduceRegistrar.java | 32 ++++ .../beam/runners/mapreduce/MapReduceRunner.java | 15 ++ .../MapReducePipelineTranslator.java | 11 ++ 5 files changed, 232 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/beam/blob/9fffd554/runners/map-reduce/pom.xml ---------------------------------------------------------------------- diff --git a/runners/map-reduce/pom.xml b/runners/map-reduce/pom.xml new file mode 100644 index 0000000..2e8a8c9 --- /dev/null +++ b/runners/map-reduce/pom.xml @@ -0,0 +1,165 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.beam</groupId> + <artifactId>beam-runners-parent</artifactId> + <version>2.1.0-SNAPSHOT</version> + <relativePath>../pom.xml</relativePath> + </parent> + + <artifactId>beam-runners-map-reduce</artifactId> + + <name>Apache Beam :: Runners :: MapReduce</name> + + <packaging>jar</packaging> + + <properties> + <mapreduce.version>2.8.1</mapreduce.version> + </properties> + + <profiles> + <profile> + <!-- This profile adds execution of ValidatesRunner integration tests + against a hadoop local cluster. --> + <id>local-validates-runner-tests</id> + <activation><activeByDefault>false</activeByDefault></activation> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <executions> + <execution> + <id>validates-runner-tests</id> + <phase>integration-test</phase> + <goals> + <goal>test</goal> + </goals> + <configuration> + <groups> + org.apache.beam.sdk.testing.ValidatesRunner + </groups> + <parallel>none</parallel> + <failIfNoTests>true</failIfNoTests> + <dependenciesToScan> + <dependency>org.apache.beam:beam-sdks-java-core</dependency> + </dependenciesToScan> + <systemPropertyVariables> + <beamTestPipelineOptions> + [ + "--runner=org.apache.beam.runners.mapreduce.MapReduceRunner" + ] + </beamTestPipelineOptions> + </systemPropertyVariables> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + </profiles> + + <dependencies> + <!-- MapRecue dependencies --> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-mapreduce-client-core</artifactId> + <version>${mapreduce.version}</version> + </dependency> + + <!-- Beam dependencies --> + <dependency> + <groupId>org.apache.beam</groupId> + <artifactId>beam-sdks-java-core</artifactId> + </dependency> + + <dependency> + <groupId>org.apache.beam</groupId> + <artifactId>beam-runners-core-java</artifactId> + </dependency> + + <dependency> + <groupId>org.apache.beam</groupId> + <artifactId>beam-runners-core-construction-java</artifactId> + </dependency> + + <!-- Module dependencies --> + <dependency> + <groupId>com.google.auto.service</groupId> + <artifactId>auto-service</artifactId> + <optional>true</optional> + </dependency> + <dependency> + <groupId>com.google.auto.value</groupId> + <artifactId>auto-value</artifactId> + </dependency> + + <!-- Depend on test jar to scan for ValidatesRunner tests --> + <dependency> + <groupId>org.apache.beam</groupId> + <artifactId>beam-sdks-java-core</artifactId> + <classifier>tests</classifier> + <scope>test</scope> + </dependency> + + <!-- transitive test dependencies from beam-sdk-java-core --> + <dependency> + <groupId>com.fasterxml.jackson.dataformat</groupId> + <artifactId>jackson-dataformat-yaml</artifactId> + <scope>test</scope> + </dependency> + + <!-- test dependencies --> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.hamcrest</groupId> + <artifactId>hamcrest-all</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-all</artifactId> + <scope>test</scope> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <executions> + <execution> + <goals><goal>analyze-only</goal></goals> + <configuration> + <!-- disable for now during runner development --> + <failOnWarning>false</failOnWarning> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> +</project> http://git-wip-us.apache.org/repos/asf/beam/blob/9fffd554/runners/map-reduce/src/main/java/org/apache/beam/runners/mapreduce/MapReducePipelineOptions.java ---------------------------------------------------------------------- diff --git a/runners/map-reduce/src/main/java/org/apache/beam/runners/mapreduce/MapReducePipelineOptions.java b/runners/map-reduce/src/main/java/org/apache/beam/runners/mapreduce/MapReducePipelineOptions.java new file mode 100644 index 0000000..da29931 --- /dev/null +++ b/runners/map-reduce/src/main/java/org/apache/beam/runners/mapreduce/MapReducePipelineOptions.java @@ -0,0 +1,9 @@ +package org.apache.beam.runners.mapreduce; + +import org.apache.beam.sdk.options.PipelineOptions; + +/** + * {@link PipelineOptions} for {@link MapReduceRunner}. + */ +public interface MapReducePipelineOptions extends PipelineOptions { +} http://git-wip-us.apache.org/repos/asf/beam/blob/9fffd554/runners/map-reduce/src/main/java/org/apache/beam/runners/mapreduce/MapReduceRegistrar.java ---------------------------------------------------------------------- diff --git a/runners/map-reduce/src/main/java/org/apache/beam/runners/mapreduce/MapReduceRegistrar.java b/runners/map-reduce/src/main/java/org/apache/beam/runners/mapreduce/MapReduceRegistrar.java new file mode 100644 index 0000000..eb960b8 --- /dev/null +++ b/runners/map-reduce/src/main/java/org/apache/beam/runners/mapreduce/MapReduceRegistrar.java @@ -0,0 +1,32 @@ +package org.apache.beam.runners.mapreduce; + +import com.google.auto.service.AutoService; +import com.google.common.collect.ImmutableList; +import org.apache.beam.sdk.PipelineRunner; +import org.apache.beam.sdk.options.PipelineOptions; +import org.apache.beam.sdk.options.PipelineOptionsRegistrar; +import org.apache.beam.sdk.runners.PipelineRunnerRegistrar; + +/** + * Registrars for {@link MapReduceRunner}. + */ +public class MapReduceRegistrar { + private MapReduceRegistrar() { + } + + @AutoService(PipelineRunnerRegistrar.class) + public static class Runner implements PipelineRunnerRegistrar { + @Override + public Iterable<Class<? extends PipelineRunner<?>>> getPipelineRunners() { + return ImmutableList.<Class<? extends PipelineRunner<?>>> of(MapReduceRunner.class); + } + } + + @AutoService(PipelineOptionsRegistrar.class) + public static class Options implements PipelineOptionsRegistrar { + @Override + public Iterable<Class<? extends PipelineOptions>> getPipelineOptions() { + return ImmutableList.<Class<? extends PipelineOptions>> of(MapReducePipelineOptions.class); + } + } +} http://git-wip-us.apache.org/repos/asf/beam/blob/9fffd554/runners/map-reduce/src/main/java/org/apache/beam/runners/mapreduce/MapReduceRunner.java ---------------------------------------------------------------------- diff --git a/runners/map-reduce/src/main/java/org/apache/beam/runners/mapreduce/MapReduceRunner.java b/runners/map-reduce/src/main/java/org/apache/beam/runners/mapreduce/MapReduceRunner.java new file mode 100644 index 0000000..bb9555e --- /dev/null +++ b/runners/map-reduce/src/main/java/org/apache/beam/runners/mapreduce/MapReduceRunner.java @@ -0,0 +1,15 @@ +package org.apache.beam.runners.mapreduce; + +import org.apache.beam.sdk.Pipeline; +import org.apache.beam.sdk.PipelineResult; +import org.apache.beam.sdk.PipelineRunner; + +/** + * {@link PipelineRunner} for crunch. + */ +public class MapReduceRunner extends PipelineRunner<PipelineResult> { + @Override + public PipelineResult run(Pipeline pipeline) { + return null; + } +} http://git-wip-us.apache.org/repos/asf/beam/blob/9fffd554/runners/map-reduce/src/main/java/org/apache/beam/runners/mapreduce/translation/MapReducePipelineTranslator.java ---------------------------------------------------------------------- diff --git a/runners/map-reduce/src/main/java/org/apache/beam/runners/mapreduce/translation/MapReducePipelineTranslator.java b/runners/map-reduce/src/main/java/org/apache/beam/runners/mapreduce/translation/MapReducePipelineTranslator.java new file mode 100644 index 0000000..b4a2e7c --- /dev/null +++ b/runners/map-reduce/src/main/java/org/apache/beam/runners/mapreduce/translation/MapReducePipelineTranslator.java @@ -0,0 +1,11 @@ +package org.apache.beam.runners.mapreduce.translation; + +import org.apache.beam.runners.mapreduce.MapReduceRunner; +import org.apache.beam.sdk.Pipeline; + +/** + * Pipeline translator for {@link MapReduceRunner}. + */ +public class MapReducePipelineTranslator extends Pipeline.PipelineVisitor.Defaults { + +}
