[BEAM-572] Remove Spark Reference in WordCount
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/cf9ce2ff Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/cf9ce2ff Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/cf9ce2ff Branch: refs/heads/gearpump-runner Commit: cf9ce2ff27dfe13715119c5a9c04900f0e5e6e93 Parents: 435054b Author: Mark Liu <mark...@markliu-macbookpro.roam.corp.google.com> Authored: Fri Aug 26 16:13:17 2016 -0700 Committer: Dan Halperin <dhalp...@google.com> Committed: Mon Sep 12 17:40:11 2016 -0700 ---------------------------------------------------------------------- .../org/apache/beam/examples/WordCount.java | 24 +------------------- .../org/apache/beam/examples/WordCountIT.java | 9 ++++++++ 2 files changed, 10 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/cf9ce2ff/examples/java/src/main/java/org/apache/beam/examples/WordCount.java ---------------------------------------------------------------------- diff --git a/examples/java/src/main/java/org/apache/beam/examples/WordCount.java b/examples/java/src/main/java/org/apache/beam/examples/WordCount.java index 0275651..793ee4b 100644 --- a/examples/java/src/main/java/org/apache/beam/examples/WordCount.java +++ b/examples/java/src/main/java/org/apache/beam/examples/WordCount.java @@ -18,7 +18,6 @@ package org.apache.beam.examples; import com.google.common.base.Strings; -import com.google.common.io.Resources; import java.io.IOException; import org.apache.beam.sdk.Pipeline; import org.apache.beam.sdk.io.TextIO; @@ -165,7 +164,7 @@ public class WordCount { */ public static interface WordCountOptions extends PipelineOptions { @Description("Path of the file to read from") - @Default.InstanceFactory(InputFactory.class) + @Default.String("gs://apache-beam-samples/shakespeare/kinglear.txt") String getInputFile(); void setInputFile(String value); @@ -194,27 +193,6 @@ public class WordCount { } } } - - /** - * Return default input file path according to runner type. - * - * <p><ul> - * <li>SparkRunner: - * .../src/test/resources/LICENSE</li> - * <li>other runners: - * gs://apache-beam-samples/apache/LICENSE</li> - * </ul> - */ - public static class InputFactory implements DefaultValueFactory<String> { - @Override - public String create(PipelineOptions options) { - if (options.getRunner().getName().contains("SparkRunner")) { - return Resources.getResource("LICENSE").getPath(); - } else { - return "gs://apache-beam-samples/apache/LICENSE"; - } - } - } } public static void main(String[] args) { http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/cf9ce2ff/examples/java/src/test/java/org/apache/beam/examples/WordCountIT.java ---------------------------------------------------------------------- diff --git a/examples/java/src/test/java/org/apache/beam/examples/WordCountIT.java b/examples/java/src/test/java/org/apache/beam/examples/WordCountIT.java index ca0c9d6..b0e0fe0 100644 --- a/examples/java/src/test/java/org/apache/beam/examples/WordCountIT.java +++ b/examples/java/src/test/java/org/apache/beam/examples/WordCountIT.java @@ -18,6 +18,7 @@ package org.apache.beam.examples; +import com.google.common.io.Resources; import java.util.Date; import org.apache.beam.examples.WordCount.WordCountOptions; import org.apache.beam.sdk.options.Default; @@ -61,6 +62,14 @@ public class WordCountIT { options.setOnSuccessMatcher( new FileChecksumMatcher(options.getOutputChecksum(), options.getOutput() + "*")); + String e2eTestInputPath = "gs://apache-beam-samples/apache/LICENSE"; + // Spark runner currently doesn't support GCS I/O, change default input to: + // .../src/test/resources/LICENSE + if (options.getRunner().getName().contains("SparkRunner")) { + e2eTestInputPath = Resources.getResource("LICENSE").getPath(); + } + options.setInputFile(e2eTestInputPath); + WordCount.main(TestPipeline.convertToArgs(options)); } }
