GORA-386 Gora Spark Backend Support addition to make forbidden api compliant
Project: http://git-wip-us.apache.org/repos/asf/gora/repo Commit: http://git-wip-us.apache.org/repos/asf/gora/commit/2ce2fda5 Tree: http://git-wip-us.apache.org/repos/asf/gora/tree/2ce2fda5 Diff: http://git-wip-us.apache.org/repos/asf/gora/diff/2ce2fda5 Branch: refs/heads/master Commit: 2ce2fda5b7f688f1c5fa9e10f7c8ff64ce99be12 Parents: 965c449 Author: Lewis John McGibbney <[email protected]> Authored: Tue Sep 1 15:26:59 2015 -0700 Committer: Lewis John McGibbney <[email protected]> Committed: Tue Sep 1 15:26:59 2015 -0700 ---------------------------------------------------------------------- .../apache/gora/examples/spark/SparkWordCount.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/gora/blob/2ce2fda5/gora-core/src/examples/java/org/apache/gora/examples/spark/SparkWordCount.java ---------------------------------------------------------------------- diff --git a/gora-core/src/examples/java/org/apache/gora/examples/spark/SparkWordCount.java b/gora-core/src/examples/java/org/apache/gora/examples/spark/SparkWordCount.java index 0a84d58..4f91fb2 100644 --- a/gora-core/src/examples/java/org/apache/gora/examples/spark/SparkWordCount.java +++ b/gora-core/src/examples/java/org/apache/gora/examples/spark/SparkWordCount.java @@ -17,6 +17,8 @@ */ package org.apache.gora.examples.spark; +import java.nio.charset.Charset; + import org.apache.gora.examples.generated.TokenDatum; import org.apache.gora.examples.generated.WebPage; import org.apache.gora.spark.GoraSparkEngine; @@ -52,7 +54,7 @@ public class SparkWordCount { @Override public Tuple2<String, Long> call(WebPage webPage) throws Exception { - String content = new String(webPage.getContent().array()); + String content = new String(webPage.getContent().array(), Charset.defaultCharset()); return new Tuple2<>(content, 1L); } }; @@ -86,20 +88,20 @@ public class SparkWordCount { JavaPairRDD<String, WebPage> goraRDD = goraSparkEngine.initialize(sc, inStore); long count = goraRDD.count(); - System.out.println("Total Web page count: " + count); + log.info("Total Web page count: {}", count); JavaRDD<Tuple2<String, Long>> mappedGoraRdd = goraRDD.values().map(mapFunc); JavaPairRDD<String, Long> reducedGoraRdd = JavaPairRDD.fromJavaRDD(mappedGoraRdd).reduceByKey(redFunc); //Print output for debug purpose - System.out.println("SparkWordCount debug purpose TokenDatum print starts:"); + log.info("SparkWordCount debug purpose TokenDatum print starts:"); Map<String, Long> tokenDatumMap = reducedGoraRdd.collectAsMap(); for (String key : tokenDatumMap.keySet()) { - System.out.println(key); - System.out.println(tokenDatumMap.get(key)); + log.info(key); + log.info(tokenDatumMap.get(key).toString()); } - System.out.println("SparkWordCount debug purpose TokenDatum print ends:"); + log.info("SparkWordCount debug purpose TokenDatum print ends:"); // //write output to datastore @@ -134,7 +136,7 @@ public class SparkWordCount { public static void main(String[] args) throws Exception { if (args.length < 2) { - System.err.println(USAGE); + log.info(USAGE); System.exit(1); }
