Repository: incubator-pirk Updated Branches: refs/heads/master 4abbefea7 -> bed244fe2
PIRK-80 Upgrade to Spark 2.0.0 - closes apache/incubator-pirk#113 Project: http://git-wip-us.apache.org/repos/asf/incubator-pirk/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-pirk/commit/bed244fe Tree: http://git-wip-us.apache.org/repos/asf/incubator-pirk/tree/bed244fe Diff: http://git-wip-us.apache.org/repos/asf/incubator-pirk/diff/bed244fe Branch: refs/heads/master Commit: bed244fe2facda1fc1eda1cb60e65a9d43b4d29d Parents: 4abbefe Author: eawilliams <eawilli...@apache.org> Authored: Thu Oct 27 10:21:46 2016 -0400 Committer: Jacob Wilder <jacobwil...@apache.org> Committed: Thu Oct 27 10:21:46 2016 -0400 ---------------------------------------------------------------------- pom.xml | 2 +- .../org/apache/pirk/responder/wideskies/spark/EncRowCalc.java | 5 +++-- .../responder/wideskies/spark/EncRowCalcPrecomputedCache.java | 5 +++-- .../pirk/responder/wideskies/spark/ExpKeyFilenameMap.java | 4 ++-- .../pirk/responder/wideskies/spark/ExpTableGenerator.java | 5 +++-- 5 files changed, 12 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/bed244fe/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 80e9a2b..88dff10 100644 --- a/pom.xml +++ b/pom.xml @@ -86,7 +86,7 @@ <junit.version>4.12</junit.version> <log4j.configuration>log4j2.xml</log4j.configuration> <hadoop.version>2.7.3</hadoop.version> - <spark.version>1.6.1</spark.version> + <spark.version>2.0.0</spark.version> <elasticsearch.version>2.3.4</elasticsearch.version> <storm.version>1.0.1</storm.version> <kafka.version>0.9.0.1</kafka.version> http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/bed244fe/src/main/java/org/apache/pirk/responder/wideskies/spark/EncRowCalc.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/responder/wideskies/spark/EncRowCalc.java b/src/main/java/org/apache/pirk/responder/wideskies/spark/EncRowCalc.java index f5a591e..e0b3b25 100644 --- a/src/main/java/org/apache/pirk/responder/wideskies/spark/EncRowCalc.java +++ b/src/main/java/org/apache/pirk/responder/wideskies/spark/EncRowCalc.java @@ -21,6 +21,7 @@ package org.apache.pirk.responder.wideskies.spark; import java.io.IOException; import java.math.BigInteger; import java.util.ArrayList; +import java.util.Iterator; import java.util.List; import org.apache.hadoop.conf.Configuration; @@ -71,7 +72,7 @@ public class EncRowCalc implements PairFlatMapFunction<Tuple2<Integer,Iterable<L } @Override - public Iterable<Tuple2<Long,BigInteger>> call(Tuple2<Integer,Iterable<List<BigInteger>>> hashDocTuple) throws Exception + public Iterator<Tuple2<Long,BigInteger>> call(Tuple2<Integer,Iterable<List<BigInteger>>> hashDocTuple) throws Exception { List<Tuple2<Long,BigInteger>> returnPairs = new ArrayList<>(); @@ -98,6 +99,6 @@ public class EncRowCalc implements PairFlatMapFunction<Tuple2<Integer,Iterable<L returnPairs.addAll(encRowValues); - return returnPairs; + return returnPairs.iterator(); } } http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/bed244fe/src/main/java/org/apache/pirk/responder/wideskies/spark/EncRowCalcPrecomputedCache.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/responder/wideskies/spark/EncRowCalcPrecomputedCache.java b/src/main/java/org/apache/pirk/responder/wideskies/spark/EncRowCalcPrecomputedCache.java index 8147ff6..5c1de76 100644 --- a/src/main/java/org/apache/pirk/responder/wideskies/spark/EncRowCalcPrecomputedCache.java +++ b/src/main/java/org/apache/pirk/responder/wideskies/spark/EncRowCalcPrecomputedCache.java @@ -21,6 +21,7 @@ package org.apache.pirk.responder.wideskies.spark; import java.math.BigInteger; import java.util.ArrayList; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import org.apache.pirk.query.wideskies.Query; @@ -65,7 +66,7 @@ public class EncRowCalcPrecomputedCache } @Override - public Iterable<Tuple2<Long,BigInteger>> call(Tuple2<Integer,Tuple2<Iterable<Tuple2<Integer,BigInteger>>,Iterable<List<BigInteger>>>> hashDocTuple) + public Iterator<Tuple2<Long, BigInteger>> call(Tuple2<Integer,Tuple2<Iterable<Tuple2<Integer,BigInteger>>,Iterable<List<BigInteger>>>> hashDocTuple) throws Exception { List<Tuple2<Long,BigInteger>> returnPairs = new ArrayList<>(); @@ -95,6 +96,6 @@ public class EncRowCalcPrecomputedCache returnPairs.addAll(encRowValues); - return returnPairs; + return returnPairs.iterator(); } } http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/bed244fe/src/main/java/org/apache/pirk/responder/wideskies/spark/ExpKeyFilenameMap.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/responder/wideskies/spark/ExpKeyFilenameMap.java b/src/main/java/org/apache/pirk/responder/wideskies/spark/ExpKeyFilenameMap.java index 09b1e52..03c80f4 100644 --- a/src/main/java/org/apache/pirk/responder/wideskies/spark/ExpKeyFilenameMap.java +++ b/src/main/java/org/apache/pirk/responder/wideskies/spark/ExpKeyFilenameMap.java @@ -52,7 +52,7 @@ public class ExpKeyFilenameMap implements PairFlatMapFunction<Iterator<Tuple2<In } @Override - public Iterable<Tuple2<Integer,String>> call(Iterator<Tuple2<Integer,Iterable<Tuple2<Integer,BigInteger>>>> iter) throws Exception + public Iterator<Tuple2<Integer, String>> call(Iterator<Tuple2<Integer,Iterable<Tuple2<Integer,BigInteger>>>> iter) throws Exception { List<Tuple2<Integer,String>> keyFileList = new ArrayList<>(); @@ -87,6 +87,6 @@ public class ExpKeyFilenameMap implements PairFlatMapFunction<Iterator<Tuple2<In } bw.close(); - return keyFileList; + return keyFileList.iterator(); } } http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/bed244fe/src/main/java/org/apache/pirk/responder/wideskies/spark/ExpTableGenerator.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/responder/wideskies/spark/ExpTableGenerator.java b/src/main/java/org/apache/pirk/responder/wideskies/spark/ExpTableGenerator.java index 4c54935..d14e36a 100644 --- a/src/main/java/org/apache/pirk/responder/wideskies/spark/ExpTableGenerator.java +++ b/src/main/java/org/apache/pirk/responder/wideskies/spark/ExpTableGenerator.java @@ -20,6 +20,7 @@ package org.apache.pirk.responder.wideskies.spark; import java.math.BigInteger; import java.util.ArrayList; +import java.util.Iterator; import java.util.List; import org.apache.pirk.encryption.ModPowAbstraction; @@ -51,7 +52,7 @@ public class ExpTableGenerator implements PairFlatMapFunction<Integer,Integer,Tu } @Override - public Iterable<Tuple2<Integer,Tuple2<Integer,BigInteger>>> call(Integer queryHashKey) throws Exception + public Iterator<Tuple2<Integer, Tuple2<Integer, BigInteger>>> call(Integer queryHashKey) throws Exception { // queryHashKey -> <<power>,<element^power mod N^2>> List<Tuple2<Integer,Tuple2<Integer,BigInteger>>> modExp = new ArrayList<>(); @@ -64,6 +65,6 @@ public class ExpTableGenerator implements PairFlatMapFunction<Integer,Integer,Tu modExp.add(new Tuple2<>(queryHashKey, modPowTuple)); } - return modExp; + return modExp.iterator(); } }
