PIRK-4 Add Streaming Implementation for Apache Storm - closes apache/incubator-pirk#4
Project: http://git-wip-us.apache.org/repos/asf/incubator-pirk/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-pirk/commit/beb69e3f Tree: http://git-wip-us.apache.org/repos/asf/incubator-pirk/tree/beb69e3f Diff: http://git-wip-us.apache.org/repos/asf/incubator-pirk/diff/beb69e3f Branch: refs/heads/master Commit: beb69e3f47b204f3e38e67ffe70d12d8391d899c Parents: d70b816 Author: charris <[email protected]> Authored: Mon Aug 29 01:05:45 2016 -0400 Committer: charris <[email protected]> Committed: Mon Aug 29 01:05:45 2016 -0400 ---------------------------------------------------------------------- .../pirk/schema/data/LoadDataSchemas.html | 335 ----- pom-with-benchmarks.xml | 160 +- pom.xml | 1394 ++++++++++-------- .../org/apache/pirk/query/wideskies/Query.java | 2 +- .../apache/pirk/query/wideskies/QueryInfo.java | 50 +- .../responder/wideskies/ResponderDriver.java | 97 +- .../responder/wideskies/ResponderProps.java | 45 +- .../wideskies/common/ComputeEncryptedRow.java | 112 +- .../common/HashSelectorAndPartitionData.java | 4 +- .../wideskies/spark/ComputeResponse.java | 5 +- .../wideskies/storm/EncColMultBolt.java | 130 ++ .../wideskies/storm/EncRowCalcBolt.java | 243 +++ .../responder/wideskies/storm/OutputBolt.java | 200 +++ .../wideskies/storm/PartitionDataBolt.java | 136 ++ .../wideskies/storm/PirkHashScheme.java | 105 ++ .../responder/wideskies/storm/PirkTopology.java | 173 +++ .../wideskies/storm/StormConstants.java | 69 + .../responder/wideskies/storm/StormUtils.java | 138 ++ .../pirk/schema/data/DataSchemaLoader.java | 4 +- .../org/apache/pirk/test/utils/BaseTests.java | 11 +- src/main/resources/log4j2.xml | 2 +- src/main/resources/pirk.properties | 6 - src/main/resources/responder.properties | 45 +- .../pirk/storm/KafkaStormIntegrationTest.java | 335 +++++ .../java/org/apache/pirk/storm/SystemTime.java | 50 + 25 files changed, 2706 insertions(+), 1145 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/beb69e3f/docs/org/apache/pirk/schema/data/LoadDataSchemas.html ---------------------------------------------------------------------- diff --git a/docs/org/apache/pirk/schema/data/LoadDataSchemas.html b/docs/org/apache/pirk/schema/data/LoadDataSchemas.html deleted file mode 100644 index 99649fe..0000000 --- a/docs/org/apache/pirk/schema/data/LoadDataSchemas.html +++ /dev/null @@ -1,335 +0,0 @@ -<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd";> -<!-- NewPage --> -<html lang="en"> -<head> -<!-- Generated by javadoc (version 1.7.0_80) on Sun Jul 24 11:37:26 EDT 2016 --> -<title>LoadDataSchemas</title> -<meta name="date" content="2016-07-24"> -<link rel="stylesheet" type="text/css" href="../../../../../stylesheet.css" title="Style"> -</head> -<body> -<script type="text/javascript"><!-- - if (location.href.indexOf('is-external=true') == -1) { - parent.document.title="LoadDataSchemas"; - } -//--> -</script> -<noscript> -<div>JavaScript is disabled on your browser.</div> -</noscript> -<!-- ========= START OF TOP NAVBAR ======= --> -<div class="topNav"><a name="navbar_top"> -<!-- --> -</a><a href="#skip-navbar_top" title="Skip navigation links"></a><a name="navbar_top_firstrow"> -<!-- --> -</a> -<ul class="navList" title="Navigation"> -<li><a href="../../../../../overview-summary.html">Overview</a></li> -<li><a href="package-summary.html">Package</a></li> -<li class="navBarCell1Rev">Class</li> -<li><a href="class-use/LoadDataSchemas.html">Use</a></li> -<li><a href="package-tree.html">Tree</a></li> -<li><a href="../../../../../deprecated-list.html">Deprecated</a></li> -<li><a href="../../../../../index-files/index-1.html">Index</a></li> -<li><a href="../../../../../help-doc.html">Help</a></li> -</ul> -</div> -<div class="subNav"> -<ul class="navList"> -<li><a href="../../../../../org/apache/pirk/schema/data/DataSchema.html" title="class in org.apache.pirk.schema.data"><span class="strong">Prev Class</span></a></li> -<li>Next Class</li> -</ul> -<ul class="navList"> -<li><a href="../../../../../index.html?org/apache/pirk/schema/data/LoadDataSchemas.html" target="_top">Frames</a></li> -<li><a href="LoadDataSchemas.html" target="_top">No Frames</a></li> -</ul> -<ul class="navList" id="allclasses_navbar_top"> -<li><a href="../../../../../allclasses-noframe.html">All Classes</a></li> -</ul> -<div> -<script type="text/javascript"><!-- - allClassesLink = document.getElementById("allclasses_navbar_top"); - if(window==top) { - allClassesLink.style.display = "block"; - } - else { - allClassesLink.style.display = "none"; - } - //--> -</script> -</div> -<div> -<ul class="subNavList"> -<li>Summary: </li> -<li>Nested | </li> -<li>Field | </li> -<li><a href="#constructor_summary">Constr</a> | </li> -<li><a href="#method_summary">Method</a></li> -</ul> -<ul class="subNavList"> -<li>Detail: </li> -<li>Field | </li> -<li><a href="#constructor_detail">Constr</a> | </li> -<li><a href="#method_detail">Method</a></li> -</ul> -</div> -<a name="skip-navbar_top"> -<!-- --> -</a></div> -<!-- ========= END OF TOP NAVBAR ========= --> -<!-- ======== START OF CLASS DATA ======== --> -<div class="header"> -<div class="subTitle">org.apache.pirk.schema.data</div> -<h2 title="Class LoadDataSchemas" class="title">Class LoadDataSchemas</h2> -</div> -<div class="contentContainer"> -<ul class="inheritance"> -<li>java.lang.Object</li> -<li> -<ul class="inheritance"> -<li>org.apache.pirk.schema.data.LoadDataSchemas</li> -</ul> -</li> -</ul> -<div class="description"> -<ul class="blockList"> -<li class="blockList"> -<hr> -<br> -<pre>public class <span class="strong">LoadDataSchemas</span> -extends java.lang.Object</pre> -<div class="block">Class to load any data schemas specified in the properties file, 'data.schemas' - <p> - Schemas should be specified as follows; all items are treated in a case insensitive manner: - - <pre> - <code><schema> - <schemaName> name of the schema </schemaName> - <element> - <name> element name /name> - <type> class name or type name (if Java primitive type) of the element </type> - <isArray> true or false -- whether or not the schema element is an array within the data </isArray> - <partitioner> optional - Partitioner class for the element; defaults to primitive java type partitioner </partitioner> - </element> - </schema> - </code> - </pre> - - Primitive types must be one of the following: "byte", "short", "int", "long", "float", "double", "char", "string" - <p></div> -</li> -</ul> -</div> -<div class="summary"> -<ul class="blockList"> -<li class="blockList"> -<!-- ======== CONSTRUCTOR SUMMARY ======== --> -<ul class="blockList"> -<li class="blockList"><a name="constructor_summary"> -<!-- --> -</a> -<h3>Constructor Summary</h3> -<table class="overviewSummary" border="0" cellpadding="3" cellspacing="0" summary="Constructor Summary table, listing constructors, and an explanation"> -<caption><span>Constructors</span><span class="tabEnd"> </span></caption> -<tr> -<th class="colOne" scope="col">Constructor and Description</th> -</tr> -<tr class="altColor"> -<td class="colOne"><code><strong><a href="../../../../../org/apache/pirk/schema/data/LoadDataSchemas.html#LoadDataSchemas()">LoadDataSchemas</a></strong>()</code> </td> -</tr> -</table> -</li> -</ul> -<!-- ========== METHOD SUMMARY =========== --> -<ul class="blockList"> -<li class="blockList"><a name="method_summary"> -<!-- --> -</a> -<h3>Method Summary</h3> -<table class="overviewSummary" border="0" cellpadding="3" cellspacing="0" summary="Method Summary table, listing methods, and an explanation"> -<caption><span>Methods</span><span class="tabEnd"> </span></caption> -<tr> -<th class="colFirst" scope="col">Modifier and Type</th> -<th class="colLast" scope="col">Method and Description</th> -</tr> -<tr class="altColor"> -<td class="colFirst"><code>static <a href="../../../../../org/apache/pirk/schema/data/DataSchema.html" title="class in org.apache.pirk.schema.data">DataSchema</a></code></td> -<td class="colLast"><code><strong><a href="../../../../../org/apache/pirk/schema/data/LoadDataSchemas.html#getSchema(java.lang.String)">getSchema</a></strong>(java.lang.String schemaName)</code> </td> -</tr> -<tr class="rowColor"> -<td class="colFirst"><code>static java.util.HashMap<java.lang.String,<a href="../../../../../org/apache/pirk/schema/data/DataSchema.html" title="class in org.apache.pirk.schema.data">DataSchema</a>></code></td> -<td class="colLast"><code><strong><a href="../../../../../org/apache/pirk/schema/data/LoadDataSchemas.html#getSchemaMap()">getSchemaMap</a></strong>()</code> </td> -</tr> -<tr class="altColor"> -<td class="colFirst"><code>static java.util.Set<java.lang.String></code></td> -<td class="colLast"><code><strong><a href="../../../../../org/apache/pirk/schema/data/LoadDataSchemas.html#getSchemaNames()">getSchemaNames</a></strong>()</code> </td> -</tr> -<tr class="rowColor"> -<td class="colFirst"><code>static void</code></td> -<td class="colLast"><code><strong><a href="../../../../../org/apache/pirk/schema/data/LoadDataSchemas.html#initialize()">initialize</a></strong>()</code> </td> -</tr> -<tr class="altColor"> -<td class="colFirst"><code>static void</code></td> -<td class="colLast"><code><strong><a href="../../../../../org/apache/pirk/schema/data/LoadDataSchemas.html#initialize(boolean,%20org.apache.hadoop.fs.FileSystem)">initialize</a></strong>(boolean hdfs, - org.apache.hadoop.fs.FileSystem fs)</code> </td> -</tr> -</table> -<ul class="blockList"> -<li class="blockList"><a name="methods_inherited_from_class_java.lang.Object"> -<!-- --> -</a> -<h3>Methods inherited from class java.lang.Object</h3> -<code>equals, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait</code></li> -</ul> -</li> -</ul> -</li> -</ul> -</div> -<div class="details"> -<ul class="blockList"> -<li class="blockList"> -<!-- ========= CONSTRUCTOR DETAIL ======== --> -<ul class="blockList"> -<li class="blockList"><a name="constructor_detail"> -<!-- --> -</a> -<h3>Constructor Detail</h3> -<a name="LoadDataSchemas()"> -<!-- --> -</a> -<ul class="blockListLast"> -<li class="blockList"> -<h4>LoadDataSchemas</h4> -<pre>public LoadDataSchemas()</pre> -</li> -</ul> -</li> -</ul> -<!-- ============ METHOD DETAIL ========== --> -<ul class="blockList"> -<li class="blockList"><a name="method_detail"> -<!-- --> -</a> -<h3>Method Detail</h3> -<a name="initialize()"> -<!-- --> -</a> -<ul class="blockList"> -<li class="blockList"> -<h4>initialize</h4> -<pre>public static void initialize() - throws java.lang.Exception</pre> -<dl><dt><span class="strong">Throws:</span></dt> -<dd><code>java.lang.Exception</code></dd></dl> -</li> -</ul> -<a name="initialize(boolean, org.apache.hadoop.fs.FileSystem)"> -<!-- --> -</a> -<ul class="blockList"> -<li class="blockList"> -<h4>initialize</h4> -<pre>public static void initialize(boolean hdfs, - org.apache.hadoop.fs.FileSystem fs) - throws java.lang.Exception</pre> -<dl><dt><span class="strong">Throws:</span></dt> -<dd><code>java.lang.Exception</code></dd></dl> -</li> -</ul> -<a name="getSchemaMap()"> -<!-- --> -</a> -<ul class="blockList"> -<li class="blockList"> -<h4>getSchemaMap</h4> -<pre>public static java.util.HashMap<java.lang.String,<a href="../../../../../org/apache/pirk/schema/data/DataSchema.html" title="class in org.apache.pirk.schema.data">DataSchema</a>> getSchemaMap()</pre> -</li> -</ul> -<a name="getSchemaNames()"> -<!-- --> -</a> -<ul class="blockList"> -<li class="blockList"> -<h4>getSchemaNames</h4> -<pre>public static java.util.Set<java.lang.String> getSchemaNames()</pre> -</li> -</ul> -<a name="getSchema(java.lang.String)"> -<!-- --> -</a> -<ul class="blockListLast"> -<li class="blockList"> -<h4>getSchema</h4> -<pre>public static <a href="../../../../../org/apache/pirk/schema/data/DataSchema.html" title="class in org.apache.pirk.schema.data">DataSchema</a> getSchema(java.lang.String schemaName)</pre> -</li> -</ul> -</li> -</ul> -</li> -</ul> -</div> -</div> -<!-- ========= END OF CLASS DATA ========= --> -<!-- ======= START OF BOTTOM NAVBAR ====== --> -<div class="bottomNav"><a name="navbar_bottom"> -<!-- --> -</a><a href="#skip-navbar_bottom" title="Skip navigation links"></a><a name="navbar_bottom_firstrow"> -<!-- --> -</a> -<ul class="navList" title="Navigation"> -<li><a href="../../../../../overview-summary.html">Overview</a></li> -<li><a href="package-summary.html">Package</a></li> -<li class="navBarCell1Rev">Class</li> -<li><a href="class-use/LoadDataSchemas.html">Use</a></li> -<li><a href="package-tree.html">Tree</a></li> -<li><a href="../../../../../deprecated-list.html">Deprecated</a></li> -<li><a href="../../../../../index-files/index-1.html">Index</a></li> -<li><a href="../../../../../help-doc.html">Help</a></li> -</ul> -</div> -<div class="subNav"> -<ul class="navList"> -<li><a href="../../../../../org/apache/pirk/schema/data/DataSchema.html" title="class in org.apache.pirk.schema.data"><span class="strong">Prev Class</span></a></li> -<li>Next Class</li> -</ul> -<ul class="navList"> -<li><a href="../../../../../index.html?org/apache/pirk/schema/data/LoadDataSchemas.html" target="_top">Frames</a></li> -<li><a href="LoadDataSchemas.html" target="_top">No Frames</a></li> -</ul> -<ul class="navList" id="allclasses_navbar_bottom"> -<li><a href="../../../../../allclasses-noframe.html">All Classes</a></li> -</ul> -<div> -<script type="text/javascript"><!-- - allClassesLink = document.getElementById("allclasses_navbar_bottom"); - if(window==top) { - allClassesLink.style.display = "block"; - } - else { - allClassesLink.style.display = "none"; - } - //--> -</script> -</div> -<div> -<ul class="subNavList"> -<li>Summary: </li> -<li>Nested | </li> -<li>Field | </li> -<li><a href="#constructor_summary">Constr</a> | </li> -<li><a href="#method_summary">Method</a></li> -</ul> -<ul class="subNavList"> -<li>Detail: </li> -<li>Field | </li> -<li><a href="#constructor_detail">Constr</a> | </li> -<li><a href="#method_detail">Method</a></li> -</ul> -</div> -<a name="skip-navbar_bottom"> -<!-- --> -</a></div> -<!-- ======== END OF BOTTOM NAVBAR ======= --> -</body> -</html> http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/beb69e3f/pom-with-benchmarks.xml ---------------------------------------------------------------------- diff --git a/pom-with-benchmarks.xml b/pom-with-benchmarks.xml index 9c85639..ee98c43 100644 --- a/pom-with-benchmarks.xml +++ b/pom-with-benchmarks.xml @@ -17,17 +17,17 @@ --> <project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - <modelVersion>4.0.0</modelVersion> + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + - <groupId>org.apache.pirk</groupId> <artifactId>pirk</artifactId> <version>0.0.1-SNAPSHOT</version> <packaging>jar</packaging> - + <name>Apache Pirk (incubating) Project</name> - <description>Apache Pirk (incubating) is a framework for scalable + <description>Apache Pirk (incubating) is a framework for scalable Private Information Retrieval (PIR). </description> <url>http://pirk.incubator.apache.org/</url> @@ -70,12 +70,12 @@ <updatePolicy>always</updatePolicy> </snapshots> </repository> - + <repository> <id>conjars.org</id> <url>http://conjars.org/repo</url> </repository> - + </repositories> <properties> @@ -123,7 +123,7 @@ <artifactId>spark-core_2.11</artifactId> <version>1.6.1</version> </dependency> - + <dependency> <groupId>com.googlecode.json-simple</groupId> <artifactId>json-simple</artifactId> @@ -135,7 +135,7 @@ <artifactId>commons-net</artifactId> <version>3.3</version> </dependency> - + <dependency> <groupId>org.elasticsearch</groupId> <artifactId>elasticsearch-hadoop</artifactId> @@ -155,61 +155,61 @@ </exclusion> </exclusions> </dependency> - + <!-- Square's JNA GMP module--> <dependency> - <groupId>com.squareup.jnagmp</groupId> - <artifactId>jnagmp</artifactId> - <version>1.1.0</version> + <groupId>com.squareup.jnagmp</groupId> + <artifactId>jnagmp</artifactId> + <version>1.1.0</version> </dependency> <!-- JMH for benchmarking the Paillier functions --> <dependency> - <groupId>org.openjdk.jmh</groupId> - <artifactId>jmh-core</artifactId> - <version>${jmh.version}</version> + <groupId>org.openjdk.jmh</groupId> + <artifactId>jmh-core</artifactId> + <version>${jmh.version}</version> </dependency> <dependency> - <groupId>org.openjdk.jmh</groupId> - <artifactId>jmh-generator-annprocess</artifactId> - <version>${jmh.version}</version> + <groupId>org.openjdk.jmh</groupId> + <artifactId>jmh-generator-annprocess</artifactId> + <version>${jmh.version}</version> </dependency> - + </dependencies> <build> <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-plugin</artifactId> - <version>2.18</version> - <configuration> - <redirectTestOutputToFile>true</redirectTestOutputToFile> - <argLine combine.children="append">-Xmx1G - -Djava.net.preferIPv4Stack=true</argLine> - </configuration> - <dependencies> - <dependency> - <!-- Force surefire to use JUnit --> - <groupId>org.apache.maven.surefire</groupId> - <artifactId>surefire-junit4</artifactId> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> <version>2.18</version> - </dependency> - </dependencies> - </plugin> + <configuration> + <redirectTestOutputToFile>true</redirectTestOutputToFile> + <argLine combine.children="append">-Xmx1G + -Djava.net.preferIPv4Stack=true</argLine> + </configuration> + <dependencies> + <dependency> + <!-- Force surefire to use JUnit --> + <groupId>org.apache.maven.surefire</groupId> + <artifactId>surefire-junit4</artifactId> + <version>2.18</version> + </dependency> + </dependencies> + </plugin> <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - <version>3.1</version> - <configuration> - <compilerVersion>${javac.target}</compilerVersion> - <source>${javac.target}</source> - <target>${javac.target}</target> - </configuration> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <version>3.1</version> + <configuration> + <compilerVersion>${javac.target}</compilerVersion> + <source>${javac.target}</source> + <target>${javac.target}</target> + </configuration> </plugin> <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-shade-plugin</artifactId> - <version>2.4.3</version> <!-- older versions of maven-shade-plugin make JMH painful --> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-shade-plugin</artifactId> + <version>2.4.3</version> <!-- older versions of maven-shade-plugin make JMH painful --> <executions> <execution> @@ -223,10 +223,10 @@ <shadedClassifierName>exe</shadedClassifierName> <transformers> <transformer - implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> + implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> </transformer> <transformer - implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"> + implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"> </transformer> </transformers> <filters> @@ -242,37 +242,37 @@ </configuration> </execution> <execution> - <phase>package</phase> - <id>benchmark</id> - <goals> - <goal>shade</goal> - </goals> - <configuration> - <!-- The jar is very corrupted if it isn't minimized --> - <minimizeJar>true</minimizeJar> - <finalName>${benchmarkjar.name}</finalName> - <transformers> - <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> - <mainClass>org.openjdk.jmh.Main</mainClass> - </transformer> - </transformers> - <filters> - <filter> - <!-- - Shading signed JARs will fail without this. - http://stackoverflow.com/questions/999489/invalid-signature-file-when-attempting-to-run-a-jar - --> - <artifact>*:*</artifact> - <excludes> - <exclude>META-INF/*.SF</exclude> - <exclude>META-INF/*.DSA</exclude> - <exclude>META-INF/*.RSA</exclude> - </excludes> - </filter> - </filters> - </configuration> + <phase>package</phase> + <id>benchmark</id> + <goals> + <goal>shade</goal> + </goals> + <configuration> + <!-- The jar is very corrupted if it isn't minimized --> + <minimizeJar>true</minimizeJar> + <finalName>${benchmarkjar.name}</finalName> + <transformers> + <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> + <mainClass>org.openjdk.jmh.Main</mainClass> + </transformer> + </transformers> + <filters> + <filter> + <!-- + Shading signed JARs will fail without this. + http://stackoverflow.com/questions/999489/invalid-signature-file-when-attempting-to-run-a-jar + --> + <artifact>*:*</artifact> + <excludes> + <exclude>META-INF/*.SF</exclude> + <exclude>META-INF/*.DSA</exclude> + <exclude>META-INF/*.RSA</exclude> + </excludes> + </filter> + </filters> + </configuration> </execution> - + </executions> </plugin> </plugins> http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/beb69e3f/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index b46325e..c14a928 100644 --- a/pom.xml +++ b/pom.xml @@ -11,653 +11,747 @@ License for the specific language governing permissions and ~ limitations under the License. --> -<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - <modelVersion>4.0.0</modelVersion> - - <parent> - <groupId>org.apache</groupId> - <artifactId>apache</artifactId> - <version>18</version> - </parent> - - <groupId>org.apache.pirk</groupId> - <artifactId>apache-pirk</artifactId> - <version>0.0.1-SNAPSHOT</version> - <packaging>jar</packaging> - - <name>Apache Pirk (incubating) Project</name> - <description>Apache Pirk (incubating) is a framework for scalable Private Information Retrieval (PIR). </description> - <url>http://pirk.incubator.apache.org/</url> - - <!-- this is the year of inception at ASF --> - <inceptionYear>2016</inceptionYear> - - <organization> - <name>The Apache Software Foundation</name> - <url>https://www.apache.org</url> - </organization> - - <licenses> - <license> - <name>Apache License, Version 2.0</name> - <url>https://www.apache.org/licenses/LICENSE-2.0</url> - </license> - </licenses> - - <mailingLists> - <mailingList> - <name>Dev</name> - <subscribe>[email protected]</subscribe> - <unsubscribe>[email protected]</unsubscribe> - <post>[email protected]</post> - <archive>http://mail-archives.apache.org/mod_mbox/incubator-pirk-dev/</archive> - </mailingList> - <mailingList> - <name>Commits</name> - <subscribe>[email protected]</subscribe> - <unsubscribe>[email protected]</unsubscribe> - <archive>http://mail-archives.apache.org/mod_mbox/incubator-pirk-commits</archive> - </mailingList> - </mailingLists> - - <scm> - <connection>scm:git:git://git.apache.org/incubator-pirk.git</connection> - <developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/incubator-pirk.git</developerConnection> - <url>https://git-wip-us.apache.org/repos/asf?p=incubator-pirk.git</url> - <tag>HEAD</tag> - </scm> - - <issueManagement> - <system>JIRA</system> - <url>https://issues.apache.org/jira/browse/PIRK</url> - </issueManagement> - - <properties> - <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> - <maven.compiler.source>1.8</maven.compiler.source> - <maven.compiler.target>1.8</maven.compiler.target> - <scala.version>2.10.4</scala.version> - <jmh.version>1.11.3</jmh.version> - <benchmarkjar.name>benchmarks</benchmarkjar.name> - <javac.target>1.8</javac.target> - <slf4j.version>1.7.21</slf4j.version> - <log4j2.version>2.6.2</log4j2.version> - <junit.version>4.12</junit.version> - <log4j.configuration>log4j2.xml</log4j.configuration> - <hadoop.version>2.7.2</hadoop.version> - <apache-commons.version>3.3</apache-commons.version> - <elasticsearch.version>2.3.3</elasticsearch.version> - <spark-streaming.version>2.0.0</spark-streaming.version> - <pirk.forkCount>1C</pirk.forkCount> - <pirk.reuseForks>true</pirk.reuseForks> - </properties> - - <dependencies> - <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <version>4.12</version> - </dependency> - - <dependency> - <groupId>org.apache.commons</groupId> - <artifactId>commons-math3</artifactId> - <version>${apache-commons.version}</version> - </dependency> - - <dependency> - <groupId>com.googlecode.json-simple</groupId> - <artifactId>json-simple</artifactId> - <version>1.1</version> - </dependency> - - <dependency> - <groupId>commons-net</groupId> - <artifactId>commons-net</artifactId> - <version>${apache-commons.version}</version> - </dependency> - - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-common</artifactId> - <version>${hadoop.version}</version> - <exclusions> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-log4j12</artifactId> - </exclusion> - </exclusions> - </dependency> - - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-mapreduce-client-core</artifactId> - <version>${hadoop.version}</version> - <exclusions> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-log4j12</artifactId> - </exclusion> - <exclusion> - <groupId>log4j</groupId> - <artifactId>log4j</artifactId> - </exclusion> - </exclusions> - </dependency> - - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-client</artifactId> - <version>${hadoop.version}</version> - <exclusions> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-log4j12</artifactId> - </exclusion> - <exclusion> - <groupId>log4j</groupId> - <artifactId>log4j</artifactId> - </exclusion> - </exclusions> - </dependency> - - <dependency> - <groupId>org.apache.spark</groupId> - <artifactId>spark-core_2.11</artifactId> - <version>1.6.1</version> - <exclusions> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-log4j12</artifactId> - </exclusion> - <exclusion> - <groupId>log4j</groupId> - <artifactId>log4j</artifactId> - </exclusion> - </exclusions> - </dependency> - - <dependency> - <groupId>org.elasticsearch</groupId> - <artifactId>elasticsearch-hadoop</artifactId> - <version>${elasticsearch.version}</version> - <exclusions> - <exclusion> - <artifactId>commons-net</artifactId> - <groupId>commons-net</groupId> - </exclusion> - <exclusion> - <groupId>org.apache.hive</groupId> - <artifactId>hive-service</artifactId> - </exclusion> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-log4j12</artifactId> - </exclusion> - <exclusion> - <groupId>log4j</groupId> - <artifactId>log4j</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.storm</groupId> - <artifactId>storm-core</artifactId> - </exclusion> - <exclusion> - <groupId>cascading</groupId> - <artifactId>cascading-local</artifactId> - </exclusion> - <exclusion> - <groupId>cascading</groupId> - <artifactId>cascading-core</artifactId> - </exclusion> - <exclusion> - <groupId>cascading</groupId> - <artifactId>cascading-hadoop</artifactId> - </exclusion> - </exclusions> - </dependency> - - <dependency> - <groupId>org.apache.spark</groupId> - <artifactId>spark-streaming_2.10</artifactId> - <version>${spark-streaming.version}</version> - </dependency> - - <!-- Square's JNA GMP module --> - <dependency> - <groupId>com.squareup.jnagmp</groupId> - <artifactId>jnagmp</artifactId> - <version>1.1.0</version> - </dependency> - - <!-- JMH for benchmarking the Paillier functions --> - <dependency> - <groupId>org.openjdk.jmh</groupId> - <artifactId>jmh-core</artifactId> - <version>${jmh.version}</version> - <scope>provided</scope> - </dependency> - - <dependency> - <groupId>org.openjdk.jmh</groupId> - <artifactId>jmh-generator-annprocess</artifactId> - <version>${jmh.version}</version> - <scope>provided</scope> - </dependency> - - <!-- Sl4j modules --> - <dependency> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - <version>${slf4j.version}</version> - </dependency> - - <dependency> - <groupId>org.apache.logging.log4j</groupId> - <artifactId>log4j-slf4j-impl</artifactId> - <version>2.6.2</version> - </dependency> - - <dependency> - <groupId>org.apache.logging.log4j</groupId> - <artifactId>log4j-core</artifactId> - <version>2.6.2</version> - </dependency> - - </dependencies> - - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-plugin</artifactId> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - </plugin> - - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-jar-plugin</artifactId> - </plugin> - - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-source-plugin</artifactId> - </plugin> - - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-shade-plugin</artifactId> - </plugin> - - <plugin> - <groupId>org.apache.rat</groupId> - <artifactId>apache-rat-plugin</artifactId> - </plugin> - - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-release-plugin</artifactId> - </plugin> - </plugins> - - <pluginManagement> - <plugins> - <plugin> - <groupId>org.apache.rat</groupId> - <artifactId>apache-rat-plugin</artifactId> - <version>0.12</version> - <configuration> - <excludes> - <exclude>.travis.yml</exclude> <!-- Travis CI Build Descriptor File --> - <exclude>findbugs-exclude.xml</exclude> <!-- False positives for FindBugs analysis --> - <exclude>KEYS</exclude> <!-- GPG keys of Release Managers --> - <exclude>eclipse*.xml</exclude> <!-- Exclude eclipse* xml --> - <exclude>docs/*</exclude> <!-- Exclude docs --> - <exclude>logs/*</exclude> <!-- Exclude logs --> - <exclude>**/m2.conf</exclude> <!-- Exclude Maven conf which gets installed on travis and fails RAT check --> - <exclude>src/main/resources/META-INF/**</exclude> - </excludes> - </configuration> - <dependencies> - <!-- workaround for RAT-158 --> - <dependency> - <groupId>org.apache.maven.doxia</groupId> - <artifactId>doxia-core</artifactId> - <version>1.6</version> - <exclusions> - <exclusion> - <groupId>xerces</groupId> - <artifactId>xercesImpl</artifactId> - </exclusion> - </exclusions> - </dependency> - </dependencies> - <executions> - <execution> - <phase>validate</phase> - <goals> - <goal>check</goal> - </goals> - </execution> - </executions> - </plugin> - - <!-- Coverage analysis for tests --> - <plugin> - <groupId>org.jacoco</groupId> - <artifactId>jacoco-maven-plugin</artifactId> - <version>0.7.5.201505241946</version> - <executions> - <execution> - <goals> - <goal>prepare-agent</goal> - </goals> - <configuration> - <output>file</output> - <dumpOnExit>true</dumpOnExit> - </configuration> - </execution> - <execution> - <id>report</id> - <phase>prepare-package</phase> - <goals> - <goal>report</goal> - </goals> - </execution> - </executions> - </plugin> - - <!-- Report jacoco coverage to coveralls.io --> - <plugin> - <groupId>org.eluder.coveralls</groupId> - <artifactId>coveralls-maven-plugin</artifactId> - <version>4.1.0</version> - </plugin> - - - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-plugin</artifactId> - <version>2.18</version> - <configuration> - <redirectTestOutputToFile>true</redirectTestOutputToFile> - <argLine combine.children="append">-Xmx1G - -Djava.net.preferIPv4Stack=true - </argLine> - <systemPropertyVariables> - <log4j.configuration>${log4j.configuration}</log4j.configuration> - </systemPropertyVariables> - <forkCount>${pirk.forkCount}</forkCount> - <reuseForks>${pirk.reuseForks}</reuseForks> - </configuration> - <dependencies> - <dependency> - <!-- Force surefire to use JUnit --> - <groupId>org.apache.maven.surefire</groupId> - <artifactId>surefire-junit4</artifactId> - <version>2.18</version> - </dependency> - </dependencies> - </plugin> - - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - <version>3.5.1</version> - <configuration> - <compilerVersion>${javac.target}</compilerVersion> - <source>${javac.target}</source> - <target>${javac.target}</target> - </configuration> - </plugin> - - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-jar-plugin</artifactId> - <version>3.0.1</version> - <configuration> - <excludes> - <exclude>org/apache/pirk/benchmark/**</exclude> - <exclude>*/openjdk/**</exclude> - <exclude>generated-sources/**</exclude> - </excludes> - </configuration> - </plugin> - - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-source-plugin</artifactId> - <version>3.0.1</version> - <configuration> - <excludes> - <exclude>org/apache/pirk/benchmark/**</exclude> - <exclude>*/openjdk/**</exclude> - <exclude>generated-sources/**</exclude> - </excludes> - </configuration> - </plugin> - - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-shade-plugin</artifactId> - <version>2.4.3</version> <!-- older versions of maven-shade-plugin make JMH painful --> - - <executions> - <execution> - <phase>package</phase> - <id>main</id> - <goals> - <goal>shade</goal> - </goals> - <configuration> - <shadedArtifactAttached>true</shadedArtifactAttached> - <shadedClassifierName>exe</shadedClassifierName> - <transformers> - <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> - </transformer> - <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"> - </transformer> - </transformers> - <filters> - <filter> - <artifact>*:*</artifact> - <excludes> - <exclude>META-INF/*.SF</exclude> - <exclude>META-INF/*.DSA</exclude> - <exclude>META-INF/*.RSA</exclude> - </excludes> - </filter> - </filters> - </configuration> - </execution> - - <!-- in the version with benchmarks (pom-with-benchmarks.xml), this - is where that <execution></execution> lives --> - - </executions> - </plugin> - - <!--This plugin's configuration is used to store Eclipse m2e settings - only. It has no influence on the Maven build itself. --> - <plugin> - <groupId>org.eclipse.m2e</groupId> - <artifactId>lifecycle-mapping</artifactId> - <version>1.0.0</version> - <configuration> - <lifecycleMappingMetadata> - <pluginExecutions> - <pluginExecution> - <pluginExecutionFilter> - <groupId> - org.scala-tools - </groupId> - <artifactId> - maven-scala-plugin - </artifactId> - <versionRange> - [2.15.2,) - </versionRange> - <goals> - <goal>testCompile</goal> - </goals> - </pluginExecutionFilter> - <action> - <ignore /> - </action> - </pluginExecution> - <pluginExecution> - <pluginExecutionFilter> - <groupId> - org.apache.rat - </groupId> - <artifactId> - apache-rat-plugin - </artifactId> - <versionRange> - [0.11,) - </versionRange> - <goals> - <goal>check</goal> - </goals> - </pluginExecutionFilter> - <action> - <ignore /> - </action> - </pluginExecution> - </pluginExecutions> - </lifecycleMappingMetadata> - </configuration> - </plugin> - - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-release-plugin</artifactId> - <version>2.5.1</version> - <configuration> - <useReleaseProfile>true</useReleaseProfile> - <releaseProfiles>signed_release</releaseProfiles> - <autoVersionSubmodules>true</autoVersionSubmodules> - <goals>deploy</goals> - <tagNameFormat>@{project.artifactId}-@{project.version}</tagNameFormat> - <pushChanges>false</pushChanges> - <localCheckout>true</localCheckout> - </configuration> - <executions> - <execution> - <id>default</id> - <goals> - <goal>perform</goal> - </goals> - <configuration> - <pomFileName>pom.xml</pomFileName> - </configuration> - </execution> - </executions> - </plugin> - - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-javadoc-plugin</artifactId> - <version>2.10.4</version> - <configuration> - <javadocDirectory>docs</javadocDirectory> - <testJavadocDirectory>docs/test</testJavadocDirectory> - <javadocVersion>1.8</javadocVersion> - </configuration> - </plugin> - </plugins> - </pluginManagement> - - </build> - - <profiles> - <profile> - <!-- Performs execution of Integration Tests using the Maven FailSafe - Plugin. The view of integration tests in this context are those tests interfacing - with external sources and services requiring additional resources or credentials - that cannot be explicitly provided. --> - <id>integration-tests</id> - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-failsafe-plugin</artifactId> - <executions> - <execution> - <goals> - <goal>integration-test</goal> - <goal>verify</goal> - </goals> - </execution> - </executions> - </plugin> - </plugins> - </build> - </profile> - <profile> - <!-- Checks style and licensing requirements. This is a good idea to run - for contributions and for the release process. While it would be nice to - run always these plugins can considerably slow the build and have proven - to create unstable builds in our multi-module project and when building using - multiple threads. The stability issues seen with Checkstyle in multi-module - builds include false-positives and false negatives. --> - <id>contrib-check</id> - <build> - <plugins> - <plugin> - <groupId>org.apache.rat</groupId> - <artifactId>apache-rat-plugin</artifactId> - <executions> - <execution> - <goals> - <goal>check</goal> - </goals> - <phase>verify</phase> - </execution> - </executions> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-checkstyle-plugin</artifactId> - <executions> - <execution> - <id>check-style</id> - <goals> - <goal>check</goal> - </goals> - </execution> - </executions> - </plugin> - </plugins> - </build> - </profile> - <profile> - <!-- This profile will disable DocLint which performs strict JavaDoc processing - which was introduced in JDK 8. These are technically errors in the JavaDoc - which we need to eventually address. However, if a release is performed using - JDK 8, the JavaDoc generation would fail. By activating this profile when - running on JDK 8 we can ensure the JavaDocs continue to generate successfully --> - <id>disable-doclint</id> - <activation> - <jdk>1.8</jdk> - </activation> - <build> - <pluginManagement> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-javadoc-plugin</artifactId> - <configuration> - <additionalparam>-Xdoclint:none</additionalparam> - </configuration> - </plugin> - </plugins> - </pluginManagement> - </build> - </profile> - </profiles> - -</project> \ No newline at end of file +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache</groupId> + <artifactId>apache</artifactId> + <version>18</version> + </parent> + + <groupId>org.apache.pirk</groupId> + <artifactId>apache-pirk</artifactId> + <version>0.0.1-SNAPSHOT</version> + <packaging>jar</packaging> + + <name>Apache Pirk (incubating) Project</name> + <description>Apache Pirk (incubating) is a framework for scalable Private Information Retrieval (PIR).</description> + <url>http://pirk.incubator.apache.org/</url> + + <!-- this is the year of inception at ASF --> + <inceptionYear>2016</inceptionYear> + + <organization> + <name>The Apache Software Foundation</name> + <url>https://www.apache.org</url> + </organization> + + <licenses> + <license> + <name>Apache License, Version 2.0</name> + <url>https://www.apache.org/licenses/LICENSE-2.0</url> + </license> + </licenses> + + <mailingLists> + <mailingList> + <name>Dev</name> + <subscribe>[email protected]</subscribe> + <unsubscribe>[email protected]</unsubscribe> + <post>[email protected]</post> + <archive>http://mail-archives.apache.org/mod_mbox/incubator-pirk-dev/</archive> + </mailingList> + <mailingList> + <name>Commits</name> + <subscribe>[email protected]</subscribe> + <unsubscribe>[email protected]</unsubscribe> + <archive>http://mail-archives.apache.org/mod_mbox/incubator-pirk-commits</archive> + </mailingList> + </mailingLists> + + <scm> + <connection>scm:git:git://git.apache.org/incubator-pirk.git</connection> + <developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/incubator-pirk.git</developerConnection> + <url>https://git-wip-us.apache.org/repos/asf?p=incubator-pirk.git</url> + <tag>HEAD</tag> + </scm> + + <issueManagement> + <system>JIRA</system> + <url>https://issues.apache.org/jira/browse/PIRK</url> + </issueManagement> + + <properties> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + <maven.compiler.source>1.8</maven.compiler.source> + <maven.compiler.target>1.8</maven.compiler.target> + <scala.version>2.10.4</scala.version> + <jmh.version>1.11.3</jmh.version> + <benchmarkjar.name>benchmarks</benchmarkjar.name> + <javac.target>1.8</javac.target> + <slf4j.version>1.7.21</slf4j.version> + <log4j2.version>2.1</log4j2.version> + <junit.version>4.12</junit.version> + <log4j.configuration>log4j2.xml</log4j.configuration> + <hadoop.version>2.7.2</hadoop.version> + <spark.version>1.6.1</spark.version> + <apache-commons.version>3.3</apache-commons.version> + <elasticsearch.version>2.3.3</elasticsearch.version> + <storm.version>1.0.1</storm.version> + <kafka.version>0.9.0.1</kafka.version> + <spark-streaming.version>2.0.0</spark-streaming.version> + <pirk.forkCount>1C</pirk.forkCount> + <pirk.reuseForks>true</pirk.reuseForks> + </properties> + + <dependencies> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <version>4.12</version> + </dependency> + + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-math3</artifactId> + <version>${apache-commons.version}</version> + </dependency> + + <dependency> + <groupId>com.googlecode.json-simple</groupId> + <artifactId>json-simple</artifactId> + <version>1.1</version> + </dependency> + + <dependency> + <groupId>commons-net</groupId> + <artifactId>commons-net</artifactId> + <version>${apache-commons.version}</version> + </dependency> + + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <version>${hadoop.version}</version> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.curator</groupId> + <artifactId>curator-client</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-mapreduce-client-core</artifactId> + <version>${hadoop.version}</version> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + <exclusion> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-client</artifactId> + <version>${hadoop.version}</version> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + <exclusion> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + </exclusion> + <exclusion> + <groupId>xerces</groupId> + <artifactId>xercesImpl</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-core_2.11</artifactId> + <version>${spark.version}</version> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + <exclusion> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + </exclusion> + <exclusion> + <groupId>org.scala-lang</groupId> + <artifactId>scalap</artifactId> + </exclusion> + <exclusion> + <groupId>org.scala-lang</groupId> + <artifactId>scala-library</artifactId> + </exclusion> + <exclusion> + <groupId>org.scala-lang</groupId> + <artifactId>scala-reflect</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-streaming_2.10</artifactId> + <version>${spark-streaming.version}</version> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>org.elasticsearch</groupId> + <artifactId>elasticsearch-hadoop</artifactId> + <version>${elasticsearch.version}</version> + <exclusions> + <exclusion> + <artifactId>commons-net</artifactId> + <groupId>commons-net</groupId> + </exclusion> + <exclusion> + <groupId>org.apache.hive</groupId> + <artifactId>hive-service</artifactId> + </exclusion> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + <exclusion> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.storm</groupId> + <artifactId>storm-core</artifactId> + </exclusion> + <exclusion> + <groupId>cascading</groupId> + <artifactId>cascading-local</artifactId> + </exclusion> + <exclusion> + <groupId>cascading</groupId> + <artifactId>cascading-core</artifactId> + </exclusion> + <exclusion> + <groupId>cascading</groupId> + <artifactId>cascading-hadoop</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>org.apache.storm</groupId> + <artifactId>storm-core</artifactId> + <version>${storm.version}</version> + <scope>provided</scope> + <exclusions> + <exclusion> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-api</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-core</artifactId> + </exclusion> + <exclusion> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-slf4j-impl</artifactId> + </exclusion> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>org.apache.storm</groupId> + <artifactId>storm-kafka</artifactId> + <version>${storm.version}</version> + </dependency> + + <dependency> + <groupId>org.apache.kafka</groupId> + <artifactId>kafka_2.10</artifactId> + <version>${kafka.version}</version> + <exclusions> + <exclusion> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + </exclusion> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + <exclusion> + <groupId>org.scala-lang</groupId> + <artifactId>scala-library</artifactId> + </exclusion> + </exclusions> + </dependency> + + <dependency> + <groupId>org.apache.curator</groupId> + <artifactId>curator-test</artifactId> + <version>2.10.0</version> + <scope>test</scope> + </dependency> + + <!-- Square's JNA GMP module --> + <dependency> + <groupId>com.squareup.jnagmp</groupId> + <artifactId>jnagmp</artifactId> + <version>1.1.0</version> + </dependency> + + <!-- JMH for benchmarking the Paillier functions --> + <dependency> + <groupId>org.openjdk.jmh</groupId> + <artifactId>jmh-core</artifactId> + <version>${jmh.version}</version> + <scope>provided</scope> + </dependency> + + <dependency> + <groupId>org.openjdk.jmh</groupId> + <artifactId>jmh-generator-annprocess</artifactId> + <version>${jmh.version}</version> + <scope>provided</scope> + </dependency> + + <!-- Sl4j modules --> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + <version>${slf4j.version}</version> + </dependency> + + <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-slf4j-impl</artifactId> + <version>${log4j2.version}</version> + </dependency> + + <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-core</artifactId> + <version>${log4j2.version}</version> + </dependency> + + <dependency> + <groupId>org.scala-lang</groupId> + <artifactId>scala-library</artifactId> + <version>${scala.version}</version> + </dependency> + + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-source-plugin</artifactId> + </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-shade-plugin</artifactId> + </plugin> + + <plugin> + <groupId>org.apache.rat</groupId> + <artifactId>apache-rat-plugin</artifactId> + </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-release-plugin</artifactId> + </plugin> + </plugins> + + <pluginManagement> + <plugins> + <plugin> + <groupId>org.apache.rat</groupId> + <artifactId>apache-rat-plugin</artifactId> + <version>0.12</version> + <configuration> + <excludes> + <exclude>.travis.yml</exclude> <!-- Travis CI Build Descriptor File --> + <exclude>findbugs-exclude.xml</exclude> <!-- False positives for FindBugs analysis --> + <exclude>KEYS</exclude> <!-- GPG keys of Release Managers --> + <exclude>eclipse*.xml</exclude> <!-- Exclude eclipse* xml --> + <exclude>docs/*</exclude> <!-- Exclude docs --> + <exclude>logs/*</exclude> <!-- Exclude logs --> + <exclude>**/m2.conf</exclude> <!-- Exclude Maven conf which gets installed on travis and fails RAT check --> + <exclude>src/main/resources/META-INF/**</exclude> + </excludes> + </configuration> + <dependencies> + <!-- workaround for RAT-158 --> + <dependency> + <groupId>org.apache.maven.doxia</groupId> + <artifactId>doxia-core</artifactId> + <version>1.6</version> + <exclusions> + <exclusion> + <groupId>xerces</groupId> + <artifactId>xercesImpl</artifactId> + </exclusion> + </exclusions> + </dependency> + </dependencies> + <executions> + <execution> + <phase>validate</phase> + <goals> + <goal>check</goal> + </goals> + </execution> + </executions> + </plugin> + + <!-- Coverage analysis for tests --> + <plugin> + <groupId>org.jacoco</groupId> + <artifactId>jacoco-maven-plugin</artifactId> + <version>0.7.5.201505241946</version> + <executions> + <execution> + <goals> + <goal>prepare-agent</goal> + </goals> + <configuration> + <output>file</output> + <dumpOnExit>true</dumpOnExit> + </configuration> + </execution> + <execution> + <id>report</id> + <phase>prepare-package</phase> + <goals> + <goal>report</goal> + </goals> + </execution> + </executions> + </plugin> + + <!-- Report jacoco coverage to coveralls.io --> + <plugin> + <groupId>org.eluder.coveralls</groupId> + <artifactId>coveralls-maven-plugin</artifactId> + <version>4.1.0</version> + </plugin> + + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <version>2.18</version> + <configuration> + <redirectTestOutputToFile>true</redirectTestOutputToFile> + <argLine combine.children="append">-Xmx1G + -Djava.net.preferIPv4Stack=true + </argLine> + <systemPropertyVariables> + <log4j.configuration>${log4j.configuration}</log4j.configuration> + </systemPropertyVariables> + <forkCount>${pirk.forkCount}</forkCount> + <reuseForks>${pirk.reuseForks}</reuseForks> + </configuration> + <dependencies> + <dependency> + <!-- Force surefire to use JUnit --> + <groupId>org.apache.maven.surefire</groupId> + <artifactId>surefire-junit4</artifactId> + <version>2.18</version> + </dependency> + </dependencies> + </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <version>3.5.1</version> + <configuration> + <compilerVersion>${javac.target}</compilerVersion> + <source>${javac.target}</source> + <target>${javac.target}</target> + </configuration> + </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <version>3.0.1</version> + <configuration> + <excludes> + <exclude>org/apache/pirk/benchmark/**</exclude> + <exclude>*/openjdk/**</exclude> + <exclude>generated-sources/**</exclude> + </excludes> + </configuration> + </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-source-plugin</artifactId> + <version>3.0.1</version> + <configuration> + <excludes> + <exclude>org/apache/pirk/benchmark/**</exclude> + <exclude>*/openjdk/**</exclude> + <exclude>generated-sources/**</exclude> + </excludes> + </configuration> + </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-shade-plugin</artifactId> + <version>2.4.3</version> <!-- older versions of maven-shade-plugin make JMH painful --> + + <executions> + <execution> + <phase>package</phase> + <id>main</id> + <goals> + <goal>shade</goal> + </goals> + <configuration> + <shadedArtifactAttached>true</shadedArtifactAttached> + <shadedClassifierName>exe</shadedClassifierName> + <transformers> + <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> + </transformer> + <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"> + </transformer> + </transformers> + <filters> + <filter> + <artifact>*:*</artifact> + <excludes> + <exclude>META-INF/*.SF</exclude> + <exclude>META-INF/*.DSA</exclude> + <exclude>META-INF/*.RSA</exclude> + </excludes> + </filter> + </filters> + </configuration> + </execution> + + <!-- in the version with benchmarks (pom-with-benchmarks.xml), this + is where that <execution></execution> lives --> + + </executions> + </plugin> + + <!--This plugin's configuration is used to store Eclipse m2e settings + only. It has no influence on the Maven build itself. --> + <plugin> + <groupId>org.eclipse.m2e</groupId> + <artifactId>lifecycle-mapping</artifactId> + <version>1.0.0</version> + <configuration> + <lifecycleMappingMetadata> + <pluginExecutions> + <pluginExecution> + <pluginExecutionFilter> + <groupId> + org.scala-tools + </groupId> + <artifactId> + maven-scala-plugin + </artifactId> + <versionRange> + [2.15.2,) + </versionRange> + <goals> + <goal>testCompile</goal> + </goals> + </pluginExecutionFilter> + <action> + <ignore/> + </action> + </pluginExecution> + <pluginExecution> + <pluginExecutionFilter> + <groupId> + org.apache.rat + </groupId> + <artifactId> + apache-rat-plugin + </artifactId> + <versionRange> + [0.11,) + </versionRange> + <goals> + <goal>check</goal> + </goals> + </pluginExecutionFilter> + <action> + <ignore/> + </action> + </pluginExecution> + </pluginExecutions> + </lifecycleMappingMetadata> + </configuration> + </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-release-plugin</artifactId> + <version>2.5.1</version> + <configuration> + <useReleaseProfile>true</useReleaseProfile> + <releaseProfiles>signed_release</releaseProfiles> + <autoVersionSubmodules>true</autoVersionSubmodules> + <goals>deploy</goals> + <tagNameFormat>@{project.artifactId}-@{project.version}</tagNameFormat> + <pushChanges>false</pushChanges> + <localCheckout>true</localCheckout> + </configuration> + <executions> + <execution> + <id>default</id> + <goals> + <goal>perform</goal> + </goals> + <configuration> + <pomFileName>pom.xml</pomFileName> + </configuration> + </execution> + </executions> + </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-javadoc-plugin</artifactId> + <version>2.10.4</version> + <configuration> + <javadocDirectory>docs</javadocDirectory> + <testJavadocDirectory>docs/test</testJavadocDirectory> + <javadocVersion>1.8</javadocVersion> + </configuration> + </plugin> + </plugins> + </pluginManagement> + + </build> + + <profiles> + <profile> + <!-- Performs execution of Integration Tests using the Maven FailSafe + Plugin. The view of integration tests in this context are those tests interfacing + with external sources and services requiring additional resources or credentials + that cannot be explicitly provided. --> + <id>integration-tests</id> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-failsafe-plugin</artifactId> + <executions> + <execution> + <goals> + <goal>integration-test</goal> + <goal>verify</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + <profile> + <!-- Checks style and licensing requirements. This is a good idea to run + for contributions and for the release process. While it would be nice to + run always these plugins can considerably slow the build and have proven + to create unstable builds in our multi-module project and when building using + multiple threads. The stability issues seen with Checkstyle in multi-module + builds include false-positives and false negatives. --> + <id>contrib-check</id> + <build> + <plugins> + <plugin> + <groupId>org.apache.rat</groupId> + <artifactId>apache-rat-plugin</artifactId> + <executions> + <execution> + <goals> + <goal>check</goal> + </goals> + <phase>verify</phase> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-checkstyle-plugin</artifactId> + <executions> + <execution> + <id>check-style</id> + <goals> + <goal>check</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + <profile> + <!-- This profile will disable DocLint which performs strict JavaDoc processing + which was introduced in JDK 8. These are technically errors in the JavaDoc + which we need to eventually address. However, if a release is performed using + JDK 8, the JavaDoc generation would fail. By activating this profile when + running on JDK 8 we can ensure the JavaDocs continue to generate successfully --> + <id>disable-doclint</id> + <activation> + <jdk>1.8</jdk> + </activation> + <build> + <pluginManagement> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-javadoc-plugin</artifactId> + <configuration> + <additionalparam>-Xdoclint:none</additionalparam> + </configuration> + </plugin> + </plugins> + </pluginManagement> + </build> + </profile> + </profiles> + +</project> http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/beb69e3f/src/main/java/org/apache/pirk/query/wideskies/Query.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/query/wideskies/Query.java b/src/main/java/org/apache/pirk/query/wideskies/Query.java index f43fe47..b0322d0 100644 --- a/src/main/java/org/apache/pirk/query/wideskies/Query.java +++ b/src/main/java/org/apache/pirk/query/wideskies/Query.java @@ -25,8 +25,8 @@ import java.util.Map; import java.util.SortedMap; import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; - import java.util.function.Consumer; + import org.apache.pirk.encryption.ModPowAbstraction; import org.apache.pirk.serialization.Storable; import org.slf4j.Logger; http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/beb69e3f/src/main/java/org/apache/pirk/query/wideskies/QueryInfo.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/query/wideskies/QueryInfo.java b/src/main/java/org/apache/pirk/query/wideskies/QueryInfo.java index 362d26f..76cd755 100644 --- a/src/main/java/org/apache/pirk/query/wideskies/QueryInfo.java +++ b/src/main/java/org/apache/pirk/query/wideskies/QueryInfo.java @@ -16,9 +16,12 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.pirk.query.wideskies; import java.io.Serializable; +import java.util.HashMap; +import java.util.Map; import java.util.UUID; import org.apache.pirk.schema.query.QuerySchema; @@ -44,7 +47,7 @@ public class QueryInfo implements Serializable, Cloneable private String queryType = null; // QueryType string const private int hashBitSize = 0; // Bit size of the keyed hash function - private String hashKey = null; // Key for the keyed hash function + private String hashKey; // Key for the keyed hash function private int numBitsPerDataElement = 0; // total num bits per returned data value - defined relative to query type private int dataPartitionBitSize = 0; // num of bits for each partition of an incoming data element, must be < 32 right now @@ -96,6 +99,33 @@ public class QueryInfo implements Serializable, Cloneable printQueryInfo(); } + public QueryInfo(Map queryInfoMap) + { + // The Storm Config serializes the map as a json and reads back in with numeric values as longs. + // So numerics need to be cast as a long and call .intValue. However, in PirkHashScheme the map contains ints. + identifier = UUID.fromString((String) queryInfoMap.get("uuid")); + queryType = (String) queryInfoMap.get("queryType"); + hashKey = (String) queryInfoMap.get("hashKey"); + useExpLookupTable = (boolean) queryInfoMap.get("useExpLookupTable"); + useHDFSExpLookupTable = (boolean) queryInfoMap.get("useHDFSExpLookupTable"); + embedSelector = (boolean) queryInfoMap.get("embedSelector"); + try + { + numSelectors = ((Long) queryInfoMap.get("numSelectors")).intValue(); + hashBitSize = ((Long) queryInfoMap.get("hashBitSize")).intValue(); + numBitsPerDataElement = ((Long) queryInfoMap.get("numBitsPerDataElement")).intValue(); + numPartitionsPerDataElement = ((Long) queryInfoMap.get("numPartitionsPerDataElement")).intValue(); + dataPartitionBitSize = ((Long) queryInfoMap.get("dataPartitionsBitSize")).intValue(); + } catch (ClassCastException e) + { + numSelectors = (int) queryInfoMap.get("numSelectors"); + hashBitSize = (int) queryInfoMap.get("hashBitSize"); + numBitsPerDataElement = (int) queryInfoMap.get("numBitsPerDataElement"); + numPartitionsPerDataElement = (int) queryInfoMap.get("numPartitionsPerDataElement"); + dataPartitionBitSize = (int) queryInfoMap.get("dataPartitionsBitSize"); + } + } + public UUID getIdentifier() { return identifier; @@ -151,6 +181,24 @@ public class QueryInfo implements Serializable, Cloneable return embedSelector; } + public Map toMap() + { + Map<String,Object> queryInfo = new HashMap<String,Object>(); + queryInfo.put("uuid", identifier.toString()); + queryInfo.put("queryType", queryType); + queryInfo.put("numSelectors", numSelectors); + queryInfo.put("hashBitSize", hashBitSize); + queryInfo.put("hashKey", hashKey); + queryInfo.put("numBitsPerDataElement", numBitsPerDataElement); + queryInfo.put("numPartitionsPerDataElement", numPartitionsPerDataElement); + queryInfo.put("dataPartitionsBitSize", dataPartitionBitSize); + queryInfo.put("useExpLookupTable", useExpLookupTable); + queryInfo.put("useHDFSExpLookupTable", useHDFSExpLookupTable); + queryInfo.put("embedSelector", embedSelector); + + return queryInfo; + } + public void addQuerySchema(QuerySchema qSchemaIn) { qSchema = qSchemaIn; http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/beb69e3f/src/main/java/org/apache/pirk/responder/wideskies/ResponderDriver.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/responder/wideskies/ResponderDriver.java b/src/main/java/org/apache/pirk/responder/wideskies/ResponderDriver.java index 6b32418..f8e396b 100644 --- a/src/main/java/org/apache/pirk/responder/wideskies/ResponderDriver.java +++ b/src/main/java/org/apache/pirk/responder/wideskies/ResponderDriver.java @@ -28,6 +28,7 @@ import org.apache.pirk.responder.wideskies.mapreduce.ComputeResponseTool; import org.apache.pirk.responder.wideskies.spark.ComputeResponse; import org.apache.pirk.responder.wideskies.spark.streaming.ComputeStreamingResponse; import org.apache.pirk.responder.wideskies.standalone.Responder; +import org.apache.pirk.responder.wideskies.storm.PirkTopology; import org.apache.pirk.serialization.LocalFileSystemStore; import org.apache.pirk.utils.SystemConfiguration; import org.slf4j.Logger; @@ -49,6 +50,11 @@ public class ResponderDriver { private static final Logger logger = LoggerFactory.getLogger(ResponderDriver.class); + enum Platform + { + MAPREDUCE, SPARK, SPARKSTREAMING, STORM, STANDALONE, NONE; + } + public static void main(String[] args) throws Exception { ResponderCLI responderCLI = new ResponderCLI(args); @@ -56,49 +62,64 @@ public class ResponderDriver // For handling System.exit calls from Spark Streaming System.setSecurityManager(new SystemExitManager()); - if (SystemConfiguration.getProperty(ResponderProps.PLATFORM).equals("mapreduce")) + Platform platform = Platform.NONE; + String platformString = SystemConfiguration.getProperty(ResponderProps.PLATFORM); + try { - logger.info("Launching MapReduce ResponderTool:"); - - ComputeResponseTool pirWLTool = new ComputeResponseTool(); - ToolRunner.run(pirWLTool, new String[] {}); - } - else if (SystemConfiguration.getProperty(ResponderProps.PLATFORM).equals("spark")) + platform = Platform.valueOf(platformString.toUpperCase()); + } catch (IllegalArgumentException e) { - logger.info("Launching Spark ComputeResponse:"); - - FileSystem fs = FileSystem.get(new Configuration()); - ComputeResponse computeResponse = new ComputeResponse(fs); - computeResponse.performQuery(); + logger.error("platform " + platformString + " not found."); } - else if (SystemConfiguration.getProperty(ResponderProps.PLATFORM).equals("sparkstreaming")) - { - logger.info("Launching Spark ComputeStreamingResponse:"); - FileSystem fs = FileSystem.get(new Configuration()); - ComputeStreamingResponse computeSR = new ComputeStreamingResponse(fs); - try - { - computeSR.performQuery(); - } catch (SystemExitException e) - { - // If System.exit(0) is not caught from Spark Streaming, - // the application will complete with a 'failed' status - logger.info("Exited with System.exit(0) from Spark Streaming"); - } - - // Teardown the context - computeSR.teardown(); - } - else if (SystemConfiguration.getProperty(ResponderProps.PLATFORM).equals("standalone")) + switch (platform) { - logger.info("Launching Standalone Responder:"); - - String queryInput = SystemConfiguration.getProperty("pir.queryInput"); - Query query = new LocalFileSystemStore().recall(queryInput, Query.class); - - Responder pirResponder = new Responder(query); - pirResponder.computeStandaloneResponse(); + case MAPREDUCE: + logger.info("Launching MapReduce ResponderTool:"); + + ComputeResponseTool pirWLTool = new ComputeResponseTool(); + ToolRunner.run(pirWLTool, new String[] {}); + break; + + case SPARK: + logger.info("Launching Spark ComputeResponse:"); + + ComputeResponse computeResponse = new ComputeResponse(FileSystem.get(new Configuration())); + computeResponse.performQuery(); + break; + + case SPARKSTREAMING: + logger.info("Launching Spark ComputeStreamingResponse:"); + + ComputeStreamingResponse computeSR = new ComputeStreamingResponse(FileSystem.get(new Configuration())); + try + { + computeSR.performQuery(); + } catch (SystemExitException e) + { + // If System.exit(0) is not caught from Spark Streaming, + // the application will complete with a 'failed' status + logger.info("Exited with System.exit(0) from Spark Streaming"); + } + + // Teardown the context + computeSR.teardown(); + break; + + case STORM: + logger.info("Launching Storm PirkTopology:"); + PirkTopology.runPirkTopology(); + break; + + case STANDALONE: + logger.info("Launching Standalone Responder:"); + + String queryInput = SystemConfiguration.getProperty("pir.queryInput"); + Query query = new LocalFileSystemStore().recall(queryInput, Query.class); + + Responder pirResponder = new Responder(query); + pirResponder.computeStandaloneResponse(); + break; } }
