Repository: incubator-pirk Updated Branches: refs/heads/master 10e08c434 -> 6c1e43efa
[PIRK-47] Change Query name and number to a unique identifier - closes apache/incubator-pirk#54 Project: http://git-wip-us.apache.org/repos/asf/incubator-pirk/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-pirk/commit/6c1e43ef Tree: http://git-wip-us.apache.org/repos/asf/incubator-pirk/tree/6c1e43ef Diff: http://git-wip-us.apache.org/repos/asf/incubator-pirk/diff/6c1e43ef Branch: refs/heads/master Commit: 6c1e43efacee7a6dd0152c41cd6e02dcb2e15e4b Parents: 10e08c4 Author: tellison <[email protected]> Authored: Thu Aug 11 09:19:13 2016 -0400 Committer: eawilliams <[email protected]> Committed: Thu Aug 11 09:19:13 2016 -0400 ---------------------------------------------------------------------- .../org/apache/pirk/encryption/Paillier.java | 2 - .../apache/pirk/querier/wideskies/Querier.java | 17 +---- .../pirk/querier/wideskies/QuerierDriver.java | 13 ++-- .../querier/wideskies/QuerierDriverCLI.java | 7 --- .../pirk/querier/wideskies/QuerierProps.java | 11 +--- .../wideskies/decrypt/DecryptResponse.java | 2 +- .../querier/wideskies/encrypt/EncryptQuery.java | 2 +- .../apache/pirk/query/wideskies/QueryInfo.java | 65 ++++++++------------ .../mapreduce/ComputeResponseTool.java | 4 +- .../wideskies/spark/ComputeExpLookupTable.java | 2 +- .../pirk/schema/response/QueryResponseJSON.java | 6 +- .../distributed/testsuite/DistTestSuite.java | 39 +++--------- .../org/apache/pirk/test/utils/BaseTests.java | 33 +++++----- .../apache/pirk/test/utils/StandaloneQuery.java | 10 +-- .../org/apache/pirk/test/utils/TestUtils.java | 9 +-- src/main/resources/querier.properties | 3 - 16 files changed, 75 insertions(+), 150 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/6c1e43ef/src/main/java/org/apache/pirk/encryption/Paillier.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/encryption/Paillier.java b/src/main/java/org/apache/pirk/encryption/Paillier.java index dec6c30..6c62fe8 100644 --- a/src/main/java/org/apache/pirk/encryption/Paillier.java +++ b/src/main/java/org/apache/pirk/encryption/Paillier.java @@ -144,8 +144,6 @@ public class Paillier implements Cloneable, Serializable public Paillier(int bitLengthInput, int certainty) throws PIRException { this(bitLengthInput, certainty, -1); - - logger.info("Parameters = " + parametersToString()); } /** http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/6c1e43ef/src/main/java/org/apache/pirk/querier/wideskies/Querier.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/querier/wideskies/Querier.java b/src/main/java/org/apache/pirk/querier/wideskies/Querier.java index 4d6523d..0a94541 100644 --- a/src/main/java/org/apache/pirk/querier/wideskies/Querier.java +++ b/src/main/java/org/apache/pirk/querier/wideskies/Querier.java @@ -24,10 +24,7 @@ import java.util.HashMap; import org.apache.pirk.encryption.Paillier; import org.apache.pirk.query.wideskies.Query; -import org.apache.pirk.query.wideskies.QueryInfo; import org.apache.pirk.serialization.Storable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Class to hold the information necessary for the PIR querier to perform decryption @@ -37,10 +34,6 @@ public class Querier implements Serializable, Storable { private static final long serialVersionUID = 1L; - private static final Logger logger = LoggerFactory.getLogger(Querier.class); - - private QueryInfo queryInfo = null; - private Query query = null; // contains the query vectors and functionality private Paillier paillier = null; // Paillier encryption functionality @@ -52,11 +45,8 @@ public class Querier implements Serializable, Storable // if the selector is of variable lengths private HashMap<Integer,String> embedSelectorMap = null; - public Querier(QueryInfo queryInfoInput, ArrayList<String> selectorsInput, Paillier paillierInput, Query pirQueryInput, - HashMap<Integer,String> embedSelectorMapInput) + public Querier(ArrayList<String> selectorsInput, Paillier paillierInput, Query pirQueryInput, HashMap<Integer,String> embedSelectorMapInput) { - queryInfo = queryInfoInput; - selectors = selectorsInput; paillier = paillierInput; @@ -66,11 +56,6 @@ public class Querier implements Serializable, Storable embedSelectorMap = embedSelectorMapInput; } - public QueryInfo getPirWatchlist() - { - return queryInfo; - } - public Query getPirQuery() { return query; http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/6c1e43ef/src/main/java/org/apache/pirk/querier/wideskies/QuerierDriver.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/querier/wideskies/QuerierDriver.java b/src/main/java/org/apache/pirk/querier/wideskies/QuerierDriver.java index c8647c6..6061f3a 100644 --- a/src/main/java/org/apache/pirk/querier/wideskies/QuerierDriver.java +++ b/src/main/java/org/apache/pirk/querier/wideskies/QuerierDriver.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.io.Serializable; import java.math.BigInteger; import java.util.ArrayList; +import java.util.UUID; import org.apache.pirk.encryption.Paillier; import org.apache.pirk.querier.wideskies.decrypt.DecryptResponse; @@ -92,7 +93,6 @@ public class QuerierDriver implements Serializable int dataPartitionBitSize = 0; int paillierBitSize = 0; int certainty = 0; - String queryName = null; int bitSet = -1; boolean embedSelector = true; boolean useMemLookupTable = false; @@ -112,7 +112,6 @@ public class QuerierDriver implements Serializable if (action.equals("encrypt")) { queryType = SystemConfiguration.getProperty(QuerierProps.QUERYTYPE); - queryName = SystemConfiguration.getProperty(QuerierProps.QUERYNAME); hashBitSize = Integer.parseInt(SystemConfiguration.getProperty(QuerierProps.HASHBITSIZE)); hashKey = SystemConfiguration.getProperty(QuerierProps.HASHBITSIZE); dataPartitionBitSize = Integer.parseInt(SystemConfiguration.getProperty(QuerierProps.DATAPARTITIONSIZE)); @@ -157,17 +156,17 @@ public class QuerierDriver implements Serializable + "\n hashBitSize = " + hashBitSize + "\n hashKey = " + hashKey + "\n dataPartitionBitSize = " + dataPartitionBitSize + "\n paillierBitSize = " + paillierBitSize + "\n certainty = " + certainty); - // Read in the selectors and extract the queryNum - first line in the file + // Read in the selectors and extract the queryIdentifier - first line in the file ArrayList<String> selectors = FileIOUtils.readToArrayList(inputFile); - double queryNum = Double.parseDouble(selectors.get(0)); + UUID queryIdentifier = UUID.fromString(selectors.get(0)); selectors.remove(0); int numSelectors = selectors.size(); - logger.info("queryNum = " + queryNum + " numSelectors = " + numSelectors); + logger.info("queryIdentifier = " + queryIdentifier + " numSelectors = " + numSelectors); // Set the necessary QueryInfo and Paillier objects - QueryInfo queryInfo = new QueryInfo(queryNum, numSelectors, hashBitSize, hashKey, dataPartitionBitSize, queryType, queryName, paillierBitSize, - useMemLookupTable, embedSelector, useHDFSLookupTable); + QueryInfo queryInfo = new QueryInfo(queryIdentifier, numSelectors, hashBitSize, hashKey, dataPartitionBitSize, queryType, useMemLookupTable, + embedSelector, useHDFSLookupTable); if (SystemConfiguration.isSetTrue("pir.embedQuerySchema")) { http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/6c1e43ef/src/main/java/org/apache/pirk/querier/wideskies/QuerierDriverCLI.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/querier/wideskies/QuerierDriverCLI.java b/src/main/java/org/apache/pirk/querier/wideskies/QuerierDriverCLI.java index a5ea321..2bd581c 100644 --- a/src/main/java/org/apache/pirk/querier/wideskies/QuerierDriverCLI.java +++ b/src/main/java/org/apache/pirk/querier/wideskies/QuerierDriverCLI.java @@ -211,13 +211,6 @@ public class QuerierDriverCLI optionTYPE.setType(String.class); options.addOption(optionTYPE); - // NAME - Option optionNAME = new Option("qn", QuerierProps.QUERYNAME, true, "required for encryption -- Name of the query"); - optionNAME.setRequired(false); - optionNAME.setArgName(QuerierProps.QUERYNAME); - optionNAME.setType(String.class); - options.addOption(optionNAME); - // HASHBITSIZE Option optionHASHBITSIZE = new Option("hb", QuerierProps.HASHBITSIZE, true, "required -- Bit size of keyed hash"); optionHASHBITSIZE.setRequired(false); http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/6c1e43ef/src/main/java/org/apache/pirk/querier/wideskies/QuerierProps.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/querier/wideskies/QuerierProps.java b/src/main/java/org/apache/pirk/querier/wideskies/QuerierProps.java index e8820af..38fb088 100644 --- a/src/main/java/org/apache/pirk/querier/wideskies/QuerierProps.java +++ b/src/main/java/org/apache/pirk/querier/wideskies/QuerierProps.java @@ -48,7 +48,6 @@ public class QuerierProps public static final String PAILLIERBITSIZE = "querier.paillierBitSize"; public static final String BITSET = "querier.bitSet"; public static final String CERTAINTY = "querier.certainty"; - public static final String QUERYNAME = "querier.queryName"; public static final String QUERYSCHEMAS = "querier.querySchemas"; public static final String DATASCHEMAS = "querier.dataSchemas"; public static final String EMBEDSELECTOR = "querier.embedSelector"; @@ -62,8 +61,8 @@ public class QuerierProps public static final String QUERIERFILE = "querier.querierFile"; public static final List<String> PROPSLIST = Arrays.asList(ACTION, INPUTFILE, OUTPUTFILE, QUERYTYPE, NUMTHREADS, EMBEDQUERYSCHEMA, HASHBITSIZE, HASHKEY, - DATAPARTITIONSIZE, PAILLIERBITSIZE, BITSET, CERTAINTY, QUERYNAME, QUERYSCHEMAS, DATASCHEMAS, EMBEDSELECTOR, USEMEMLOOKUPTABLE, USEHDFSLOOKUPTABLE, - SR_ALGORITHM, SR_PROVIDER); + DATAPARTITIONSIZE, PAILLIERBITSIZE, BITSET, CERTAINTY, QUERYSCHEMAS, DATASCHEMAS, EMBEDSELECTOR, USEMEMLOOKUPTABLE, USEHDFSLOOKUPTABLE, SR_ALGORITHM, + SR_PROVIDER); /** * Validates the querier properties @@ -151,12 +150,6 @@ public class QuerierProps valid = false; } - if (!SystemConfiguration.hasProperty(QUERYNAME)) - { - logger.info("For action='encrypt': Must have the option " + QUERYNAME); - valid = false; - } - if (!SystemConfiguration.hasProperty(BITSET)) { logger.info("For action='encrypt': Must have the option " + BITSET); http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/6c1e43ef/src/main/java/org/apache/pirk/querier/wideskies/decrypt/DecryptResponse.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/querier/wideskies/decrypt/DecryptResponse.java b/src/main/java/org/apache/pirk/querier/wideskies/decrypt/DecryptResponse.java index 8eebf82..a9004fe 100644 --- a/src/main/java/org/apache/pirk/querier/wideskies/decrypt/DecryptResponse.java +++ b/src/main/java/org/apache/pirk/querier/wideskies/decrypt/DecryptResponse.java @@ -139,7 +139,7 @@ public class DecryptResponse // Create the runnable and execute // selectorMaskMap and rElements are synchronized, pirWatchlist is copied, selectors is partitioned - DecryptResponseRunnable runDec = new DecryptResponseRunnable(rElements, selectorsPartition, selectorMaskMap, queryInfo.copy(), embedSelectorMap); + DecryptResponseRunnable runDec = new DecryptResponseRunnable(rElements, selectorsPartition, selectorMaskMap, queryInfo.clone(), embedSelectorMap); runnables.add(runDec); es.execute(runDec); } http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/6c1e43ef/src/main/java/org/apache/pirk/querier/wideskies/encrypt/EncryptQuery.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/querier/wideskies/encrypt/EncryptQuery.java b/src/main/java/org/apache/pirk/querier/wideskies/encrypt/EncryptQuery.java index 9e2099a..e02dacc 100644 --- a/src/main/java/org/apache/pirk/querier/wideskies/encrypt/EncryptQuery.java +++ b/src/main/java/org/apache/pirk/querier/wideskies/encrypt/EncryptQuery.java @@ -158,7 +158,7 @@ public class EncryptQuery } // Set the Querier object - querier = new Querier(queryInfo, selectors, paillier, query, embedSelectorMap); + querier = new Querier(selectors, paillier, query, embedSelectorMap); } private HashMap<Integer,Integer> computeSelectorQueryVecMap() http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/6c1e43ef/src/main/java/org/apache/pirk/query/wideskies/QueryInfo.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/query/wideskies/QueryInfo.java b/src/main/java/org/apache/pirk/query/wideskies/QueryInfo.java index 8c69fa9..6a3bb15 100644 --- a/src/main/java/org/apache/pirk/query/wideskies/QueryInfo.java +++ b/src/main/java/org/apache/pirk/query/wideskies/QueryInfo.java @@ -19,6 +19,7 @@ package org.apache.pirk.query.wideskies; import java.io.Serializable; +import java.util.UUID; import org.apache.pirk.schema.query.QuerySchema; import org.apache.pirk.schema.query.QuerySchemaRegistry; @@ -32,21 +33,17 @@ import org.slf4j.LoggerFactory; * we no longer have collisions * */ -public class QueryInfo implements Serializable +public class QueryInfo implements Serializable, Cloneable { private static final long serialVersionUID = 1L; private static final Logger logger = LoggerFactory.getLogger(QueryInfo.class); - private double queryNum = 0.0; // the identifier num of the query + private UUID identifier; // the identifier of the query private int numSelectors = 0; // the number of selectors in the query, given by \floor{paillerBitSize/dataPartitionBitSize} private String queryType = null; // QueryType string const - private String queryName = null; // Name of query - - private int paillierBitSize = 0; // Paillier modulus size - private int hashBitSize = 0; // Bit size of the keyed hash function private String hashKey = null; // Key for the keyed hash function @@ -66,13 +63,18 @@ public class QueryInfo implements Serializable QuerySchema qSchema = null; - public QueryInfo(double queryNumInput, int numSelectorsInput, int hashBitSizeInput, String hashKeyInput, int dataPartitionBitSizeInput, - String queryTypeInput, String queryNameInput, int paillierBitSizeIn, boolean useExpLookupTableInput, boolean embedSelectorInput, - boolean useHDFSExpLookupTableInput) + public QueryInfo(int numSelectorsInput, int hashBitSizeInput, String hashKeyInput, int dataPartitionBitSizeInput, String queryTypeInput, + boolean useExpLookupTableInput, boolean embedSelectorInput, boolean useHDFSExpLookupTableInput) + { + this(UUID.randomUUID(), numSelectorsInput, hashBitSizeInput, hashKeyInput, dataPartitionBitSizeInput, queryTypeInput, useExpLookupTableInput, + embedSelectorInput, useHDFSExpLookupTableInput); + } + + public QueryInfo(UUID identifierInput, int numSelectorsInput, int hashBitSizeInput, String hashKeyInput, int dataPartitionBitSizeInput, + String queryTypeInput, boolean useExpLookupTableInput, boolean embedSelectorInput, boolean useHDFSExpLookupTableInput) { - queryNum = queryNumInput; + identifier = identifierInput; queryType = queryTypeInput; - queryName = queryNameInput; numSelectors = numSelectorsInput; @@ -87,8 +89,6 @@ public class QueryInfo implements Serializable dataPartitionBitSize = dataPartitionBitSizeInput; numPartitionsPerDataElement = numBitsPerDataElement / dataPartitionBitSizeInput; - paillierBitSize = paillierBitSizeIn; - if (embedSelectorInput) { numPartitionsPerDataElement += 4; // using a 8-bit partition size and a 32-bit embedded selector @@ -97,16 +97,9 @@ public class QueryInfo implements Serializable printQueryInfo(); } - public QueryInfo(double queryNumInput, int numSelectorsInput, int hashBitSizeInput, String hashKeyInput, int dataPartitionBitSizeInput, - String queryTypeInput, String queryNameInput, int paillierBitSizeIn) - { - this(queryNumInput, numSelectorsInput, hashBitSizeInput, hashKeyInput, dataPartitionBitSizeInput, queryTypeInput, queryNameInput, paillierBitSizeIn, false, - true, false); - } - - public double getQueryNum() + public UUID getIdentifier() { - return queryNum; + return identifier; } public String getQueryType() @@ -114,21 +107,11 @@ public class QueryInfo implements Serializable return queryType; } - public String getQueryName() - { - return queryName; - } - public int getNumSelectors() { return numSelectors; } - public int getPaillierBitSize() - { - return paillierBitSize; - } - public int getHashBitSize() { return hashBitSize; @@ -181,15 +164,21 @@ public class QueryInfo implements Serializable public void printQueryInfo() { - logger.info("queryNum = " + queryNum + " numSelectors = " + numSelectors + " hashBitSize = " + hashBitSize + " hashKey = " + hashKey + logger.info("identifier = " + identifier + " numSelectors = " + numSelectors + " hashBitSize = " + hashBitSize + " hashKey = " + hashKey + " dataPartitionBitSize = " + dataPartitionBitSize + " numBitsPerDataElement = " + numBitsPerDataElement + " numPartitionsPerDataElement = " - + numPartitionsPerDataElement + " queryType = " + queryType + " queryName = " + queryName + " paillierBitSize = " + paillierBitSize - + " useExpLookupTable = " + useExpLookupTable + " useHDFSExpLookupTable = " + useHDFSExpLookupTable + " embedSelector = " + embedSelector); + + numPartitionsPerDataElement + " queryType = " + queryType + " useExpLookupTable = " + useExpLookupTable + " useHDFSExpLookupTable = " + + useHDFSExpLookupTable + " embedSelector = " + embedSelector); } - public QueryInfo copy() + @Override + public QueryInfo clone() { - return new QueryInfo(this.queryNum, this.numSelectors, this.hashBitSize, this.hashKey, this.dataPartitionBitSize, this.queryType, this.queryName, - this.paillierBitSize, this.useExpLookupTable, this.embedSelector, this.useHDFSExpLookupTable); + try + { + return (QueryInfo) super.clone(); + } catch (CloneNotSupportedException e) + { + throw new RuntimeException(e); + } } } http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/6c1e43ef/src/main/java/org/apache/pirk/responder/wideskies/mapreduce/ComputeResponseTool.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/responder/wideskies/mapreduce/ComputeResponseTool.java b/src/main/java/org/apache/pirk/responder/wideskies/mapreduce/ComputeResponseTool.java index 15d62df..98f58c6 100644 --- a/src/main/java/org/apache/pirk/responder/wideskies/mapreduce/ComputeResponseTool.java +++ b/src/main/java/org/apache/pirk/responder/wideskies/mapreduce/ComputeResponseTool.java @@ -237,7 +237,7 @@ public class ComputeResponseTool extends Configured implements Tool logger.info("Creating expTable"); // The split location for the interim calculations, delete upon completion - Path splitDir = new Path("/tmp/splits-" + queryInfo.getQueryNum()); + Path splitDir = new Path("/tmp/splits-" + queryInfo.getIdentifier()); if (fs.exists(splitDir)) { fs.delete(splitDir, true); @@ -263,7 +263,7 @@ public class ComputeResponseTool extends Configured implements Tool // Run the job to generate the expTable // Job jobExp = new Job(mrConfig.getConfig(), "pirExp-" + pirWL.getWatchlistNum()); - Job jobExp = new Job(conf, "pirExp-" + queryInfo.getQueryNum()); + Job jobExp = new Job(conf, "pirExp-" + queryInfo.getIdentifier()); jobExp.setSpeculativeExecution(false); jobExp.getConfiguration().set("mapreduce.map.speculative", "false"); http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/6c1e43ef/src/main/java/org/apache/pirk/responder/wideskies/spark/ComputeExpLookupTable.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/responder/wideskies/spark/ComputeExpLookupTable.java b/src/main/java/org/apache/pirk/responder/wideskies/spark/ComputeExpLookupTable.java index 969b6ec..6b64e35 100644 --- a/src/main/java/org/apache/pirk/responder/wideskies/spark/ComputeExpLookupTable.java +++ b/src/main/java/org/apache/pirk/responder/wideskies/spark/ComputeExpLookupTable.java @@ -67,7 +67,7 @@ public class ComputeExpLookupTable { JavaPairRDD<Integer,Iterable<Tuple2<Integer,BigInteger>>> expCalculations; - logger.info("Creating expTable in hdfs for queryName = " + query.getQueryInfo().getQueryName()); + logger.info("Creating expTable in hdfs for query identifier = " + query.getQueryInfo().getIdentifier()); // Prep the output directory Path outPathExp = new Path(outputDirExp); http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/6c1e43ef/src/main/java/org/apache/pirk/schema/response/QueryResponseJSON.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/schema/response/QueryResponseJSON.java b/src/main/java/org/apache/pirk/schema/response/QueryResponseJSON.java index e9c1161..d32d69b 100644 --- a/src/main/java/org/apache/pirk/schema/response/QueryResponseJSON.java +++ b/src/main/java/org/apache/pirk/schema/response/QueryResponseJSON.java @@ -57,9 +57,6 @@ public class QueryResponseJSON implements Serializable public static final String QUERY_ID = "query_id"; // query ID that generated the notification public static final Text QUERY_ID_TEXT = new Text(QUERY_ID); - public static final String QUERY_NAME = "query_name"; // name of the query that generated the notification - public static final Text QUERY_NAME_TEXT = new Text(QUERY_NAME); - public static final String SELECTOR = "match"; // tag for selector that generated the hit public static final Text SELECTOR_TEXT = new Text(SELECTOR); @@ -203,8 +200,7 @@ public class QueryResponseJSON implements Serializable public void setGeneralQueryResponseFields(QueryInfo queryInfo) { jsonObj.put(EVENT_TYPE, queryInfo.getQueryType()); - jsonObj.put(QUERY_ID, queryInfo.getQueryNum()); - jsonObj.put(QUERY_NAME, queryInfo.getQueryName()); + jsonObj.put(QUERY_ID, queryInfo.getIdentifier().toString()); } @Override http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/6c1e43ef/src/main/java/org/apache/pirk/test/distributed/testsuite/DistTestSuite.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/test/distributed/testsuite/DistTestSuite.java b/src/main/java/org/apache/pirk/test/distributed/testsuite/DistTestSuite.java index 2e3464e..0556ead 100644 --- a/src/main/java/org/apache/pirk/test/distributed/testsuite/DistTestSuite.java +++ b/src/main/java/org/apache/pirk/test/distributed/testsuite/DistTestSuite.java @@ -20,6 +20,7 @@ package org.apache.pirk.test.distributed.testsuite; import java.io.File; import java.util.ArrayList; +import java.util.List; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -321,52 +322,32 @@ public class DistTestSuite } // Base method to perform query - @SuppressWarnings("unused") - public static ArrayList<QueryResponseJSON> performQuery(String queryType, ArrayList<String> selectors, FileSystem fs, boolean isSpark, int numThreads) + public static List<QueryResponseJSON> performQuery(String queryType, ArrayList<String> selectors, FileSystem fs, boolean isSpark, int numThreads) throws Exception { logger.info("performQuery: "); String queryInputDir = SystemConfiguration.getProperty(DistributedTestDriver.PIR_QUERY_INPUT_DIR); String outputFile = SystemConfiguration.getProperty(DistributedTestDriver.OUTPUT_DIRECTORY_PROPERTY); - Path outputFilePath = new Path(outputFile); - if (fs.exists(outputFilePath)) - { - fs.delete(new Path(outputFile), true); - } + fs.delete(new Path(outputFile), true); // Ensure old output does not exist. + SystemConfiguration.setProperty("pir.queryInput", queryInputDir); SystemConfiguration.setProperty("pir.outputFile", outputFile); SystemConfiguration.setProperty("pir.numReduceTasks", "1"); SystemConfiguration.setProperty("pir.stopListFile", SystemConfiguration.getProperty(DistributedTestDriver.PIR_STOPLIST_FILE)); - ArrayList<QueryResponseJSON> results; - // Create the temp result file File fileFinalResults = File.createTempFile("finalResultsFile", ".txt"); fileFinalResults.deleteOnExit(); logger.info("fileFinalResults = " + fileFinalResults.getAbsolutePath()); - boolean embedSelector = false; - if (SystemConfiguration.getBooleanProperty("pirTest.embedSelector", false)) - { - embedSelector = true; - } - - boolean useExpLookupTable = false; - if (SystemConfiguration.getBooleanProperty("pirTest.useExpLookupTable", false)) - { - useExpLookupTable = true; - } - - boolean useHDFSExpLookupTable = false; - if (SystemConfiguration.getBooleanProperty("pirTest.useHDFSExpLookupTable", false)) - { - useHDFSExpLookupTable = true; - } + boolean embedSelector = SystemConfiguration.getBooleanProperty("pirTest.embedSelector", false); + boolean useExpLookupTable = SystemConfiguration.getBooleanProperty("pirTest.useExpLookupTable", false); + boolean useHDFSExpLookupTable = SystemConfiguration.getBooleanProperty("pirTest.useHDFSExpLookupTable", false); // Set the necessary objects - QueryInfo queryInfo = new QueryInfo(BaseTests.queryNum, selectors.size(), BaseTests.hashBitSize, BaseTests.hashKey, BaseTests.dataPartitionBitSize, - queryType, queryType + "_" + BaseTests.queryNum, BaseTests.paillierBitSize, useExpLookupTable, embedSelector, useHDFSExpLookupTable); + QueryInfo queryInfo = new QueryInfo(BaseTests.queryIdentifier, selectors.size(), BaseTests.hashBitSize, BaseTests.hashKey, BaseTests.dataPartitionBitSize, + queryType, useExpLookupTable, embedSelector, useHDFSExpLookupTable); Paillier paillier = new Paillier(BaseTests.paillierBitSize, BaseTests.certainty); @@ -463,7 +444,7 @@ public class DistTestSuite // Read in results logger.info("Reading in and checking results"); - results = TestUtils.readResultsFile(fileFinalResults); + List<QueryResponseJSON> results = TestUtils.readResultsFile(fileFinalResults); // Reset data and query schema properties SystemConfiguration.setProperty("data.schemas", dataSchemaProp); http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/6c1e43ef/src/main/java/org/apache/pirk/test/utils/BaseTests.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/test/utils/BaseTests.java b/src/main/java/org/apache/pirk/test/utils/BaseTests.java index afb3144..e99744c 100644 --- a/src/main/java/org/apache/pirk/test/utils/BaseTests.java +++ b/src/main/java/org/apache/pirk/test/utils/BaseTests.java @@ -23,7 +23,9 @@ import static org.junit.Assert.fail; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; +import java.util.List; import java.util.Set; +import java.util.UUID; import org.apache.hadoop.fs.FileSystem; import org.apache.pirk.query.wideskies.QueryUtils; @@ -45,10 +47,10 @@ public class BaseTests { private static final Logger logger = LoggerFactory.getLogger(BaseTests.class); - public static double queryNum = 1.0; + public static UUID queryIdentifier = UUID.randomUUID(); public static int dataPartitionBitSize = 8; - // Selectors for domain and IP queries, queryNum is the first entry for file generation + // Selectors for domain and IP queries, queryIdentifier is the first entry for file generation private static ArrayList<String> selectorsDomain = new ArrayList<>(Arrays.asList("s.t.u.net", "d.e.com", "r.r.r.r", "a.b.c.com", "something.else", "x.y.net")); private static ArrayList<String> selectorsIP = new ArrayList<>(Arrays.asList("55.55.55.55", "5.6.7.8", "10.20.30.40", "13.14.15.16", "21.22.23.24")); @@ -78,7 +80,7 @@ public class BaseTests QuerySchema qSchema = QuerySchemaRegistry.get(Inputs.DNS_HOSTNAME_QUERY); int numExpectedResults = 6; - ArrayList<QueryResponseJSON> results; + List<QueryResponseJSON> results; if (isDistributed) { results = DistTestSuite.performQuery(Inputs.DNS_HOSTNAME_QUERY, selectorsDomain, fs, isSpark, numThreads); @@ -155,8 +157,7 @@ public class BaseTests if (addElement) { QueryResponseJSON wlJSON = new QueryResponseJSON(); - wlJSON.setMapping(QueryResponseJSON.QUERY_ID, queryNum); - wlJSON.setMapping(QueryResponseJSON.QUERY_NAME, Inputs.DNS_HOSTNAME_QUERY + "_" + queryNum); + wlJSON.setMapping(QueryResponseJSON.QUERY_ID, queryIdentifier.toString()); wlJSON.setMapping(QueryResponseJSON.EVENT_TYPE, Inputs.DNS_HOSTNAME_QUERY); wlJSON.setMapping(Inputs.DATE, dataMap.get(Inputs.DATE)); wlJSON.setMapping(Inputs.SRCIP, dataMap.get(Inputs.SRCIP)); @@ -201,7 +202,7 @@ public class BaseTests logger.info("Running testDNSIPQuery(): "); QuerySchema qSchema = QuerySchemaRegistry.get(Inputs.DNS_IP_QUERY); - ArrayList<QueryResponseJSON> results; + List<QueryResponseJSON> results; if (isDistributed) { @@ -237,8 +238,7 @@ public class BaseTests if (addElement) { QueryResponseJSON wlJSON = new QueryResponseJSON(); - wlJSON.setMapping(QueryResponseJSON.QUERY_ID, queryNum); - wlJSON.setMapping(QueryResponseJSON.QUERY_NAME, Inputs.DNS_IP_QUERY + "_" + queryNum); + wlJSON.setMapping(QueryResponseJSON.QUERY_ID, queryIdentifier); wlJSON.setMapping(QueryResponseJSON.EVENT_TYPE, Inputs.DNS_IP_QUERY); wlJSON.setMapping(Inputs.SRCIP, dataMap.get(Inputs.SRCIP)); wlJSON.setMapping(Inputs.DSTIP, dataMap.get(Inputs.DSTIP)); @@ -276,7 +276,7 @@ public class BaseTests logger.info("Running testDNSNXDOMAINQuery(): "); QuerySchema qSchema = QuerySchemaRegistry.get(Inputs.DNS_NXDOMAIN_QUERY); - ArrayList<QueryResponseJSON> results; + List<QueryResponseJSON> results; if (isDistributed) { @@ -302,8 +302,7 @@ public class BaseTests if (dataMap.get(Inputs.RCODE).toString().equals("3")) { QueryResponseJSON wlJSON = new QueryResponseJSON(); - wlJSON.setMapping(QueryResponseJSON.QUERY_ID, queryNum); - wlJSON.setMapping(QueryResponseJSON.QUERY_NAME, Inputs.DNS_NXDOMAIN_QUERY + "_" + queryNum); + wlJSON.setMapping(QueryResponseJSON.QUERY_ID, queryIdentifier); wlJSON.setMapping(QueryResponseJSON.EVENT_TYPE, Inputs.DNS_NXDOMAIN_QUERY); wlJSON.setMapping(Inputs.QNAME, dataMap.get(Inputs.QNAME)); // this gets re-embedded as the original selector after decryption wlJSON.setMapping(Inputs.DSTIP, dataMap.get(Inputs.DSTIP)); @@ -340,7 +339,7 @@ public class BaseTests logger.info("Running testSRCIPQuery(): "); QuerySchema qSchema = QuerySchemaRegistry.get(Inputs.DNS_SRCIP_QUERY); - ArrayList<QueryResponseJSON> results; + List<QueryResponseJSON> results; int removeTailElements = 0; int numExpectedResults = 1; @@ -376,8 +375,7 @@ public class BaseTests { // Form the correct result QueryResponseJSON object QueryResponseJSON qrJSON = new QueryResponseJSON(); - qrJSON.setMapping(QueryResponseJSON.QUERY_ID, queryNum); - qrJSON.setMapping(QueryResponseJSON.QUERY_NAME, Inputs.DNS_SRCIP_QUERY + "_" + queryNum); + qrJSON.setMapping(QueryResponseJSON.QUERY_ID, queryIdentifier); qrJSON.setMapping(QueryResponseJSON.EVENT_TYPE, Inputs.DNS_SRCIP_QUERY); qrJSON.setMapping(Inputs.QNAME, parseString(dataMap, Inputs.QNAME)); qrJSON.setMapping(Inputs.DSTIP, dataMap.get(Inputs.DSTIP)); @@ -414,7 +412,7 @@ public class BaseTests logger.info("Running testSRCIPQueryNoFilter(): "); QuerySchema qSchema = QuerySchemaRegistry.get(Inputs.DNS_SRCIP_QUERY_NO_FILTER); - ArrayList<QueryResponseJSON> results; + List<QueryResponseJSON> results; int numExpectedResults = 3; if (isDistributed) @@ -447,8 +445,7 @@ public class BaseTests { // Form the correct result QueryResponseJSON object QueryResponseJSON qrJSON = new QueryResponseJSON(); - qrJSON.setMapping(QueryResponseJSON.QUERY_ID, queryNum); - qrJSON.setMapping(QueryResponseJSON.QUERY_NAME, Inputs.DNS_SRCIP_QUERY_NO_FILTER + "_" + queryNum); + qrJSON.setMapping(QueryResponseJSON.QUERY_ID, queryIdentifier); qrJSON.setMapping(QueryResponseJSON.EVENT_TYPE, Inputs.DNS_SRCIP_QUERY_NO_FILTER); qrJSON.setMapping(Inputs.QNAME, parseString(dataMap, Inputs.QNAME)); qrJSON.setMapping(Inputs.DSTIP, dataMap.get(Inputs.DSTIP)); @@ -636,7 +633,7 @@ public class BaseTests return set; } - private static void printResultList(ArrayList<QueryResponseJSON> list) + private static void printResultList(List<QueryResponseJSON> list) { for (QueryResponseJSON obj : list) { http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/6c1e43ef/src/main/java/org/apache/pirk/test/utils/StandaloneQuery.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/test/utils/StandaloneQuery.java b/src/main/java/org/apache/pirk/test/utils/StandaloneQuery.java index 4e98f5d..a4bf67d 100644 --- a/src/main/java/org/apache/pirk/test/utils/StandaloneQuery.java +++ b/src/main/java/org/apache/pirk/test/utils/StandaloneQuery.java @@ -24,6 +24,7 @@ import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import org.apache.pirk.encryption.Paillier; import org.apache.pirk.querier.wideskies.Querier; @@ -56,12 +57,11 @@ public class StandaloneQuery String testQuerySchemaName = "testQuerySchema"; // Base method to perform the query - public static ArrayList<QueryResponseJSON> performStandaloneQuery(ArrayList<JSONObject> dataElements, String queryType, ArrayList<String> selectors, + public static List<QueryResponseJSON> performStandaloneQuery(ArrayList<JSONObject> dataElements, String queryType, ArrayList<String> selectors, int numThreads, boolean testFalsePositive) throws IOException, InterruptedException, PIRException { logger.info("Performing watchlisting: "); - ArrayList<QueryResponseJSON> results = null; QuerySchema qSchema = QuerySchemaRegistry.get(queryType); // Create the necessary files @@ -82,8 +82,8 @@ public class StandaloneQuery boolean useHDFSExpLookupTable = SystemConfiguration.getBooleanProperty("pirTest.useHDFSExpLookupTable", false); // Set the necessary objects - QueryInfo queryInfo = new QueryInfo(BaseTests.queryNum, selectors.size(), BaseTests.hashBitSize, BaseTests.hashKey, BaseTests.dataPartitionBitSize, - queryType, queryType + "_" + BaseTests.queryNum, BaseTests.paillierBitSize, useExpLookupTable, embedSelector, useHDFSExpLookupTable); + QueryInfo queryInfo = new QueryInfo(BaseTests.queryIdentifier, selectors.size(), BaseTests.hashBitSize, BaseTests.hashKey, BaseTests.dataPartitionBitSize, + queryType, useExpLookupTable, embedSelector, useHDFSExpLookupTable); if (SystemConfiguration.getBooleanProperty("pir.embedQuerySchema", false)) { @@ -152,7 +152,7 @@ public class StandaloneQuery // Read in results logger.info("Reading in and checking results"); - results = TestUtils.readResultsFile(fileFinalResults); + List<QueryResponseJSON> results = TestUtils.readResultsFile(fileFinalResults); // Clean up fileQuerier.delete(); http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/6c1e43ef/src/main/java/org/apache/pirk/test/utils/TestUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/test/utils/TestUtils.java b/src/main/java/org/apache/pirk/test/utils/TestUtils.java index e1e03dc..c30e204 100644 --- a/src/main/java/org/apache/pirk/test/utils/TestUtils.java +++ b/src/main/java/org/apache/pirk/test/utils/TestUtils.java @@ -267,14 +267,11 @@ public class TestUtils /** * Converts the result file into an ArrayList of QueryResponseJSON objects */ - public static ArrayList<QueryResponseJSON> readResultsFile(File file) + public static List<QueryResponseJSON> readResultsFile(File file) { - ArrayList<QueryResponseJSON> results = new ArrayList<>(); - try + List<QueryResponseJSON> results = new ArrayList<>(); + try (BufferedReader br = new BufferedReader(new FileReader(file))) { - FileReader fr = new FileReader(file); - BufferedReader br = new BufferedReader(fr); - String line; while ((line = br.readLine()) != null) { http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/6c1e43ef/src/main/resources/querier.properties ---------------------------------------------------------------------- diff --git a/src/main/resources/querier.properties b/src/main/resources/querier.properties index 2051d46..88352d7 100644 --- a/src/main/resources/querier.properties +++ b/src/main/resources/querier.properties @@ -100,9 +100,6 @@ querier.numThreads= #paillierBitSize -- required for encryption -- Paillier modulus size N #querier.paillierBitSize= -#queryName -- required for encryption -- Name of the query -#querier.queryName= - #queryType -- required for encryption #Type of the query as defined in the 'schemaName' tag of the corresponding query schema file #querier.queryType=
