Repository: incubator-pirk Updated Branches: refs/heads/master f4c40e514 -> 483fac0ab
various cleanup items - closes apache/incubator-pirk#73 Project: http://git-wip-us.apache.org/repos/asf/incubator-pirk/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-pirk/commit/483fac0a Tree: http://git-wip-us.apache.org/repos/asf/incubator-pirk/tree/483fac0a Diff: http://git-wip-us.apache.org/repos/asf/incubator-pirk/diff/483fac0a Branch: refs/heads/master Commit: 483fac0abe0a7903e7fa6de379d237aead817a64 Parents: f4c40e5 Author: eawilliams <[email protected]> Authored: Fri Aug 19 14:58:26 2016 -0400 Committer: eawilliams <[email protected]> Committed: Fri Aug 19 14:58:26 2016 -0400 ---------------------------------------------------------------------- .../apache/pirk/encryption/PrimeGenerator.java | 8 +- .../inputformat/hadoop/BytesArrayWritable.java | 10 +- .../apache/pirk/querier/wideskies/Querier.java | 1 - .../pirk/querier/wideskies/QuerierCLI.java | 327 +++++++++++++++++++ .../pirk/querier/wideskies/QuerierDriver.java | 2 +- .../querier/wideskies/QuerierDriverCLI.java | 327 ------------------- .../apache/pirk/query/wideskies/QueryUtils.java | 12 +- .../wideskies/common/ComputeEncryptedRow.java | 24 +- .../HashSelectorsAndPartitionDataMapper.java | 1 + .../wideskies/mapreduce/RowCalcReducer.java | 9 +- .../responder/wideskies/spark/EncRowCalc.java | 1 + .../wideskies/spark/ExpTableGenerator.java | 1 + .../spark/HashSelectorsAndPartitionData.java | 1 + .../wideskies/standalone/Responder.java | 16 +- .../pirk/schema/data/DataSchemaLoader.java | 1 + .../test/distributed/DistributedTestDriver.java | 15 +- .../distributed/testsuite/DistTestSuite.java | 4 +- .../org/apache/pirk/test/utils/BaseTests.java | 18 +- .../java/org/apache/pirk/test/utils/Inputs.java | 18 +- .../apache/pirk/test/utils/StandaloneQuery.java | 14 +- .../java/org/apache/pirk/utils/KeyedHash.java | 6 +- src/main/resources/pirk.properties | 13 +- src/main/resources/querier.properties | 2 +- .../apache/pirk/general/PartitionUtilsTest.java | 14 +- .../pirk/schema/query/LoadQuerySchemaTest.java | 2 +- .../wideskies/standalone/StandaloneTest.java | 2 +- 26 files changed, 425 insertions(+), 424 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/483fac0a/src/main/java/org/apache/pirk/encryption/PrimeGenerator.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/encryption/PrimeGenerator.java b/src/main/java/org/apache/pirk/encryption/PrimeGenerator.java index 8e82fc7..c2fad01 100644 --- a/src/main/java/org/apache/pirk/encryption/PrimeGenerator.java +++ b/src/main/java/org/apache/pirk/encryption/PrimeGenerator.java @@ -18,15 +18,15 @@ */ package org.apache.pirk.encryption; -import org.apache.pirk.utils.SystemConfiguration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.math.BigDecimal; import java.math.BigInteger; import java.util.HashMap; import java.util.Random; +import org.apache.pirk.utils.SystemConfiguration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * Class to generate the primes used in the Paillier cryptosystem * <p> http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/483fac0a/src/main/java/org/apache/pirk/inputformat/hadoop/BytesArrayWritable.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/inputformat/hadoop/BytesArrayWritable.java b/src/main/java/org/apache/pirk/inputformat/hadoop/BytesArrayWritable.java index b8db7fd..ac94f2a 100644 --- a/src/main/java/org/apache/pirk/inputformat/hadoop/BytesArrayWritable.java +++ b/src/main/java/org/apache/pirk/inputformat/hadoop/BytesArrayWritable.java @@ -18,17 +18,17 @@ */ package org.apache.pirk.inputformat.hadoop; -import org.apache.hadoop.io.ArrayWritable; -import org.apache.hadoop.io.BytesWritable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.ByteArrayOutputStream; import java.io.IOException; import java.math.BigInteger; import java.util.Arrays; import java.util.List; +import org.apache.hadoop.io.ArrayWritable; +import org.apache.hadoop.io.BytesWritable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * ArrayWritable class with ByteWritable entries * http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/483fac0a/src/main/java/org/apache/pirk/querier/wideskies/Querier.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/querier/wideskies/Querier.java b/src/main/java/org/apache/pirk/querier/wideskies/Querier.java index 48be51d..172e1e5 100644 --- a/src/main/java/org/apache/pirk/querier/wideskies/Querier.java +++ b/src/main/java/org/apache/pirk/querier/wideskies/Querier.java @@ -19,7 +19,6 @@ package org.apache.pirk.querier.wideskies; import java.io.Serializable; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/483fac0a/src/main/java/org/apache/pirk/querier/wideskies/QuerierCLI.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/querier/wideskies/QuerierCLI.java b/src/main/java/org/apache/pirk/querier/wideskies/QuerierCLI.java new file mode 100644 index 0000000..024a89d --- /dev/null +++ b/src/main/java/org/apache/pirk/querier/wideskies/QuerierCLI.java @@ -0,0 +1,327 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pirk.querier.wideskies; + +import java.io.File; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.GnuParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.pirk.utils.SystemConfiguration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Class for parsing the command line options for the QuerierDriver + */ +public class QuerierCLI +{ + private static final Logger logger = LoggerFactory.getLogger(QuerierCLI.class); + + private Options cliOptions = null; + private CommandLine commandLine = null; + + private static final String LOCALPROPFILE = "local.querier.properties"; + + /** + * Create and parse allowable options + * + */ + public QuerierCLI(String[] args) + { + // Create the command line options + cliOptions = createOptions(); + + try + { + // Parse the command line options + CommandLineParser parser = new GnuParser(); + commandLine = parser.parse(cliOptions, args, true); + + // if help option is selected, just print help text and exit + if (hasOption("h")) + { + printHelp(); + System.exit(1); + } + + // Parse and validate the options provided + if (!parseOptions()) + { + logger.info("The provided options are not valid"); + printHelp(); + System.exit(1); + } + } catch (Exception e) + { + e.printStackTrace(); + System.exit(1); + } + } + + /** + * Determine if an option was provided by the user via the CLI + * + * @param option + * - the option of interest + * @return true if option was provided, false otherwise + */ + public boolean hasOption(String option) + { + return commandLine.hasOption(option); + } + + /** + * Obtain the argument of the option provided by the user via the CLI + * + * @param option + * - the option of interest + * @return value of the argument of the option + */ + public String getOptionValue(String option) + { + return commandLine.getOptionValue(option); + } + + /** + * Method to parse and validate the options provided + * + * @return - true if valid, false otherwise + */ + private boolean parseOptions() + { + boolean valid = true; + + // If we have a local.querier.properties file specified, load it + if (hasOption(LOCALPROPFILE)) + { + SystemConfiguration.loadPropsFromFile(new File(getOptionValue(LOCALPROPFILE))); + } + else + { + // Pull options, set as properties + for (String prop : QuerierProps.PROPSLIST) + { + if (hasOption(prop)) + { + SystemConfiguration.setProperty(prop, getOptionValue(prop)); + } + } + } + + // Validate properties + valid = QuerierProps.validateQuerierProperties(); + + return valid; + } + + /** + * Create the options available for the DistributedTestDriver + * + * @return Apache's CLI Options object + */ + private Options createOptions() + { + Options options = new Options(); + + // help + Option optionHelp = new Option("h", "help", false, "Print out the help documentation for this command line execution"); + optionHelp.setRequired(false); + options.addOption(optionHelp); + + // local.querier.properties + Option optionLocalPropFile = new Option("localPropFile", LOCALPROPFILE, true, "Optional local properties file"); + optionLocalPropFile.setRequired(false); + optionLocalPropFile.setArgName(LOCALPROPFILE); + optionLocalPropFile.setType(String.class); + options.addOption(optionLocalPropFile); + + // ACTION + Option optionACTION = new Option("a", QuerierProps.ACTION, true, "required - 'encrypt' or 'decrypt' -- The action performed by the QuerierDriver"); + optionACTION.setRequired(false); + optionACTION.setArgName(QuerierProps.ACTION); + optionACTION.setType(String.class); + options.addOption(optionACTION); + + // INPUTFILE + Option optionINPUTFILE = new Option("i", QuerierProps.INPUTFILE, true, "required - Fully qualified file containing input " + + "-- \n The input is either: \n (1) For Encryption: A query file - Contains the query selectors, one per line; " + + "the first line must be the query number \n OR \n (2) For Decryption: A response file - Contains the serialized Response object"); + optionINPUTFILE.setRequired(false); + optionINPUTFILE.setArgName(QuerierProps.INPUTFILE); + optionINPUTFILE.setType(String.class); + options.addOption(optionINPUTFILE); + + // OUTPUTFILE + Option optionOUTPUTFILE = new Option("o", QuerierProps.OUTPUTFILE, true, "required - Fully qualified file for the result output. " + + "\n The output file specifies either: \n (1) For encryption: \n \t (a) A file to contain the serialized Querier object named: " + "<outputFile>-" + + QuerierConst.QUERIER_FILETAG + " AND \n \t " + "(b) A file to contain the serialized Query object named: <outputFile>-" + QuerierConst.QUERY_FILETAG + + "\n " + "OR \n (2) A file to contain the decryption results where each line is where each line " + + "corresponds to one hit and is a JSON object with the schema QuerySchema"); + optionOUTPUTFILE.setRequired(false); + optionOUTPUTFILE.setArgName(QuerierProps.OUTPUTFILE); + optionOUTPUTFILE.setType(String.class); + options.addOption(optionOUTPUTFILE); + + // NUMTHREADS + Option optionNUMTHREADS = new Option("nt", QuerierProps.NUMTHREADS, true, "required -- Number of threads to use for encryption/decryption"); + optionNUMTHREADS.setRequired(false); + optionNUMTHREADS.setArgName(QuerierProps.NUMTHREADS); + optionNUMTHREADS.setType(String.class); + options.addOption(optionNUMTHREADS); + + // DATASCHEMAS + Option optionDataSchemas = new Option("ds", QuerierProps.DATASCHEMAS, true, "optional -- Comma separated list of data schema file names"); + optionDataSchemas.setRequired(false); + optionDataSchemas.setArgName(QuerierProps.DATASCHEMAS); + optionDataSchemas.setType(String.class); + options.addOption(optionDataSchemas); + + // QUERYSCHEMAS + Option optionQuerySchemas = new Option("qs", QuerierProps.QUERYSCHEMAS, true, "optional -- Comma separated list of query schema file names"); + optionQuerySchemas.setRequired(false); + optionQuerySchemas.setArgName(QuerierProps.QUERYSCHEMAS); + optionQuerySchemas.setType(String.class); + options.addOption(optionQuerySchemas); + + // TYPE + Option optionTYPE = new Option("qt", QuerierProps.QUERYTYPE, true, "required for encryption -- Type of the query as defined " + + "in the 'schemaName' tag of the corresponding query schema file"); + optionTYPE.setRequired(false); + optionTYPE.setArgName(QuerierProps.QUERYTYPE); + optionTYPE.setType(String.class); + options.addOption(optionTYPE); + + // HASHBITSIZE + Option optionHASHBITSIZE = new Option("hb", QuerierProps.HASHBITSIZE, true, "required -- Bit size of keyed hash"); + optionHASHBITSIZE.setRequired(false); + optionHASHBITSIZE.setArgName(QuerierProps.HASHBITSIZE); + optionHASHBITSIZE.setType(String.class); + options.addOption(optionHASHBITSIZE); + + // HASHKEY + Option optionHASHKEY = new Option("hk", QuerierProps.HASHKEY, true, "required for encryption -- String key for the keyed hash functionality"); + optionHASHKEY.setRequired(false); + optionHASHKEY.setArgName(QuerierProps.HASHKEY); + optionHASHKEY.setType(String.class); + options.addOption(optionHASHKEY); + + // DATAPARTITIONSIZE + Option optionDATAPARTITIONSIZE = new Option("dps", QuerierProps.DATAPARTITIONSIZE, true, + "required for encryption -- Partition bit size in data partitioning"); + optionDATAPARTITIONSIZE.setRequired(false); + optionDATAPARTITIONSIZE.setArgName(QuerierProps.DATAPARTITIONSIZE); + optionDATAPARTITIONSIZE.setType(String.class); + options.addOption(optionDATAPARTITIONSIZE); + + // PAILLIERBITSIZE + Option optionPAILLIERBITSIZE = new Option("pbs", QuerierProps.PAILLIERBITSIZE, true, "required for encryption -- Paillier modulus size N"); + optionPAILLIERBITSIZE.setRequired(false); + optionPAILLIERBITSIZE.setArgName(QuerierProps.PAILLIERBITSIZE); + optionPAILLIERBITSIZE.setType(String.class); + options.addOption(optionPAILLIERBITSIZE); + + // CERTAINTY + Option optionCERTAINTY = new Option("c", QuerierProps.CERTAINTY, true, + "required for encryption -- Certainty of prime generation for Paillier -- must be greater than or " + "equal to " + + SystemConfiguration.getProperty("pir.primeCertainty") + ""); + optionCERTAINTY.setRequired(false); + optionCERTAINTY.setArgName(QuerierProps.CERTAINTY); + optionCERTAINTY.setType(String.class); + options.addOption(optionCERTAINTY); + + // BITSET + Option optionBITSET = new Option("b", QuerierProps.BITSET, true, "required for encryption -- Ensure that this bit position is set in the " + + "Paillier modulus (will generate Paillier moduli until finding one in which this bit is set)"); + optionBITSET.setRequired(false); + optionBITSET.setArgName(QuerierProps.BITSET); + optionBITSET.setType(String.class); + options.addOption(optionBITSET); + + // embedSelector + Option optionEmbedSelector = new Option("embed", QuerierProps.EMBEDSELECTOR, true, + "required for encryption -- 'true' or 'false' - Whether or not to embed " + "the selector in the results to reduce false positives"); + optionEmbedSelector.setRequired(false); + optionEmbedSelector.setArgName(QuerierProps.EMBEDSELECTOR); + optionEmbedSelector.setType(String.class); + options.addOption(optionEmbedSelector); + + // useMemLookupTable + Option optionUseMemLookupTable = new Option("mlu", QuerierProps.USEMEMLOOKUPTABLE, true, + "required for encryption -- 'true' or 'false' - Whether or not to generate and use " + + "an in memory modular exponentation lookup table - only for standalone/testing right now..."); + optionUseMemLookupTable.setRequired(false); + optionUseMemLookupTable.setArgName(QuerierProps.USEMEMLOOKUPTABLE); + optionUseMemLookupTable.setType(String.class); + options.addOption(optionUseMemLookupTable); + + // useHDFSLookupTable + Option optionUseHDFSLookupTable = new Option("lu", QuerierProps.USEHDFSLOOKUPTABLE, true, + "required for encryption -- 'true' or 'false' -- Whether or not to generate and use " + "a hdfs modular exponentation lookup table"); + optionUseHDFSLookupTable.setRequired(false); + optionUseHDFSLookupTable.setArgName(QuerierProps.USEHDFSLOOKUPTABLE); + optionUseHDFSLookupTable.setType(String.class); + options.addOption(optionUseHDFSLookupTable); + + // QUERIERFILE + Option optionQUERIERFILE = new Option("qf", QuerierProps.QUERIERFILE, true, + "required for decryption -- Fully qualified file containing the serialized Querier object"); + optionQUERIERFILE.setRequired(false); + optionQUERIERFILE.setArgName(QuerierProps.QUERIERFILE); + optionQUERIERFILE.setType(String.class); + options.addOption(optionQUERIERFILE); + + // embedQuerySchema + Option optionEMBEDQUERYSCHEMA = new Option("embedQS", QuerierProps.EMBEDQUERYSCHEMA, true, + "optional (defaults to false) -- Whether or not to embed the QuerySchema in the Query (via QueryInfo)"); + optionEMBEDQUERYSCHEMA.setRequired(false); + optionEMBEDQUERYSCHEMA.setArgName(QuerierProps.EMBEDQUERYSCHEMA); + optionEMBEDQUERYSCHEMA.setType(String.class); + options.addOption(optionEMBEDQUERYSCHEMA); + + // SR_ALGORITHM + Option optionSR_ALGORITHM = new Option("srAlg", QuerierProps.SR_ALGORITHM, true, "optional - specify the SecureRandom algorithm, defaults to NativePRNG"); + optionSR_ALGORITHM.setRequired(false); + optionSR_ALGORITHM.setArgName(QuerierProps.SR_ALGORITHM); + optionSR_ALGORITHM.setType(String.class); + options.addOption(optionSR_ALGORITHM); + + // SR_PROVIDERS + Option optionSR_PROVIDER = new Option("srProvider", QuerierProps.SR_PROVIDER, true, "optional - specify the SecureRandom provider, defaults to SUN"); + optionSR_PROVIDER.setRequired(false); + optionSR_PROVIDER.setArgName(QuerierProps.SR_PROVIDER); + optionSR_PROVIDER.setType(String.class); + options.addOption(optionSR_PROVIDER); + + return options; + } + + /** + * Prints out the help message + */ + private void printHelp() + { + HelpFormatter formatter = new HelpFormatter(); + formatter.setWidth(140); + formatter.printHelp("QuerierDriver", cliOptions); + } +} http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/483fac0a/src/main/java/org/apache/pirk/querier/wideskies/QuerierDriver.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/querier/wideskies/QuerierDriver.java b/src/main/java/org/apache/pirk/querier/wideskies/QuerierDriver.java index 1c3b345..d91fa2d 100644 --- a/src/main/java/org/apache/pirk/querier/wideskies/QuerierDriver.java +++ b/src/main/java/org/apache/pirk/querier/wideskies/QuerierDriver.java @@ -102,7 +102,7 @@ public class QuerierDriver implements Serializable String querierFile = null; // Parse the args - QuerierDriverCLI qdriverCLI = new QuerierDriverCLI(args); + QuerierCLI qdriverCLI = new QuerierCLI(args); // Set the variables action = SystemConfiguration.getProperty(QuerierProps.ACTION); http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/483fac0a/src/main/java/org/apache/pirk/querier/wideskies/QuerierDriverCLI.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/querier/wideskies/QuerierDriverCLI.java b/src/main/java/org/apache/pirk/querier/wideskies/QuerierDriverCLI.java deleted file mode 100644 index 2bd581c..0000000 --- a/src/main/java/org/apache/pirk/querier/wideskies/QuerierDriverCLI.java +++ /dev/null @@ -1,327 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pirk.querier.wideskies; - -import java.io.File; - -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.CommandLineParser; -import org.apache.commons.cli.GnuParser; -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.Options; -import org.apache.pirk.utils.SystemConfiguration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Class for parsing the command line options for the QuerierDriver - */ -public class QuerierDriverCLI -{ - private static final Logger logger = LoggerFactory.getLogger(QuerierDriverCLI.class); - - private Options cliOptions = null; - private CommandLine commandLine = null; - - private static final String LOCALPROPFILE = "local.querier.properties"; - - /** - * Create and parse allowable options - * - */ - public QuerierDriverCLI(String[] args) - { - // Create the command line options - cliOptions = createOptions(); - - try - { - // Parse the command line options - CommandLineParser parser = new GnuParser(); - commandLine = parser.parse(cliOptions, args, true); - - // if help option is selected, just print help text and exit - if (hasOption("h")) - { - printHelp(); - System.exit(1); - } - - // Parse and validate the options provided - if (!parseOptions()) - { - logger.info("The provided options are not valid"); - printHelp(); - System.exit(1); - } - } catch (Exception e) - { - e.printStackTrace(); - System.exit(1); - } - } - - /** - * Determine if an option was provided by the user via the CLI - * - * @param option - * - the option of interest - * @return true if option was provided, false otherwise - */ - public boolean hasOption(String option) - { - return commandLine.hasOption(option); - } - - /** - * Obtain the argument of the option provided by the user via the CLI - * - * @param option - * - the option of interest - * @return value of the argument of the option - */ - public String getOptionValue(String option) - { - return commandLine.getOptionValue(option); - } - - /** - * Method to parse and validate the options provided - * - * @return - true if valid, false otherwise - */ - private boolean parseOptions() - { - boolean valid = true; - - // If we have a local.querier.properties file specified, load it - if (hasOption(LOCALPROPFILE)) - { - SystemConfiguration.loadPropsFromFile(new File(getOptionValue(LOCALPROPFILE))); - } - else - { - // Pull options, set as properties - for (String prop : QuerierProps.PROPSLIST) - { - if (hasOption(prop)) - { - SystemConfiguration.setProperty(prop, getOptionValue(prop)); - } - } - } - - // Validate properties - valid = QuerierProps.validateQuerierProperties(); - - return valid; - } - - /** - * Create the options available for the DistributedTestDriver - * - * @return Apache's CLI Options object - */ - private Options createOptions() - { - Options options = new Options(); - - // help - Option optionHelp = new Option("h", "help", false, "Print out the help documentation for this command line execution"); - optionHelp.setRequired(false); - options.addOption(optionHelp); - - // local.querier.properties - Option optionLocalPropFile = new Option("localPropFile", LOCALPROPFILE, true, "Optional local properties file"); - optionLocalPropFile.setRequired(false); - optionLocalPropFile.setArgName(LOCALPROPFILE); - optionLocalPropFile.setType(String.class); - options.addOption(optionLocalPropFile); - - // ACTION - Option optionACTION = new Option("a", QuerierProps.ACTION, true, "required - 'encrypt' or 'decrypt' -- The action performed by the QuerierDriver"); - optionACTION.setRequired(false); - optionACTION.setArgName(QuerierProps.ACTION); - optionACTION.setType(String.class); - options.addOption(optionACTION); - - // INPUTFILE - Option optionINPUTFILE = new Option("i", QuerierProps.INPUTFILE, true, "required - Fully qualified file containing input " - + "-- \n The input is either: \n (1) For Encryption: A query file - Contains the query selectors, one per line; " - + "the first line must be the query number \n OR \n (2) For Decryption: A response file - Contains the serialized Response object"); - optionINPUTFILE.setRequired(false); - optionINPUTFILE.setArgName(QuerierProps.INPUTFILE); - optionINPUTFILE.setType(String.class); - options.addOption(optionINPUTFILE); - - // OUTPUTFILE - Option optionOUTPUTFILE = new Option("o", QuerierProps.OUTPUTFILE, true, "required - Fully qualified file for the result output. " - + "\n The output file specifies either: \n (1) For encryption: \n \t (a) A file to contain the serialized Querier object named: " + "<outputFile>-" - + QuerierConst.QUERIER_FILETAG + " AND \n \t " + "(b) A file to contain the serialized Query object named: <outputFile>-" + QuerierConst.QUERY_FILETAG - + "\n " + "OR \n (2) A file to contain the decryption results where each line is where each line " - + "corresponds to one hit and is a JSON object with the schema QuerySchema"); - optionOUTPUTFILE.setRequired(false); - optionOUTPUTFILE.setArgName(QuerierProps.OUTPUTFILE); - optionOUTPUTFILE.setType(String.class); - options.addOption(optionOUTPUTFILE); - - // NUMTHREADS - Option optionNUMTHREADS = new Option("nt", QuerierProps.NUMTHREADS, true, "required -- Number of threads to use for encryption/decryption"); - optionNUMTHREADS.setRequired(false); - optionNUMTHREADS.setArgName(QuerierProps.NUMTHREADS); - optionNUMTHREADS.setType(String.class); - options.addOption(optionNUMTHREADS); - - // DATASCHEMAS - Option optionDataSchemas = new Option("ds", QuerierProps.DATASCHEMAS, true, "optional -- Comma separated list of data schema file names"); - optionDataSchemas.setRequired(false); - optionDataSchemas.setArgName(QuerierProps.DATASCHEMAS); - optionDataSchemas.setType(String.class); - options.addOption(optionDataSchemas); - - // QUERYSCHEMAS - Option optionQuerySchemas = new Option("qs", QuerierProps.QUERYSCHEMAS, true, "optional -- Comma separated list of query schema file names"); - optionQuerySchemas.setRequired(false); - optionQuerySchemas.setArgName(QuerierProps.QUERYSCHEMAS); - optionQuerySchemas.setType(String.class); - options.addOption(optionQuerySchemas); - - // TYPE - Option optionTYPE = new Option("qt", QuerierProps.QUERYTYPE, true, "required for encryption -- Type of the query as defined " - + "in the 'schemaName' tag of the corresponding query schema file"); - optionTYPE.setRequired(false); - optionTYPE.setArgName(QuerierProps.QUERYTYPE); - optionTYPE.setType(String.class); - options.addOption(optionTYPE); - - // HASHBITSIZE - Option optionHASHBITSIZE = new Option("hb", QuerierProps.HASHBITSIZE, true, "required -- Bit size of keyed hash"); - optionHASHBITSIZE.setRequired(false); - optionHASHBITSIZE.setArgName(QuerierProps.HASHBITSIZE); - optionHASHBITSIZE.setType(String.class); - options.addOption(optionHASHBITSIZE); - - // HASHKEY - Option optionHASHKEY = new Option("hk", QuerierProps.HASHKEY, true, "required for encryption -- String key for the keyed hash functionality"); - optionHASHKEY.setRequired(false); - optionHASHKEY.setArgName(QuerierProps.HASHKEY); - optionHASHKEY.setType(String.class); - options.addOption(optionHASHKEY); - - // DATAPARTITIONSIZE - Option optionDATAPARTITIONSIZE = new Option("dps", QuerierProps.DATAPARTITIONSIZE, true, - "required for encryption -- Partition bit size in data partitioning"); - optionDATAPARTITIONSIZE.setRequired(false); - optionDATAPARTITIONSIZE.setArgName(QuerierProps.DATAPARTITIONSIZE); - optionDATAPARTITIONSIZE.setType(String.class); - options.addOption(optionDATAPARTITIONSIZE); - - // PAILLIERBITSIZE - Option optionPAILLIERBITSIZE = new Option("pbs", QuerierProps.PAILLIERBITSIZE, true, "required for encryption -- Paillier modulus size N"); - optionPAILLIERBITSIZE.setRequired(false); - optionPAILLIERBITSIZE.setArgName(QuerierProps.PAILLIERBITSIZE); - optionPAILLIERBITSIZE.setType(String.class); - options.addOption(optionPAILLIERBITSIZE); - - // CERTAINTY - Option optionCERTAINTY = new Option("c", QuerierProps.CERTAINTY, true, - "required for encryption -- Certainty of prime generation for Paillier -- must be greater than or " + "equal to " - + SystemConfiguration.getProperty("pir.primeCertainty") + ""); - optionCERTAINTY.setRequired(false); - optionCERTAINTY.setArgName(QuerierProps.CERTAINTY); - optionCERTAINTY.setType(String.class); - options.addOption(optionCERTAINTY); - - // BITSET - Option optionBITSET = new Option("b", QuerierProps.BITSET, true, "required for encryption -- Ensure that this bit position is set in the " - + "Paillier modulus (will generate Paillier moduli until finding one in which this bit is set)"); - optionBITSET.setRequired(false); - optionBITSET.setArgName(QuerierProps.BITSET); - optionBITSET.setType(String.class); - options.addOption(optionBITSET); - - // embedSelector - Option optionEmbedSelector = new Option("embed", QuerierProps.EMBEDSELECTOR, true, - "required for encryption -- 'true' or 'false' - Whether or not to embed " + "the selector in the results to reduce false positives"); - optionEmbedSelector.setRequired(false); - optionEmbedSelector.setArgName(QuerierProps.EMBEDSELECTOR); - optionEmbedSelector.setType(String.class); - options.addOption(optionEmbedSelector); - - // useMemLookupTable - Option optionUseMemLookupTable = new Option("mlu", QuerierProps.USEMEMLOOKUPTABLE, true, - "required for encryption -- 'true' or 'false' - Whether or not to generate and use " - + "an in memory modular exponentation lookup table - only for standalone/testing right now..."); - optionUseMemLookupTable.setRequired(false); - optionUseMemLookupTable.setArgName(QuerierProps.USEMEMLOOKUPTABLE); - optionUseMemLookupTable.setType(String.class); - options.addOption(optionUseMemLookupTable); - - // useHDFSLookupTable - Option optionUseHDFSLookupTable = new Option("lu", QuerierProps.USEHDFSLOOKUPTABLE, true, - "required for encryption -- 'true' or 'false' -- Whether or not to generate and use " + "a hdfs modular exponentation lookup table"); - optionUseHDFSLookupTable.setRequired(false); - optionUseHDFSLookupTable.setArgName(QuerierProps.USEHDFSLOOKUPTABLE); - optionUseHDFSLookupTable.setType(String.class); - options.addOption(optionUseHDFSLookupTable); - - // QUERIERFILE - Option optionQUERIERFILE = new Option("qf", QuerierProps.QUERIERFILE, true, - "required for decryption -- Fully qualified file containing the serialized Querier object"); - optionQUERIERFILE.setRequired(false); - optionQUERIERFILE.setArgName(QuerierProps.QUERIERFILE); - optionQUERIERFILE.setType(String.class); - options.addOption(optionQUERIERFILE); - - // embedQuerySchema - Option optionEMBEDQUERYSCHEMA = new Option("embedQS", QuerierProps.EMBEDQUERYSCHEMA, true, - "optional (defaults to false) -- Whether or not to embed the QuerySchema in the Query (via QueryInfo)"); - optionEMBEDQUERYSCHEMA.setRequired(false); - optionEMBEDQUERYSCHEMA.setArgName(QuerierProps.EMBEDQUERYSCHEMA); - optionEMBEDQUERYSCHEMA.setType(String.class); - options.addOption(optionEMBEDQUERYSCHEMA); - - // SR_ALGORITHM - Option optionSR_ALGORITHM = new Option("srAlg", QuerierProps.SR_ALGORITHM, true, "optional - specify the SecureRandom algorithm, defaults to NativePRNG"); - optionSR_ALGORITHM.setRequired(false); - optionSR_ALGORITHM.setArgName(QuerierProps.SR_ALGORITHM); - optionSR_ALGORITHM.setType(String.class); - options.addOption(optionSR_ALGORITHM); - - // SR_PROVIDERS - Option optionSR_PROVIDER = new Option("srProvider", QuerierProps.SR_PROVIDER, true, "optional - specify the SecureRandom provider, defaults to SUN"); - optionSR_PROVIDER.setRequired(false); - optionSR_PROVIDER.setArgName(QuerierProps.SR_PROVIDER); - optionSR_PROVIDER.setType(String.class); - options.addOption(optionSR_PROVIDER); - - return options; - } - - /** - * Prints out the help message - */ - private void printHelp() - { - HelpFormatter formatter = new HelpFormatter(); - formatter.setWidth(140); - formatter.printHelp("QuerierDriver", cliOptions); - } -} http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/483fac0a/src/main/java/org/apache/pirk/query/wideskies/QueryUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/query/wideskies/QueryUtils.java b/src/main/java/org/apache/pirk/query/wideskies/QueryUtils.java index 929056e..abde492 100644 --- a/src/main/java/org/apache/pirk/query/wideskies/QueryUtils.java +++ b/src/main/java/org/apache/pirk/query/wideskies/QueryUtils.java @@ -18,6 +18,12 @@ */ package org.apache.pirk.query.wideskies; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + import org.apache.hadoop.io.ArrayWritable; import org.apache.hadoop.io.MapWritable; import org.apache.hadoop.io.Text; @@ -36,12 +42,6 @@ import org.json.simple.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.math.BigInteger; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; - /** * Class for helper methods to perform the encrypted query */ http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/483fac0a/src/main/java/org/apache/pirk/responder/wideskies/common/ComputeEncryptedRow.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/responder/wideskies/common/ComputeEncryptedRow.java b/src/main/java/org/apache/pirk/responder/wideskies/common/ComputeEncryptedRow.java index f65f1cb..6ef1bdc 100644 --- a/src/main/java/org/apache/pirk/responder/wideskies/common/ComputeEncryptedRow.java +++ b/src/main/java/org/apache/pirk/responder/wideskies/common/ComputeEncryptedRow.java @@ -18,9 +18,15 @@ */ package org.apache.pirk.responder.wideskies.common; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.concurrent.ExecutionException; + import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.pirk.encryption.ModPowAbstraction; @@ -28,17 +34,13 @@ import org.apache.pirk.inputformat.hadoop.BytesArrayWritable; import org.apache.pirk.query.wideskies.Query; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import scala.Tuple2; import scala.Tuple3; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.math.BigInteger; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.concurrent.ExecutionException; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; /** * Class to compute the encrypted row elements for a query from extracted data partitions http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/483fac0a/src/main/java/org/apache/pirk/responder/wideskies/mapreduce/HashSelectorsAndPartitionDataMapper.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/responder/wideskies/mapreduce/HashSelectorsAndPartitionDataMapper.java b/src/main/java/org/apache/pirk/responder/wideskies/mapreduce/HashSelectorsAndPartitionDataMapper.java index 1399fd7..c05235f 100644 --- a/src/main/java/org/apache/pirk/responder/wideskies/mapreduce/HashSelectorsAndPartitionDataMapper.java +++ b/src/main/java/org/apache/pirk/responder/wideskies/mapreduce/HashSelectorsAndPartitionDataMapper.java @@ -41,6 +41,7 @@ import org.apache.pirk.utils.StringUtils; import org.apache.pirk.utils.SystemConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import scala.Tuple2; /** http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/483fac0a/src/main/java/org/apache/pirk/responder/wideskies/mapreduce/RowCalcReducer.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/responder/wideskies/mapreduce/RowCalcReducer.java b/src/main/java/org/apache/pirk/responder/wideskies/mapreduce/RowCalcReducer.java index 61723d0..75b0529 100644 --- a/src/main/java/org/apache/pirk/responder/wideskies/mapreduce/RowCalcReducer.java +++ b/src/main/java/org/apache/pirk/responder/wideskies/mapreduce/RowCalcReducer.java @@ -18,6 +18,10 @@ */ package org.apache.pirk.responder.wideskies.mapreduce; +import java.io.IOException; +import java.math.BigInteger; +import java.util.List; + import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.LongWritable; @@ -35,11 +39,8 @@ import org.apache.pirk.utils.FileConst; import org.apache.pirk.utils.SystemConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import scala.Tuple2; -import java.io.IOException; -import java.math.BigInteger; -import java.util.List; +import scala.Tuple2; /** * Reducer class to calculate the encrypted rows of the encrypted query http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/483fac0a/src/main/java/org/apache/pirk/responder/wideskies/spark/EncRowCalc.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/responder/wideskies/spark/EncRowCalc.java b/src/main/java/org/apache/pirk/responder/wideskies/spark/EncRowCalc.java index f298ffe..dc25439 100644 --- a/src/main/java/org/apache/pirk/responder/wideskies/spark/EncRowCalc.java +++ b/src/main/java/org/apache/pirk/responder/wideskies/spark/EncRowCalc.java @@ -31,6 +31,7 @@ import org.apache.pirk.responder.wideskies.common.ComputeEncryptedRow; import org.apache.spark.api.java.function.PairFlatMapFunction; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import scala.Tuple2; /** http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/483fac0a/src/main/java/org/apache/pirk/responder/wideskies/spark/ExpTableGenerator.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/responder/wideskies/spark/ExpTableGenerator.java b/src/main/java/org/apache/pirk/responder/wideskies/spark/ExpTableGenerator.java index 8d19c3f..4c54935 100644 --- a/src/main/java/org/apache/pirk/responder/wideskies/spark/ExpTableGenerator.java +++ b/src/main/java/org/apache/pirk/responder/wideskies/spark/ExpTableGenerator.java @@ -25,6 +25,7 @@ import java.util.List; import org.apache.pirk.encryption.ModPowAbstraction; import org.apache.pirk.query.wideskies.Query; import org.apache.spark.api.java.function.PairFlatMapFunction; + import scala.Tuple2; /** http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/483fac0a/src/main/java/org/apache/pirk/responder/wideskies/spark/HashSelectorsAndPartitionData.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/responder/wideskies/spark/HashSelectorsAndPartitionData.java b/src/main/java/org/apache/pirk/responder/wideskies/spark/HashSelectorsAndPartitionData.java index a696778..8fb96c3 100644 --- a/src/main/java/org/apache/pirk/responder/wideskies/spark/HashSelectorsAndPartitionData.java +++ b/src/main/java/org/apache/pirk/responder/wideskies/spark/HashSelectorsAndPartitionData.java @@ -29,6 +29,7 @@ import org.apache.pirk.schema.query.QuerySchema; import org.apache.spark.api.java.function.PairFunction; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import scala.Tuple2; /** http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/483fac0a/src/main/java/org/apache/pirk/responder/wideskies/standalone/Responder.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/responder/wideskies/standalone/Responder.java b/src/main/java/org/apache/pirk/responder/wideskies/standalone/Responder.java index 03e5a76..e4bc03f 100644 --- a/src/main/java/org/apache/pirk/responder/wideskies/standalone/Responder.java +++ b/src/main/java/org/apache/pirk/responder/wideskies/standalone/Responder.java @@ -18,6 +18,14 @@ */ package org.apache.pirk.responder.wideskies.standalone; +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.IOException; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.List; +import java.util.TreeMap; + import org.apache.pirk.encryption.ModPowAbstraction; import org.apache.pirk.query.wideskies.Query; import org.apache.pirk.query.wideskies.QueryInfo; @@ -33,14 +41,6 @@ import org.json.simple.parser.JSONParser; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.BufferedReader; -import java.io.FileReader; -import java.io.IOException; -import java.math.BigInteger; -import java.util.ArrayList; -import java.util.List; -import java.util.TreeMap; - /** * Class to perform stand alone responder functionalities * <p> http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/483fac0a/src/main/java/org/apache/pirk/schema/data/DataSchemaLoader.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/schema/data/DataSchemaLoader.java b/src/main/java/org/apache/pirk/schema/data/DataSchemaLoader.java index 2f210dc..28de96e 100644 --- a/src/main/java/org/apache/pirk/schema/data/DataSchemaLoader.java +++ b/src/main/java/org/apache/pirk/schema/data/DataSchemaLoader.java @@ -25,6 +25,7 @@ import java.io.InputStream; import java.util.Arrays; import java.util.HashSet; import java.util.Set; + import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/483fac0a/src/main/java/org/apache/pirk/test/distributed/DistributedTestDriver.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/test/distributed/DistributedTestDriver.java b/src/main/java/org/apache/pirk/test/distributed/DistributedTestDriver.java index ee37e63..a8cec45 100755 --- a/src/main/java/org/apache/pirk/test/distributed/DistributedTestDriver.java +++ b/src/main/java/org/apache/pirk/test/distributed/DistributedTestDriver.java @@ -18,7 +18,6 @@ */ package org.apache.pirk.test.distributed; -import java.util.ArrayList; import java.util.List; import org.apache.hadoop.conf.Configuration; @@ -42,11 +41,9 @@ public class DistributedTestDriver private static final Logger logger = LoggerFactory.getLogger(DistributedTestDriver.class); // Input - public static final String JSON_PIR_INPUT_FILE_PROPERTY = "test.pir.inputJSONFile"; - public static final String ES_PIR_INPUT_INDEX_PROPERTY = "test.pir.es.index"; - public static final String PIR_QUERY_INPUT_DIR = "test.pir.queryInputDir"; - public static final String PIR_STOPLIST_FILE = "test.pir.stopListFile"; - public static final String ES_PIR_INPUT_RESOURCE_PROPERTY = "test.pir.es.resource"; + public static final String JSON_PIR_INPUT_FILE_PROPERTY = "test.inputJSONFile"; + public static final String PIR_QUERY_INPUT_DIR = "test.queryInputDir"; + public static final String PIR_STOPLIST_FILE = "test.stopListFile"; // Elastic Search public static final String ES_INPUT_NODES_PROPERTY = "es.nodes"; @@ -88,9 +85,9 @@ public class DistributedTestDriver */ public static List<JSONObject> initialize(FileSystem fs) throws Exception { - List<JSONObject> dataElements = Inputs.createPIRJSONInput(fs); + List<JSONObject> dataElements = Inputs.createJSONInput(fs); - String localStopListFile = Inputs.createPIRStopList(fs, true); + String localStopListFile = Inputs.createStopList(fs, true); SystemConfiguration.setProperty("pir.stopListFile", localStopListFile); Inputs.createSchemaFiles(fs, true, StopListFilter.class.getName()); @@ -109,7 +106,7 @@ public class DistributedTestDriver } if (cli.run("1:E") || cli.run("1:ES")) { - Inputs.createPIRESInput(); + Inputs.createESInput(); if (cli.run("1:E")) { DistTestSuite.testESInputMR(fs, pirDataElements); http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/483fac0a/src/main/java/org/apache/pirk/test/distributed/testsuite/DistTestSuite.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/test/distributed/testsuite/DistTestSuite.java b/src/main/java/org/apache/pirk/test/distributed/testsuite/DistTestSuite.java index 58f835c..12a941e 100644 --- a/src/main/java/org/apache/pirk/test/distributed/testsuite/DistTestSuite.java +++ b/src/main/java/org/apache/pirk/test/distributed/testsuite/DistTestSuite.java @@ -167,7 +167,7 @@ public class DistTestSuite // Set up ES configs SystemConfiguration.setProperty("pir.dataInputFormat", InputFormatConst.ES); SystemConfiguration.setProperty("pir.esQuery", "?q=rcode:0"); - SystemConfiguration.setProperty("pir.esResource", SystemConfiguration.getProperty(DistributedTestDriver.ES_PIR_INPUT_RESOURCE_PROPERTY)); + SystemConfiguration.setProperty("pir.esResource", SystemConfiguration.getProperty(DistributedTestDriver.ES_INPUT_RESOURCE_PROPERTY)); // Run tests SystemConfiguration.setProperty("pirTest.embedSelector", "true"); @@ -298,7 +298,7 @@ public class DistTestSuite // Set up ES configs SystemConfiguration.setProperty("pir.dataInputFormat", InputFormatConst.ES); SystemConfiguration.setProperty("pir.esQuery", "?q=rcode:0"); - SystemConfiguration.setProperty("pir.esResource", SystemConfiguration.getProperty(DistributedTestDriver.ES_PIR_INPUT_RESOURCE_PROPERTY)); + SystemConfiguration.setProperty("pir.esResource", SystemConfiguration.getProperty(DistributedTestDriver.ES_INPUT_RESOURCE_PROPERTY)); // Run tests SystemConfiguration.setProperty("pirTest.embedSelector", "true"); http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/483fac0a/src/main/java/org/apache/pirk/test/utils/BaseTests.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/test/utils/BaseTests.java b/src/main/java/org/apache/pirk/test/utils/BaseTests.java index a55ed4d..c1fa1e9 100644 --- a/src/main/java/org/apache/pirk/test/utils/BaseTests.java +++ b/src/main/java/org/apache/pirk/test/utils/BaseTests.java @@ -18,6 +18,15 @@ */ package org.apache.pirk.test.utils; +import static org.junit.Assert.fail; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.UUID; + import org.apache.hadoop.fs.FileSystem; import org.apache.pirk.query.wideskies.QueryUtils; import org.apache.pirk.schema.query.QuerySchema; @@ -31,15 +40,6 @@ import org.json.simple.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.UUID; - -import static org.junit.Assert.fail; - /** * Class to hold the base functional distributed tests */ http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/483fac0a/src/main/java/org/apache/pirk/test/utils/Inputs.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/test/utils/Inputs.java b/src/main/java/org/apache/pirk/test/utils/Inputs.java index 10c1386..587ac99 100644 --- a/src/main/java/org/apache/pirk/test/utils/Inputs.java +++ b/src/main/java/org/apache/pirk/test/utils/Inputs.java @@ -108,7 +108,7 @@ public class Inputs { String esPIRIndex = SystemConfiguration.getProperty(DistributedTestDriver.ES_INPUT_NODES_PROPERTY) + ":" + SystemConfiguration.getProperty(DistributedTestDriver.ES_INPUT_PORT_PROPERTY) + "/" - + SystemConfiguration.getProperty(DistributedTestDriver.ES_PIR_INPUT_INDEX_PROPERTY); + + SystemConfiguration.getProperty(DistributedTestDriver.ES_INPUT_INDEX_PROPERTY); logger.info("ES input being deleted at " + esPIRIndex); ProcessBuilder pDeletePIR = new ProcessBuilder("curl", "-XDELETE", esPIRIndex); @@ -258,9 +258,9 @@ public class Inputs } /** - * Creates PIR JSON input and writes to hdfs + * Creates JSON input and writes to hdfs */ - public static List<JSONObject> createPIRJSONInput(FileSystem fs) + public static List<JSONObject> createJSONInput(FileSystem fs) { String inputJSONFile = SystemConfiguration.getProperty(DistributedTestDriver.JSON_PIR_INPUT_FILE_PROPERTY); logger.info("PIR JSON input being created at " + inputJSONFile); @@ -274,13 +274,13 @@ public class Inputs } /** - * Creates PIR Elasticsearch input + * Creates Elasticsearch input */ - public static void createPIRESInput() + public static void createESInput() { String esTestIndex = SystemConfiguration.getProperty(DistributedTestDriver.ES_INPUT_NODES_PROPERTY) + ":" + SystemConfiguration.getProperty(DistributedTestDriver.ES_INPUT_PORT_PROPERTY) + "/" - + SystemConfiguration.getProperty(DistributedTestDriver.ES_PIR_INPUT_INDEX_PROPERTY); + + SystemConfiguration.getProperty(DistributedTestDriver.ES_INPUT_INDEX_PROPERTY); String esType = SystemConfiguration.getProperty(DistributedTestDriver.ES_INPUT_TYPE_PROPERTY); logger.info("ES input being created at " + esTestIndex + " with type " + esType); @@ -388,11 +388,11 @@ public class Inputs } /** - * Creates PIR stoplist file + * Creates stoplist file */ - public static String createPIRStopList(FileSystem fs, boolean hdfs) throws IOException, PIRException + public static String createStopList(FileSystem fs, boolean hdfs) throws IOException, PIRException { - logger.info("PIR stopList file being created"); + logger.info("StopList file being created"); List<String> elements = Arrays.asList("something.else.on.stoplist", "3.3.3.132"); http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/483fac0a/src/main/java/org/apache/pirk/test/utils/StandaloneQuery.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/test/utils/StandaloneQuery.java b/src/main/java/org/apache/pirk/test/utils/StandaloneQuery.java index a931871..afac2b0 100644 --- a/src/main/java/org/apache/pirk/test/utils/StandaloneQuery.java +++ b/src/main/java/org/apache/pirk/test/utils/StandaloneQuery.java @@ -18,6 +18,13 @@ */ package org.apache.pirk.test.utils; +import static org.junit.Assert.fail; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.List; + import org.apache.pirk.encryption.Paillier; import org.apache.pirk.querier.wideskies.Querier; import org.apache.pirk.querier.wideskies.QuerierConst; @@ -38,13 +45,6 @@ import org.json.simple.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.IOException; -import java.util.HashMap; -import java.util.List; - -import static org.junit.Assert.fail; - public class StandaloneQuery { private static final Logger logger = LoggerFactory.getLogger(StandaloneQuery.class); http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/483fac0a/src/main/java/org/apache/pirk/utils/KeyedHash.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/utils/KeyedHash.java b/src/main/java/org/apache/pirk/utils/KeyedHash.java index 3dfa15d..665cb3f 100644 --- a/src/main/java/org/apache/pirk/utils/KeyedHash.java +++ b/src/main/java/org/apache/pirk/utils/KeyedHash.java @@ -18,12 +18,12 @@ */ package org.apache.pirk.utils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * Class for the PIR keyed hash * <p> http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/483fac0a/src/main/resources/pirk.properties ---------------------------------------------------------------------- diff --git a/src/main/resources/pirk.properties b/src/main/resources/pirk.properties index 94379d5..f8efea7 100755 --- a/src/main/resources/pirk.properties +++ b/src/main/resources/pirk.properties @@ -100,7 +100,6 @@ es.field.read.empty.as.null=yes #Test index to create in ES (created and deleted within the tests) #Need final form to be: <host>:<port>/<test.esTestIndex> test.es.index = testindex -test.pir.es.index = testindexpir #Type of elements to insert in ES test.es.type = pkt @@ -108,20 +107,18 @@ test.es.type = pkt #Elasticsearch resource - Elasticsearch resource location where data is read and written to. #Requires the format <index>/<type> test.es.resource= none -test.pir.es.resource = none #Pathname in hdfs to place input JSON file testing -test.inputJSONFile = none -test.pir.inputJSONFile = /tmp/testJSONInput +test.inputJSONFile = /tmp/testJSONInput #Pathname in hdfs to place output file for testing test.outputHDFSFile = /tmp/testOutput -#PIR query input dir in hdfs for testing -test.pir.queryInputDir = none +#Query input dir in hdfs for testing +test.queryInputDir = none -#PIR stoplist file -test.pir.stopListFile = /tmp/testStopListFile +#Test stoplist file +test.stopListFile = /tmp/testStopListFile #Whether or not we are running PIR testing (used as a flag to dump intermediate RDDs for checking) #This should default to false; it is changed to true in the test suite, as applicable http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/483fac0a/src/main/resources/querier.properties ---------------------------------------------------------------------- diff --git a/src/main/resources/querier.properties b/src/main/resources/querier.properties index 88352d7..15e3acd 100644 --- a/src/main/resources/querier.properties +++ b/src/main/resources/querier.properties @@ -54,7 +54,7 @@ querier.numThreads= #querier.dataSchemas= #querySchemas -- optional -- Comma separated list of query schema file names to load -#querier.dataSchemas= +#querier.querySchemas= ## Optional, but required for Encryption (ignored if not encrypting) http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/483fac0a/src/test/java/org/apache/pirk/general/PartitionUtilsTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/pirk/general/PartitionUtilsTest.java b/src/test/java/org/apache/pirk/general/PartitionUtilsTest.java index 21d44b6..c78bf33 100644 --- a/src/test/java/org/apache/pirk/general/PartitionUtilsTest.java +++ b/src/test/java/org/apache/pirk/general/PartitionUtilsTest.java @@ -18,6 +18,13 @@ */ package org.apache.pirk.general; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.math.BigInteger; +import java.util.Arrays; +import java.util.List; + import org.apache.pirk.schema.data.partitioner.IPDataPartitioner; import org.apache.pirk.schema.data.partitioner.ISO8601DatePartitioner; import org.apache.pirk.schema.data.partitioner.PrimitiveTypePartitioner; @@ -27,13 +34,6 @@ import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.math.BigInteger; -import java.util.Arrays; -import java.util.List; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - /** * Class to functionally test the bit conversion utils */ http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/483fac0a/src/test/java/org/apache/pirk/schema/query/LoadQuerySchemaTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/pirk/schema/query/LoadQuerySchemaTest.java b/src/test/java/org/apache/pirk/schema/query/LoadQuerySchemaTest.java index 55cb0a9..a801b01 100644 --- a/src/test/java/org/apache/pirk/schema/query/LoadQuerySchemaTest.java +++ b/src/test/java/org/apache/pirk/schema/query/LoadQuerySchemaTest.java @@ -306,7 +306,7 @@ public class LoadQuerySchemaTest private void createStopListFile() throws IOException, PIRException { SystemConfiguration.setProperty("pir.stopListFile", "testStopListFile"); - String newSLFile = Inputs.createPIRStopList(null, false); + String newSLFile = Inputs.createStopList(null, false); SystemConfiguration.setProperty("pir.stopListFile", newSLFile); } http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/483fac0a/src/test/java/org/apache/pirk/wideskies/standalone/StandaloneTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/pirk/wideskies/standalone/StandaloneTest.java b/src/test/java/org/apache/pirk/wideskies/standalone/StandaloneTest.java index 06c2bff..394609d 100644 --- a/src/test/java/org/apache/pirk/wideskies/standalone/StandaloneTest.java +++ b/src/test/java/org/apache/pirk/wideskies/standalone/StandaloneTest.java @@ -63,7 +63,7 @@ public class StandaloneTest // Create the stoplist file stopListFileProp = SystemConfiguration.getProperty("pir.stopListFile"); SystemConfiguration.setProperty("pir.stopListFile", STOPLIST_FILE); - String newSLFile = Inputs.createPIRStopList(null, false); + String newSLFile = Inputs.createStopList(null, false); SystemConfiguration.setProperty("pir.stopListFile", newSLFile); logger.info("stopListFileProp = " + stopListFileProp + " new prop = " + SystemConfiguration.getProperty("pir.stopListFile"));
