http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/9244df72/src/test/java/org/apache/pirk/test/distributed/testsuite/DistTestSuite.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/pirk/test/distributed/testsuite/DistTestSuite.java b/src/test/java/org/apache/pirk/test/distributed/testsuite/DistTestSuite.java deleted file mode 100644 index 58f835c..0000000 --- a/src/test/java/org/apache/pirk/test/distributed/testsuite/DistTestSuite.java +++ /dev/null @@ -1,458 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pirk.test.distributed.testsuite; - -import java.io.File; -import java.util.ArrayList; -import java.util.List; - -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.util.ToolRunner; -import org.apache.pirk.encryption.Paillier; -import org.apache.pirk.inputformat.hadoop.InputFormatConst; -import org.apache.pirk.inputformat.hadoop.json.JSONInputFormatBase; -import org.apache.pirk.querier.wideskies.Querier; -import org.apache.pirk.querier.wideskies.decrypt.DecryptResponse; -import org.apache.pirk.querier.wideskies.encrypt.EncryptQuery; -import org.apache.pirk.query.wideskies.Query; -import org.apache.pirk.query.wideskies.QueryInfo; -import org.apache.pirk.responder.wideskies.ResponderProps; -import org.apache.pirk.responder.wideskies.mapreduce.ComputeResponseTool; -import org.apache.pirk.response.wideskies.Response; -import org.apache.pirk.schema.response.QueryResponseJSON; -import org.apache.pirk.serialization.HadoopFileSystemStore; -import org.apache.pirk.test.distributed.DistributedTestDriver; -import org.apache.pirk.test.utils.BaseTests; -import org.apache.pirk.test.utils.Inputs; -import org.apache.pirk.test.utils.TestUtils; -import org.apache.pirk.utils.SystemConfiguration; -import org.apache.spark.launcher.SparkLauncher; -import org.json.simple.JSONObject; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Distributed test class for PIR - * - */ -public class DistTestSuite -{ - private static final Logger logger = LoggerFactory.getLogger(DistTestSuite.class); - - // This method also tests all non-query specific configuration options/properties - // for the MapReduce version of PIR - public static void testJSONInputMR(FileSystem fs, List<JSONObject> dataElements) throws Exception - { - logger.info("Starting testJSONInputMR"); - - // Pull original data and query schema properties - SystemConfiguration.setProperty("pirTest.useHDFSExpLookupTable", "false"); - SystemConfiguration.setProperty("pirTest.useExpLookupTable", "false"); - - SystemConfiguration.setProperty("pir.limitHitsPerSelector", "false"); - SystemConfiguration.setProperty("pir.maxHitsPerSelector", "100"); - - SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "false"); - SystemConfiguration.setProperty("pir.embedQuerySchema", "false"); - - // Set up base configs - SystemConfiguration.setProperty("pir.dataInputFormat", InputFormatConst.BASE_FORMAT); - SystemConfiguration.setProperty("pir.inputData", SystemConfiguration.getProperty(DistributedTestDriver.JSON_PIR_INPUT_FILE_PROPERTY)); - SystemConfiguration.setProperty("pir.baseQuery", "?q=rcode:0"); - - // Run tests - SystemConfiguration.setProperty("pirTest.embedSelector", "true"); - BaseTests.testDNSHostnameQuery(dataElements, fs, false, true, 1); - BaseTests.testDNSIPQuery(dataElements, fs, false, true, 1); - - SystemConfiguration.setProperty("pirTest.embedSelector", "false"); - BaseTests.testDNSHostnameQuery(dataElements, fs, false, true, 2); - BaseTests.testDNSIPQuery(dataElements, fs, false, true, 2); - - BaseTests.testSRCIPQueryNoFilter(dataElements, fs, false, true, 2); - - // Test hit limits per selector - SystemConfiguration.setProperty("pir.limitHitsPerSelector", "true"); - SystemConfiguration.setProperty("pir.maxHitsPerSelector", "1"); - BaseTests.testDNSHostnameQuery(dataElements, fs, false, true, 3); - SystemConfiguration.setProperty("pir.limitHitsPerSelector", "false"); - SystemConfiguration.setProperty("pir.maxHitsPerSelector", "1000"); - - // Test the local cache for modular exponentiation - SystemConfiguration.setProperty("pir.useLocalCache", "true"); - BaseTests.testDNSIPQuery(dataElements, fs, false, true, 2); - BaseTests.testSRCIPQuery(dataElements, fs, false, true, 2); - SystemConfiguration.setProperty("pir.useLocalCache", "false"); - - // Change query for NXDOMAIN - SystemConfiguration.setProperty("pir.baseQuery", "?q=rcode:3"); - BaseTests.testDNSNXDOMAINQuery(dataElements, fs, false, true, 2); - SystemConfiguration.setProperty("pirTest.embedSelector", "false"); - BaseTests.testDNSNXDOMAINQuery(dataElements, fs, false, true, 2); - SystemConfiguration.setProperty("pir.baseQuery", "?q=rcode:0"); - - // Test the expTable cases - SystemConfiguration.setProperty("pirTest.embedSelector", "true"); - - // In memory table - SystemConfiguration.setProperty("pirTest.useHDFSExpLookupTable", "false"); - SystemConfiguration.setProperty("pirTest.useExpLookupTable", "true"); - BaseTests.testDNSIPQuery(dataElements, fs, false, true, 2); - - // Create exp table in hdfs - SystemConfiguration.setProperty("mapreduce.map.memory.mb", "10000"); - SystemConfiguration.setProperty("mapreduce.reduce.memory.mb", "10000"); - SystemConfiguration.setProperty("mapreduce.map.java.opts", "-Xmx9000m"); - SystemConfiguration.setProperty("mapreduce.reduce.java.opts", "-Xmx9000m"); - - SystemConfiguration.setProperty("pirTest.useHDFSExpLookupTable", "true"); - SystemConfiguration.setProperty("pirTest.useExpLookupTable", "false"); - SystemConfiguration.setProperty("pir.expCreationSplits", "50"); - SystemConfiguration.setProperty("pir.numExpLookupPartitions", "150"); - BaseTests.testDNSIPQuery(dataElements, fs, false, true, 2); - - // Reset exp properties - SystemConfiguration.setProperty("pirTest.useHDFSExpLookupTable", "false"); - SystemConfiguration.setProperty("pirTest.useExpLookupTable", "false"); - - // Reset property - SystemConfiguration.setProperty("pirTest.embedSelector", "true"); - - // Test embedded QuerySchema - SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "true"); - SystemConfiguration.setProperty("pir.embedQuerySchema", "false"); - BaseTests.testDNSHostnameQuery(dataElements, fs, false, true, 1); - - SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "true"); - SystemConfiguration.setProperty("pir.embedQuerySchema", "true"); - BaseTests.testDNSHostnameQuery(dataElements, fs, false, true, 1); - - SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "false"); - SystemConfiguration.setProperty("pir.embedQuerySchema", "true"); - BaseTests.testDNSHostnameQuery(dataElements, fs, false, true, 1); - - logger.info("Completed testJSONInputMR"); - } - - public static void testESInputMR(FileSystem fs, List<JSONObject> dataElements) throws Exception - { - logger.info("Starting testESInputMR"); - - SystemConfiguration.setProperty("pirTest.useHDFSExpLookupTable", "false"); - SystemConfiguration.setProperty("pirTest.useExpLookupTable", "false"); - - SystemConfiguration.setProperty("pir.limitHitsPerSelector", "false"); - SystemConfiguration.setProperty("pir.maxHitsPerSelector", "1000"); - - SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "false"); - SystemConfiguration.setProperty("pir.embedQuerySchema", "false"); - - // Set up ES configs - SystemConfiguration.setProperty("pir.dataInputFormat", InputFormatConst.ES); - SystemConfiguration.setProperty("pir.esQuery", "?q=rcode:0"); - SystemConfiguration.setProperty("pir.esResource", SystemConfiguration.getProperty(DistributedTestDriver.ES_PIR_INPUT_RESOURCE_PROPERTY)); - - // Run tests - SystemConfiguration.setProperty("pirTest.embedSelector", "true"); - BaseTests.testDNSHostnameQuery(dataElements, fs, false, true, 1); - BaseTests.testSRCIPQuery(dataElements, fs, false, true, 2); - BaseTests.testDNSIPQuery(dataElements, fs, false, true, 1); - - SystemConfiguration.setProperty("pirTest.embedSelector", "false"); - BaseTests.testDNSHostnameQuery(dataElements, fs, false, true, 2); - BaseTests.testDNSIPQuery(dataElements, fs, false, true, 2); - - // Change query for NXDOMAIN - SystemConfiguration.setProperty("pir.esQuery", "?q=rcode:3"); - - SystemConfiguration.setProperty("pirTest.embedSelector", "true"); - BaseTests.testDNSNXDOMAINQuery(dataElements, fs, false, true, 3); - SystemConfiguration.setProperty("pirTest.embedSelector", "false"); - BaseTests.testDNSNXDOMAINQuery(dataElements, fs, false, true, 3); - - logger.info("Completed testESInputMR"); - } - - public static void testJSONInputSpark(FileSystem fs, List<JSONObject> dataElements) throws Exception - { - logger.info("Starting testJSONInputSpark"); - - SystemConfiguration.setProperty("pirTest.useHDFSExpLookupTable", "false"); - SystemConfiguration.setProperty("pirTest.useExpLookupTable", "false"); - SystemConfiguration.setProperty("pir.useModExpJoin", "false"); - - SystemConfiguration.setProperty("pir.limitHitsPerSelector", "false"); - SystemConfiguration.setProperty("pir.maxHitsPerSelector", "1000"); - - SystemConfiguration.setProperty("pir.numColMultPartitions", "20"); - SystemConfiguration.setProperty("pir.colMultReduceByKey", "false"); - - SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "false"); - SystemConfiguration.setProperty("pir.embedQuerySchema", "false"); - - // Set up JSON configs - SystemConfiguration.setProperty("pir.dataInputFormat", InputFormatConst.BASE_FORMAT); - SystemConfiguration.setProperty("pir.inputData", SystemConfiguration.getProperty(DistributedTestDriver.JSON_PIR_INPUT_FILE_PROPERTY)); - SystemConfiguration.setProperty("pir.baseQuery", "?q=rcode:0"); - - // Run tests - SystemConfiguration.setProperty("pirTest.embedSelector", "true"); - BaseTests.testDNSHostnameQuery(dataElements, fs, true, true, 1); - BaseTests.testDNSIPQuery(dataElements, fs, true, true, 1); - - SystemConfiguration.setProperty("pirTest.embedSelector", "false"); - BaseTests.testDNSHostnameQuery(dataElements, fs, true, true, 2); - BaseTests.testDNSIPQuery(dataElements, fs, true, true, 2); - BaseTests.testSRCIPQuery(dataElements, fs, true, true, 2); - - BaseTests.testSRCIPQueryNoFilter(dataElements, fs, true, true, 2); - - // Test embedded QuerySchema - SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "true"); - SystemConfiguration.setProperty("pir.embedQuerySchema", "false"); - BaseTests.testDNSHostnameQuery(dataElements, fs, true, true, 1); - - SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "true"); - SystemConfiguration.setProperty("pir.embedQuerySchema", "true"); - BaseTests.testDNSHostnameQuery(dataElements, fs, true, true, 1); - - SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "false"); - SystemConfiguration.setProperty("pir.embedQuerySchema", "true"); - BaseTests.testDNSHostnameQuery(dataElements, fs, true, true, 1); - SystemConfiguration.setProperty("pir.embedQuerySchema", "false"); - - // Test pad columns - SystemConfiguration.setProperty("pir.padEmptyColumns", "true"); - BaseTests.testDNSHostnameQuery(dataElements, fs, true, true, 1); - SystemConfiguration.setProperty("pir.padEmptyColumns", "false"); - - // Test hit limits per selector - SystemConfiguration.setProperty("pir.limitHitsPerSelector", "true"); - SystemConfiguration.setProperty("pir.maxHitsPerSelector", "1"); - BaseTests.testDNSHostnameQuery(dataElements, fs, true, true, 3); - SystemConfiguration.setProperty("pir.limitHitsPerSelector", "false"); - SystemConfiguration.setProperty("pir.maxHitsPerSelector", "1000"); - - // Test the local cache for modular exponentiation - SystemConfiguration.setProperty("pirTest.embedSelector", "true"); - SystemConfiguration.setProperty("pir.useLocalCache", "true"); - BaseTests.testDNSIPQuery(dataElements, fs, true, true, 3); - - // Test the join functionality for the modular exponentiation table - SystemConfiguration.setProperty("pir.useModExpJoin", "true"); - BaseTests.testDNSIPQuery(dataElements, fs, true, true, 3); - SystemConfiguration.setProperty("pir.useModExpJoin", "false"); - - // Test file based exp lookup table for modular exponentiation - SystemConfiguration.setProperty("pirTest.useHDFSExpLookupTable", "true"); - SystemConfiguration.setProperty("pir.expCreationSplits", "500"); - SystemConfiguration.setProperty("pir.numExpLookupPartitions", "150"); - BaseTests.testDNSIPQuery(dataElements, fs, true, true, 2); - SystemConfiguration.setProperty("pirTest.useHDFSExpLookupTable", "false"); - - // Change query for NXDOMAIN - SystemConfiguration.setProperty("pir.baseQuery", "?q=rcode:3"); - - SystemConfiguration.setProperty("pirTest.embedSelector", "true"); - BaseTests.testDNSNXDOMAINQuery(dataElements, fs, true, true, 3); - SystemConfiguration.setProperty("pirTest.embedSelector", "false"); - BaseTests.testDNSNXDOMAINQuery(dataElements, fs, true, true, 3); - - // Test with reduceByKey for column mult - SystemConfiguration.setProperty("pir.colMultReduceByKey", "true"); - BaseTests.testDNSNXDOMAINQuery(dataElements, fs, true, true, 3); - - logger.info("Completed testJSONInputSpark"); - } - - public static void testESInputSpark(FileSystem fs, List<JSONObject> dataElements) throws Exception - { - logger.info("Starting testESInputSpark"); - - SystemConfiguration.setProperty("pirTest.useHDFSExpLookupTable", "false"); - SystemConfiguration.setProperty("pirTest.useExpLookupTable", "false"); - - SystemConfiguration.setProperty("pir.limitHitsPerSelector", "false"); - SystemConfiguration.setProperty("pir.maxHitsPerSelector", "1000"); - - SystemConfiguration.setProperty("pir.allowAdHocQuerySchemas", "false"); - SystemConfiguration.setProperty("pir.embedQuerySchema", "false"); - - // Set up ES configs - SystemConfiguration.setProperty("pir.dataInputFormat", InputFormatConst.ES); - SystemConfiguration.setProperty("pir.esQuery", "?q=rcode:0"); - SystemConfiguration.setProperty("pir.esResource", SystemConfiguration.getProperty(DistributedTestDriver.ES_PIR_INPUT_RESOURCE_PROPERTY)); - - // Run tests - SystemConfiguration.setProperty("pirTest.embedSelector", "true"); - BaseTests.testDNSHostnameQuery(dataElements, fs, true, true, 1); - BaseTests.testDNSIPQuery(dataElements, fs, true, true, 1); - BaseTests.testSRCIPQuery(dataElements, fs, true, true, 2); - - SystemConfiguration.setProperty("pirTest.embedSelector", "false"); - BaseTests.testDNSHostnameQuery(dataElements, fs, true, true, 2); - BaseTests.testDNSIPQuery(dataElements, fs, true, true, 2); - - // Change query for NXDOMAIN - SystemConfiguration.setProperty("pir.esQuery", "?q=rcode:3"); - - SystemConfiguration.setProperty("pirTest.embedSelector", "true"); - BaseTests.testDNSNXDOMAINQuery(dataElements, fs, true, true, 3); - SystemConfiguration.setProperty("pirTest.embedSelector", "false"); - BaseTests.testDNSNXDOMAINQuery(dataElements, fs, true, true, 3); - - logger.info("Completed testESInputSpark"); - } - - // Base method to perform query - public static List<QueryResponseJSON> performQuery(String queryType, ArrayList<String> selectors, FileSystem fs, boolean isSpark, int numThreads) - throws Exception - { - logger.info("performQuery: "); - - String queryInputDir = SystemConfiguration.getProperty(DistributedTestDriver.PIR_QUERY_INPUT_DIR); - String outputFile = SystemConfiguration.getProperty(DistributedTestDriver.OUTPUT_DIRECTORY_PROPERTY); - fs.delete(new Path(outputFile), true); // Ensure old output does not exist. - - SystemConfiguration.setProperty("pir.queryInput", queryInputDir); - SystemConfiguration.setProperty("pir.outputFile", outputFile); - SystemConfiguration.setProperty("pir.numReduceTasks", "1"); - SystemConfiguration.setProperty("pir.stopListFile", SystemConfiguration.getProperty(DistributedTestDriver.PIR_STOPLIST_FILE)); - - // Create the temp result file - File fileFinalResults = File.createTempFile("finalResultsFile", ".txt"); - fileFinalResults.deleteOnExit(); - logger.info("fileFinalResults = " + fileFinalResults.getAbsolutePath()); - - boolean embedSelector = SystemConfiguration.getBooleanProperty("pirTest.embedSelector", false); - boolean useExpLookupTable = SystemConfiguration.getBooleanProperty("pirTest.useExpLookupTable", false); - boolean useHDFSExpLookupTable = SystemConfiguration.getBooleanProperty("pirTest.useHDFSExpLookupTable", false); - - // Set the necessary objects - QueryInfo queryInfo = new QueryInfo(BaseTests.queryIdentifier, selectors.size(), BaseTests.hashBitSize, BaseTests.hashKey, BaseTests.dataPartitionBitSize, - queryType, useExpLookupTable, embedSelector, useHDFSExpLookupTable); - - Paillier paillier = new Paillier(BaseTests.paillierBitSize, BaseTests.certainty); - - // Perform the encryption - logger.info("Performing encryption of the selectors - forming encrypted query vectors:"); - EncryptQuery encryptQuery = new EncryptQuery(queryInfo, selectors, paillier); - encryptQuery.encrypt(numThreads); - logger.info("Completed encryption of the selectors - completed formation of the encrypted query vectors:"); - - // Grab the necessary objects - Querier querier = encryptQuery.getQuerier(); - Query query = encryptQuery.getQuery(); - - // Write the Querier object to a file - Path queryInputDirPath = new Path(queryInputDir); - new HadoopFileSystemStore(fs).store(queryInputDirPath, query); - fs.deleteOnExit(queryInputDirPath); - - // Grab the original data and query schema properties to reset upon completion - String dataSchemaProp = SystemConfiguration.getProperty("data.schemas"); - String querySchemaProp = SystemConfiguration.getProperty("query.schemas"); - - // Get the correct input format class name - JSONInputFormatBase jFormat = new JSONInputFormatBase(); - String jsonBaseInputFormatString = jFormat.getClass().getName(); - SystemConfiguration.setProperty("pir.baseInputFormat", jsonBaseInputFormatString); - - // Submitting the tool for encrypted query - logger.info("Performing encrypted query:"); - if (isSpark) - { - // Build args - String inputFormat = SystemConfiguration.getProperty("pir.dataInputFormat"); - logger.info("inputFormat = " + inputFormat); - ArrayList<String> args = new ArrayList<>(); - args.add("-" + ResponderProps.PLATFORM + "=spark"); - args.add("-" + ResponderProps.DATAINPUTFORMAT + "=" + inputFormat); - args.add("-" + ResponderProps.QUERYINPUT + "=" + SystemConfiguration.getProperty("pir.queryInput")); - args.add("-" + ResponderProps.OUTPUTFILE + "=" + SystemConfiguration.getProperty("pir.outputFile")); - args.add("-" + ResponderProps.STOPLISTFILE + "=" + SystemConfiguration.getProperty("pir.stopListFile")); - args.add("-" + ResponderProps.USELOCALCACHE + "=" + SystemConfiguration.getProperty("pir.useLocalCache", "true")); - args.add("-" + ResponderProps.LIMITHITSPERSELECTOR + "=" + SystemConfiguration.getProperty("pir.limitHitsPerSelector", "false")); - args.add("-" + ResponderProps.MAXHITSPERSELECTOR + "=" + SystemConfiguration.getProperty("pir.maxHitsPerSelector", "1000")); - args.add("-" + ResponderProps.QUERYSCHEMAS + "=" + Inputs.HDFS_QUERY_FILES); - args.add("-" + ResponderProps.DATASCHEMAS + "=" + Inputs.DATA_SCHEMA_FILE_HDFS); - args.add("-" + ResponderProps.NUMEXPLOOKUPPARTS + "=" + SystemConfiguration.getProperty("pir.numExpLookupPartitions", "100")); - args.add("-" + ResponderProps.USEMODEXPJOIN + "=" + SystemConfiguration.getProperty("pir.useModExpJoin", "false")); - args.add("-" + ResponderProps.NUMCOLMULTPARTITIONS + "=" + SystemConfiguration.getProperty("pir.numColMultPartitions", "20")); - args.add("-" + ResponderProps.COLMULTREDUCEBYKEY + "=" + SystemConfiguration.getProperty("pir.colMultReduceByKey", "false")); - if (inputFormat.equals(InputFormatConst.BASE_FORMAT)) - { - args.add("-" + ResponderProps.INPUTDATA + "=" + SystemConfiguration.getProperty("pir.inputData")); - args.add("-" + ResponderProps.BASEQUERY + "=" + SystemConfiguration.getProperty("pir.baseQuery")); - args.add("-" + ResponderProps.BASEINPUTFORMAT + "=" + SystemConfiguration.getProperty("pir.baseInputFormat")); - } - else if (inputFormat.equals(InputFormatConst.ES)) - { - args.add("-" + ResponderProps.ESQUERY + "=" + SystemConfiguration.getProperty("pir.esQuery")); - args.add("-" + ResponderProps.ESRESOURCE + "=" + SystemConfiguration.getProperty("pir.esResource")); - } - - for (String arg : args) - { - logger.info("arg = " + arg); - } - - // Run spark application - Process sLauncher = new SparkLauncher().setAppResource(SystemConfiguration.getProperty("jarFile")) - .setSparkHome(SystemConfiguration.getProperty("spark.home")).setMainClass("org.apache.pirk.responder.wideskies.ResponderDriver") - .addAppArgs(args.toArray(new String[args.size()])).setMaster("yarn-cluster").setConf(SparkLauncher.EXECUTOR_MEMORY, "2g") - .setConf(SparkLauncher.DRIVER_MEMORY, "2g").setConf(SparkLauncher.EXECUTOR_CORES, "1").launch(); - sLauncher.waitFor(); - } - else - { - SystemConfiguration.setProperty("data.schemas", Inputs.DATA_SCHEMA_FILE_HDFS); - SystemConfiguration.setProperty("query.schemas", Inputs.HDFS_QUERY_FILES); - - ComputeResponseTool responseTool = new ComputeResponseTool(); - ToolRunner.run(responseTool, new String[] {}); - } - logger.info("Completed encrypted query"); - - // Perform decryption - // Reconstruct the necessary objects from the files - logger.info("Performing decryption; writing final results file"); - Response response = new HadoopFileSystemStore(fs).recall(outputFile, Response.class); - - // Perform decryption and output the result file - DecryptResponse decryptResponse = new DecryptResponse(response, querier); - decryptResponse.decrypt(numThreads); - decryptResponse.writeResultFile(fileFinalResults); - logger.info("Completed performing decryption and writing final results file"); - - // Read in results - logger.info("Reading in and checking results"); - List<QueryResponseJSON> results = TestUtils.readResultsFile(fileFinalResults); - - // Reset data and query schema properties - SystemConfiguration.setProperty("data.schemas", dataSchemaProp); - SystemConfiguration.setProperty("query.schemas", querySchemaProp); - - // Clean up output dir in hdfs - fs.delete(new Path(outputFile), true); - - return results; - } -}
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/9244df72/src/test/java/org/apache/pirk/test/general/ISO8601DateParserTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/pirk/test/general/ISO8601DateParserTest.java b/src/test/java/org/apache/pirk/test/general/ISO8601DateParserTest.java deleted file mode 100644 index 9821db3..0000000 --- a/src/test/java/org/apache/pirk/test/general/ISO8601DateParserTest.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pirk.test.general; - -import static org.junit.Assert.assertEquals; - -import java.text.ParseException; - -import org.apache.pirk.utils.ISO8601DateParser; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Class to test basic functionality of ISO8601DateParser class - */ -public class ISO8601DateParserTest -{ - private static final Logger logger = LoggerFactory.getLogger(ISO8601DateParserTest.class); - - @Test - public void testDateParsing() throws ParseException - { - logger.info("Starting testDateParsing: "); - - String date = "2016-02-20T23:29:05.000Z"; - long longDate = Long.parseLong("1456010945000"); // date in UTC - - assertEquals(longDate, ISO8601DateParser.getLongDate(date)); - assertEquals(date, ISO8601DateParser.fromLongDate(longDate)); - - logger.info("Successfully completed testDateParsing"); - } -} http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/9244df72/src/test/java/org/apache/pirk/test/general/KeyedHashTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/pirk/test/general/KeyedHashTest.java b/src/test/java/org/apache/pirk/test/general/KeyedHashTest.java deleted file mode 100644 index e53cebf..0000000 --- a/src/test/java/org/apache/pirk/test/general/KeyedHashTest.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pirk.test.general; - -import static org.junit.Assert.assertEquals; - -import org.apache.pirk.utils.KeyedHash; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Basic functional tests for KeyedHash - * - */ -public class KeyedHashTest -{ - private static final Logger logger = LoggerFactory.getLogger(KeyedHashTest.class); - - @Test - public void testKeyedHash() - { - logger.info("Starting testKeyedHash: "); - - int hash1 = KeyedHash.hash("someKey", 12, "someInput"); - logger.info("hash1 = " + hash1 + " hash1 = " + Integer.toString(hash1, 2)); - - int hash2 = KeyedHash.hash("someKey", 32, "someInput"); - logger.info("hash2 = " + hash2 + " hash2 = " + Integer.toString(hash2, 2)); - - int hash3 = KeyedHash.hash("someKey", 34, "someInput"); - logger.info("hash3 = " + hash3 + " hash3 = " + Integer.toString(hash3, 2)); - - assertEquals(hash2, hash3); - assertEquals(hash1, hash2 & 0xFFF); - - logger.info("Successfully completed testKeyedHash"); - } - - @Test - public void testKeyedHashWithType() - { - testKeyedHashType("MD5"); - testKeyedHashType("SHA-1"); - testKeyedHashType("SHA-256"); - testKeyedHashType("FAKE-HASH-TYPE"); - } - - private void testKeyedHashType(String type) - { - logger.info("Starting testKeyedHashType with type: " + type); - - int hash1 = KeyedHash.hash("someKey", 12, "someInput", type); - logger.info("hash1 = " + hash1 + " hash1 = " + Integer.toString(hash1, 2)); - - int hash2 = KeyedHash.hash("someKey", 32, "someInput", type); - logger.info("hash2 = " + hash2 + " hash2 = " + Integer.toString(hash2, 2)); - - int hash3 = KeyedHash.hash("someKey", 34, "someInput", type); - logger.info("hash3 = " + hash3 + " hash3 = " + Integer.toString(hash3, 2)); - - assertEquals(hash2, hash3); - assertEquals(hash1, hash2 & 0xFFF); - - logger.info("Successfully completed testKeyedHashType with type: " + type); - } -} http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/9244df72/src/test/java/org/apache/pirk/test/general/PaillierTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/pirk/test/general/PaillierTest.java b/src/test/java/org/apache/pirk/test/general/PaillierTest.java deleted file mode 100644 index 6aa1396..0000000 --- a/src/test/java/org/apache/pirk/test/general/PaillierTest.java +++ /dev/null @@ -1,303 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pirk.test.general; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.fail; - -import java.math.BigInteger; -import java.util.Random; - -import org.apache.pirk.encryption.Paillier; -import org.apache.pirk.utils.PIRException; -import org.apache.pirk.utils.SystemConfiguration; -import org.junit.BeforeClass; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Basic test functionality for Paillier library - * - */ -public class PaillierTest -{ - private static final Logger logger = LoggerFactory.getLogger(PaillierTest.class); - - private static BigInteger p = null; // large prime - private static BigInteger q = null; // large prime - private static BigInteger N = null; // N=pq, RSA modulus - private static BigInteger NSquared = null; // N^2 - private static BigInteger lambdaN = null; // lambda(N) = lcm(p-1,q-1) - - private static int bitLength = 0; // bit length of the modulus N - private static int certainty = 64; // prob that new BigInteger values represents primes will exceed (1 - (1/2)^certainty) - - private static BigInteger r1 = null; // random number in (Z/NZ)* - private static BigInteger r2 = null; // random number in (Z/NZ)* - - private static BigInteger m1 = null; // message to encrypt - private static BigInteger m2 = null; // message to encrypt - - @BeforeClass - public static void setup() - { - p = BigInteger.valueOf(7); - q = BigInteger.valueOf(17); - N = p.multiply(q); - NSquared = N.multiply(N); - - lambdaN = BigInteger.valueOf(48); - - r1 = BigInteger.valueOf(3); - r2 = BigInteger.valueOf(4); - - m1 = BigInteger.valueOf(5); - m2 = BigInteger.valueOf(2); - - bitLength = 201;// bitLength = 384; - certainty = 128; - - logger.info("p = " + p.intValue() + " q = " + q.intValue() + " N = " + N.intValue() + " bitLength = " + N.bitLength() + " lambdaN = " + lambdaN + " m1 = " - + m1.intValue() + " m2 = " + m2.intValue() + " r1 = " + r1.intValue() + " r2 = " + r2.intValue()); - } - - @Test - public void testPIRExceptions() - { - try - { - Paillier paillier = new Paillier(BigInteger.valueOf(2), BigInteger.valueOf(2), 128); - assertNotNull(paillier); - fail("Paillier constructor did not throw PIRException for p,q < 3"); - } catch (PIRException ignore) - {} - - try - { - Paillier paillier = new Paillier(BigInteger.valueOf(2), BigInteger.valueOf(3), 128); - assertNotNull(paillier); - fail("Paillier constructor did not throw PIRException for p < 3"); - } catch (PIRException ignore) - {} - - try - { - Paillier paillier = new Paillier(BigInteger.valueOf(3), BigInteger.valueOf(2), 128); - assertNotNull(paillier); - fail("Paillier constructor did not throw PIRException for q < 3"); - } catch (PIRException ignore) - {} - - try - { - Paillier paillier = new Paillier(BigInteger.valueOf(7), BigInteger.valueOf(7), 128); - assertNotNull(paillier); - fail("Paillier constructor did not throw PIRException for p = q"); - } catch (PIRException ignore) - {} - - try - { - Paillier paillier = new Paillier(BigInteger.valueOf(8), BigInteger.valueOf(7), 128); - assertNotNull(paillier); - fail("Paillier constructor did not throw PIRException for p not prime"); - } catch (PIRException ignore) - {} - - try - { - Paillier paillier = new Paillier(BigInteger.valueOf(7), BigInteger.valueOf(10), 128); - assertNotNull(paillier); - fail("Paillier constructor did not throw PIRException for q not prime"); - } catch (PIRException ignore) - {} - - try - { - int systemPrimeCertainty = SystemConfiguration.getIntProperty("pir.primeCertainty", 128); - Paillier paillier = new Paillier(3072, systemPrimeCertainty - 10); - assertNotNull(paillier); - fail("Paillier constructor did not throw PIRException for certainty less than system default of " + systemPrimeCertainty); - } catch (PIRException ignore) - {} - - try - { - Paillier pailler = new Paillier(p, q, bitLength); - BigInteger encM1 = pailler.encrypt(N); - assertNotNull(encM1); - fail("Paillier encryption did not throw PIRException for message m = N"); - } catch (PIRException ignore) - {} - - try - { - Paillier pailler = new Paillier(p, q, bitLength); - BigInteger encM1 = pailler.encrypt(N.add(BigInteger.TEN)); - assertNotNull(encM1); - fail("Paillier encryption did not throw PIRException for message m > N"); - } catch (PIRException ignore) - {} - - try - { - Paillier pailler = new Paillier(bitLength, 128, bitLength); - assertNotNull(pailler); - fail("Paillier constructor did not throw PIRException for ensureBitSet = bitLength"); - } catch (PIRException ignore) - {} - - try - { - Paillier pailler = new Paillier(bitLength, 128, bitLength + 1); - assertNotNull(pailler); - fail("Paillier constructor did not throw PIRException for ensureBitSet > bitLength"); - } catch (PIRException ignore) - {} - } - - @Test - public void testPaillierGivenAllParameters() throws Exception - { - logger.info("Starting testPaillierGivenAllParameters: "); - - Paillier pailler = new Paillier(p, q, bitLength); - - assertEquals(pailler.getN(), N); - assertEquals(pailler.getLambdaN(), lambdaN); - - // Check encryption - BigInteger encM1 = pailler.encrypt(m1, r1); - BigInteger encM2 = pailler.encrypt(m2, r2); - logger.info("encM1 = " + encM1.intValue() + " encM2 = " + encM2.intValue()); - - assertEquals(encM1, BigInteger.valueOf(14019)); - assertEquals(encM2, BigInteger.valueOf(8836)); - - // Check decryption - BigInteger decM1 = pailler.decrypt(encM1); - BigInteger decM2 = pailler.decrypt(encM2); - logger.info("decM1 = " + decM1.intValue() + " decM2 = " + decM2.intValue()); - - assertEquals(decM1, m1); - assertEquals(decM2, m2); - - // Check homomorphic property: E_r1(m1)*E_r2(m2) mod N^2 = E_r1r2((m1+m2) mod N) mod N^2 - BigInteger encM1_times_encM2 = (encM1.multiply(encM2)).mod(NSquared); - BigInteger encM1plusM2 = pailler.encrypt((m1.add(m2)).mod(N), r1.multiply(r2)); - logger.info("encM1_times_encM2 = " + encM1_times_encM2.intValue() + " encM1plusM2 = " + encM1plusM2.intValue()); - - assertEquals(encM1_times_encM2, BigInteger.valueOf(5617)); - assertEquals(encM1plusM2, BigInteger.valueOf(5617)); - - logger.info("Successfully completed testPaillierGivenAllParameters: "); - } - - @Test - public void testPaillierWithKeyGeneration() throws Exception - { - logger.info("Starting testPaillierWithKeyGeneration: "); - - // Test with and without gmp optimization for modPow - SystemConfiguration.setProperty("pallier.FIPSPrimeGenerationChecks", "true"); - SystemConfiguration.setProperty("paillier.useGMPForModPow", "true"); - SystemConfiguration.setProperty("paillier.GMPConstantTimeMode", "true"); - testPaillerWithKeyGenerationGeneral(); - - SystemConfiguration.setProperty("pallier.FIPSPrimeGenerationChecks", "false"); - - SystemConfiguration.setProperty("paillier.useGMPForModPow", "true"); - SystemConfiguration.setProperty("paillier.GMPConstantTimeMode", "true"); - testPaillerWithKeyGenerationGeneral(); - - SystemConfiguration.setProperty("paillier.useGMPForModPow", "true"); - SystemConfiguration.setProperty("paillier.GMPConstantTimeMode", "false"); - testPaillerWithKeyGenerationGeneral(); - - SystemConfiguration.setProperty("paillier.useGMPForModPow", "false"); - SystemConfiguration.setProperty("paillier.GMPConstantTimeMode", "false"); - testPaillerWithKeyGenerationGeneral(); - - // Reset the properties - SystemConfiguration.initialize(); - - logger.info("Ending testPaillierWithKeyGeneration: "); - } - - public void testPaillerWithKeyGenerationGeneral() throws Exception - { - // Test without requiring highest bit to be set - logger.info("Starting testPaillierWithKeyGenerationBitSetOption with ensureHighBitSet = false"); - testPaillierWithKeyGenerationBitSetOption(-1); - - // Test requiring highest bit to be set - logger.info("Starting testPaillierWithKeyGenerationBitSetOption with ensureHighBitSet = true"); - testPaillierWithKeyGenerationBitSetOption(5); - } - - public void testPaillierWithKeyGenerationBitSetOption(int ensureBitSet) throws Exception - { - Random r = new Random(); - int lowBitLength = 3073; // inclusive - int highBitLength = 7001; // exclusive - - int loopVal = 1; // int loopVal = 1000; //change this and re-test for high loop testing - for (int i = 0; i < loopVal; ++i) - { - logger.info("i = " + i); - - basicTestPaillierWithKeyGeneration(bitLength, certainty, ensureBitSet); - basicTestPaillierWithKeyGeneration(3072, certainty, ensureBitSet); - - // Test with random bit length between 3073 and 7000 - int randomLargeBitLength = r.nextInt(highBitLength - lowBitLength) + lowBitLength; - basicTestPaillierWithKeyGeneration(randomLargeBitLength, certainty, ensureBitSet); - } - } - - private void basicTestPaillierWithKeyGeneration(int bitLengthInput, int certaintyInput, int ensureBitSet) throws Exception - { - Paillier pailler = new Paillier(bitLengthInput, certaintyInput, ensureBitSet); - BigInteger generatedN = pailler.getN(); - BigInteger geneartedNsquared = generatedN.multiply(generatedN); - - // Check the decrypting the encryption yields the message - BigInteger encM1 = pailler.encrypt(m1); - BigInteger encM2 = pailler.encrypt(m2); - logger.info("encM1 = " + encM1.intValue() + " encM2 = " + encM2.intValue()); - - BigInteger decM1 = pailler.decrypt(encM1); - BigInteger decM2 = pailler.decrypt(encM2); - logger.info("decM1 = " + decM1.intValue() + " decM2 = " + decM2.intValue()); - - assertEquals(decM1, m1); - assertEquals(decM2, m2); - - // Check homomorphic property: E_r1(m1)*E_r2(m2) mod N^2 = E_r1r2((m1+m2) mod N) mod N^2 - BigInteger encM1_times_encM2 = (encM1.multiply(encM2)).mod(geneartedNsquared); - BigInteger multDecrypt = pailler.decrypt(encM1_times_encM2); - BigInteger m1_plus_m2 = (m1.add(m2)).mod(N); - - logger.info("encM1_times_encM2 = " + encM1_times_encM2.intValue() + " multDecrypt = " + multDecrypt.intValue() + " m1_plus_m2 = " + m1_plus_m2.intValue()); - - assertEquals(multDecrypt, m1_plus_m2); - } -} http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/9244df72/src/test/java/org/apache/pirk/test/general/PartitionUtilsTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/pirk/test/general/PartitionUtilsTest.java b/src/test/java/org/apache/pirk/test/general/PartitionUtilsTest.java deleted file mode 100644 index c1022ad..0000000 --- a/src/test/java/org/apache/pirk/test/general/PartitionUtilsTest.java +++ /dev/null @@ -1,269 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pirk.test.general; - -import org.apache.pirk.schema.data.partitioner.IPDataPartitioner; -import org.apache.pirk.schema.data.partitioner.ISO8601DatePartitioner; -import org.apache.pirk.schema.data.partitioner.PrimitiveTypePartitioner; -import org.apache.pirk.utils.PIRException; -import org.apache.pirk.utils.SystemConfiguration; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.math.BigInteger; -import java.util.Arrays; -import java.util.List; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - -/** - * Class to functionally test the bit conversion utils - */ -public class PartitionUtilsTest -{ - private static final Logger logger = LoggerFactory.getLogger(PartitionUtilsTest.class); - - @Test - public void testMask() - { - logger.info("Starting testMask: "); - - assertEquals(0, PrimitiveTypePartitioner.formBitMask(0).intValue()); - - assertEquals(0b000000000000001, PrimitiveTypePartitioner.formBitMask(1).intValue()); - assertEquals(0b000000000001111, PrimitiveTypePartitioner.formBitMask(4).intValue()); - assertEquals(0b000000001111111, PrimitiveTypePartitioner.formBitMask(7).intValue()); - assertEquals(0b111111111111111, PrimitiveTypePartitioner.formBitMask(15).intValue()); - - assertEquals(new BigInteger("FFFFF", 16), PrimitiveTypePartitioner.formBitMask(20)); - assertEquals(new BigInteger("FFFFFFFF", 16), PrimitiveTypePartitioner.formBitMask(32)); - assertEquals(new BigInteger("3FFFFFFFFFF", 16), PrimitiveTypePartitioner.formBitMask(42)); - assertEquals(new BigInteger("7FFFFFFFFFFFFFFF", 16), PrimitiveTypePartitioner.formBitMask(63)); - - logger.info("Successfully completed testMask"); - } - - @Test - public void testPartitionBits() throws PIRException - { - logger.info("Starting testPartitionBits: "); - - BigInteger value = new BigInteger("245"); // 11110101 - BigInteger value2 = new BigInteger("983"); // 1111010111 - - BigInteger mask4 = PrimitiveTypePartitioner.formBitMask(4); // 1111 - BigInteger mask8 = PrimitiveTypePartitioner.formBitMask(8); // 11111111 - - List<BigInteger> partitions = PrimitiveTypePartitioner.partitionBits(value, 4, mask4); - assertEquals(2, partitions.size()); - assertEquals(0b1111, partitions.get(0).intValue()); - assertEquals(0b0101, partitions.get(1).intValue()); - - partitions = PrimitiveTypePartitioner.partitionBits(value2, 4, mask4); - assertEquals(3, partitions.size()); - assertEquals(0b1111, partitions.get(0).intValue()); - assertEquals(0b0101, partitions.get(1).intValue()); - assertEquals(0b0011, partitions.get(2).intValue()); - - partitions = PrimitiveTypePartitioner.partitionBits(value, 8, mask8); - assertEquals(1, partitions.size()); - assertEquals(0b11110101, partitions.get(0).intValue()); - - try - { - partitions = PrimitiveTypePartitioner.partitionBits(value, 4, mask8); - fail("BitConversionUtils.partitionBits did not throw error for mismatched partitionSize and mask size"); - } catch (Exception ignore) - { - // Expected. - } - - logger.info("Successfully completed testPartitionBits"); - } - - @Test - public void testPartitions() throws Exception - { - logger.info("Starting testToPartitions:"); - - PrimitiveTypePartitioner primitivePartitioner = new PrimitiveTypePartitioner(); - IPDataPartitioner ipPartitioner = new IPDataPartitioner(); - ISO8601DatePartitioner datePartitioner = new ISO8601DatePartitioner(); - - // Test IP - String ipTest = "127.0.0.1"; - List<BigInteger> partsIP = ipPartitioner.toPartitions(ipTest, PrimitiveTypePartitioner.STRING); - assertEquals(4, partsIP.size()); - assertEquals(ipTest, ipPartitioner.fromPartitions(partsIP, 0, PrimitiveTypePartitioner.STRING)); - - // Test Date - String dateTest = "2016-02-20T23:29:05.000Z"; - List<BigInteger> partsDate = datePartitioner.toPartitions(dateTest, null); - assertEquals(8, partsDate.size()); - assertEquals(dateTest, datePartitioner.fromPartitions(partsDate, 0, null)); - - // Test byte - byte bTest = Byte.parseByte("10"); - List<BigInteger> partsByte = primitivePartitioner.toPartitions(bTest, PrimitiveTypePartitioner.BYTE); - assertEquals(1, partsByte.size()); - assertEquals(bTest, primitivePartitioner.fromPartitions(partsByte, 0, PrimitiveTypePartitioner.BYTE)); - - partsByte = primitivePartitioner.toPartitions("12", PrimitiveTypePartitioner.BYTE); - assertEquals(1, partsByte.size()); - assertEquals((byte) 12, primitivePartitioner.fromPartitions(partsByte, 0, PrimitiveTypePartitioner.BYTE)); - - List<BigInteger> partsByteMax = primitivePartitioner.toPartitions(Byte.MAX_VALUE, PrimitiveTypePartitioner.BYTE); - assertEquals(1, partsByteMax.size()); - assertEquals(Byte.MAX_VALUE, primitivePartitioner.fromPartitions(partsByteMax, 0, PrimitiveTypePartitioner.BYTE)); - - // Test string - String stringBits = SystemConfiguration.getProperty("pir.stringBits"); - SystemConfiguration.setProperty("pir.stringBits", "64"); - testString("testString"); // over the allowed bit size - testString("t"); // under the allowed bit size - SystemConfiguration.setProperty("pir.stringBits", stringBits); - - // Test short - short shortTest = Short.valueOf("2456"); - List<BigInteger> partsShort = primitivePartitioner.toPartitions(shortTest, PrimitiveTypePartitioner.SHORT); - assertEquals(2, partsShort.size()); - assertEquals(shortTest, primitivePartitioner.fromPartitions(partsShort, 0, PrimitiveTypePartitioner.SHORT)); - - partsShort = primitivePartitioner.toPartitions("32767", PrimitiveTypePartitioner.SHORT); - assertEquals(2, partsShort.size()); - assertEquals((short) 32767, primitivePartitioner.fromPartitions(partsShort, 0, PrimitiveTypePartitioner.SHORT)); - - partsShort = primitivePartitioner.toPartitions((short) -42, PrimitiveTypePartitioner.SHORT); - assertEquals(2, partsShort.size()); - assertEquals((short) -42, primitivePartitioner.fromPartitions(partsShort, 0, PrimitiveTypePartitioner.SHORT)); - - List<BigInteger> partsShortMax = primitivePartitioner.toPartitions(Short.MAX_VALUE, PrimitiveTypePartitioner.SHORT); - assertEquals(2, partsShortMax.size()); - assertEquals(Short.MAX_VALUE, primitivePartitioner.fromPartitions(partsShortMax, 0, PrimitiveTypePartitioner.SHORT)); - - // Test int - int intTest = Integer.parseInt("-5789"); - List<BigInteger> partsInt = primitivePartitioner.toPartitions(intTest, PrimitiveTypePartitioner.INT); - assertEquals(4, partsInt.size()); - assertEquals(intTest, primitivePartitioner.fromPartitions(partsInt, 0, PrimitiveTypePartitioner.INT)); - - partsInt = primitivePartitioner.toPartitions("2016", PrimitiveTypePartitioner.INT); - assertEquals(4, partsInt.size()); - assertEquals(2016, primitivePartitioner.fromPartitions(partsInt, 0, PrimitiveTypePartitioner.INT)); - - partsInt = primitivePartitioner.toPartitions(1386681237, PrimitiveTypePartitioner.INT); - assertEquals(4, partsInt.size()); - assertEquals(1386681237, primitivePartitioner.fromPartitions(partsInt, 0, PrimitiveTypePartitioner.INT)); - - List<BigInteger> partsIntMax = primitivePartitioner.toPartitions(Integer.MAX_VALUE, PrimitiveTypePartitioner.INT); - assertEquals(4, partsIntMax.size()); - assertEquals(Integer.MAX_VALUE, primitivePartitioner.fromPartitions(partsIntMax, 0, PrimitiveTypePartitioner.INT)); - - // Test long - long longTest = Long.parseLong("56789"); - List<BigInteger> partsLong = primitivePartitioner.toPartitions(longTest, PrimitiveTypePartitioner.LONG); - assertEquals(8, partsLong.size()); - assertEquals(longTest, primitivePartitioner.fromPartitions(partsLong, 0, PrimitiveTypePartitioner.LONG)); - - List<BigInteger> partsLongMax = primitivePartitioner.toPartitions(Long.MAX_VALUE, PrimitiveTypePartitioner.LONG); - assertEquals(8, partsLongMax.size()); - assertEquals(Long.MAX_VALUE, primitivePartitioner.fromPartitions(partsLongMax, 0, PrimitiveTypePartitioner.LONG)); - - // Test float - float floatTest = Float.parseFloat("567.77"); - List<BigInteger> partsFloat = primitivePartitioner.toPartitions(floatTest, PrimitiveTypePartitioner.FLOAT); - assertEquals(4, partsFloat.size()); - assertEquals(floatTest, primitivePartitioner.fromPartitions(partsFloat, 0, PrimitiveTypePartitioner.FLOAT)); - - partsFloat = primitivePartitioner.toPartitions(-99.99f, PrimitiveTypePartitioner.FLOAT); - assertEquals(4, partsFloat.size()); - assertEquals(-99.99f, primitivePartitioner.fromPartitions(partsFloat, 0, PrimitiveTypePartitioner.FLOAT)); - - List<BigInteger> partsFloatMax = primitivePartitioner.toPartitions(Float.MAX_VALUE, PrimitiveTypePartitioner.FLOAT); - assertEquals(4, partsFloatMax.size()); - assertEquals(Float.MAX_VALUE, primitivePartitioner.fromPartitions(partsFloatMax, 0, PrimitiveTypePartitioner.FLOAT)); - - // Test double - double doubleTest = Double.parseDouble("567.77"); - List<BigInteger> partsDouble = primitivePartitioner.toPartitions(doubleTest, PrimitiveTypePartitioner.DOUBLE); - assertEquals(8, partsDouble.size()); - assertEquals(doubleTest, primitivePartitioner.fromPartitions(partsDouble, 0, PrimitiveTypePartitioner.DOUBLE)); - - List<BigInteger> partsDoubleMax = primitivePartitioner.toPartitions(Double.MAX_VALUE, PrimitiveTypePartitioner.DOUBLE); - assertEquals(8, partsDoubleMax.size()); - assertEquals(Double.MAX_VALUE, primitivePartitioner.fromPartitions(partsDoubleMax, 0, PrimitiveTypePartitioner.DOUBLE)); - - // Test char - char charTest = 'b'; - List<BigInteger> partsChar = primitivePartitioner.toPartitions(charTest, PrimitiveTypePartitioner.CHAR); - assertEquals(2, partsChar.size()); - assertEquals(charTest, primitivePartitioner.fromPartitions(partsChar, 0, PrimitiveTypePartitioner.CHAR)); - - // Ensure Endianness preserved - charTest = '\uFFFE'; - partsChar = primitivePartitioner.toPartitions(charTest, PrimitiveTypePartitioner.CHAR); - assertEquals(2, partsChar.size()); - assertEquals(charTest, primitivePartitioner.fromPartitions(partsChar, 0, PrimitiveTypePartitioner.CHAR)); - - charTest = '\uFEFF'; - partsChar = primitivePartitioner.toPartitions(charTest, PrimitiveTypePartitioner.CHAR); - assertEquals(2, partsChar.size()); - assertEquals(charTest, primitivePartitioner.fromPartitions(partsChar, 0, PrimitiveTypePartitioner.CHAR)); - - List<BigInteger> partsCharMax = primitivePartitioner.toPartitions(Character.MAX_VALUE, PrimitiveTypePartitioner.CHAR); - assertEquals(2, partsCharMax.size()); - assertEquals(Character.MAX_VALUE, primitivePartitioner.fromPartitions(partsCharMax, 0, PrimitiveTypePartitioner.CHAR)); - - logger.info("Sucessfully completed testToPartitions:"); - } - - @Test - public void testPaddedPartitions() throws PIRException - { - PrimitiveTypePartitioner primitivePartitioner = new PrimitiveTypePartitioner(); - - List<String> primitiveTypes = Arrays.asList(PrimitiveTypePartitioner.BYTE, PrimitiveTypePartitioner.CHAR, PrimitiveTypePartitioner.SHORT, - PrimitiveTypePartitioner.INT, PrimitiveTypePartitioner.LONG, PrimitiveTypePartitioner.FLOAT, PrimitiveTypePartitioner.DOUBLE, - PrimitiveTypePartitioner.STRING); - for (String type : primitiveTypes) - { - assertEquals(primitivePartitioner.getNumPartitions(type), primitivePartitioner.getPaddedPartitions(type).size()); - } - } - - private void testString(String testString) throws Exception - { - PrimitiveTypePartitioner ptp = new PrimitiveTypePartitioner(); - - List<BigInteger> partsString = ptp.toPartitions(testString, PrimitiveTypePartitioner.STRING); - int numParts = Integer.parseInt(SystemConfiguration.getProperty("pir.stringBits")) / 8; - assertEquals(numParts, partsString.size()); - - logger.info("testString.getBytes().length = " + testString.getBytes().length); - int offset = numParts; - if (testString.getBytes().length < numParts) - { - offset = testString.getBytes().length; - } - String element = new String(testString.getBytes(), 0, offset); - assertEquals(element, ptp.fromPartitions(partsString, 0, PrimitiveTypePartitioner.STRING)); - } -} http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/9244df72/src/test/java/org/apache/pirk/test/general/QueryParserUtilsTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/pirk/test/general/QueryParserUtilsTest.java b/src/test/java/org/apache/pirk/test/general/QueryParserUtilsTest.java deleted file mode 100644 index b3db140..0000000 --- a/src/test/java/org/apache/pirk/test/general/QueryParserUtilsTest.java +++ /dev/null @@ -1,421 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pirk.test.general; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import java.util.ArrayList; -import java.util.Map; - -import org.apache.hadoop.io.MapWritable; -import org.apache.pirk.schema.data.DataSchema; -import org.apache.pirk.schema.data.DataSchemaRegistry; -import org.apache.pirk.schema.query.QuerySchemaRegistry; -import org.apache.pirk.test.utils.Inputs; -import org.apache.pirk.utils.QueryParserUtils; -import org.apache.pirk.utils.StringUtils; -import org.apache.pirk.utils.SystemConfiguration; -import org.json.simple.JSONObject; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Class for testing the QueryParser methods - */ -public class QueryParserUtilsTest -{ - private static final Logger logger = LoggerFactory.getLogger(QueryParserUtilsTest.class); - - private static MapWritable doc = null; // MapWritable with arrays in json string representation - private static MapWritable docWAW = null; // MapWritable with arrays as WritableArrayWritable objects - private static Map<String,Object> docMap = null; // arrays as ArrayList<String> - - private static DataSchema dSchema = null; - - @BeforeClass - public static void setup() throws Exception - { - ArrayList<JSONObject> dataElementsJSON = Inputs.createJSONDataElements(); - - // Reset the schema properties and registries - DataSchemaRegistry.clearRegistry(); - QuerySchemaRegistry.clearRegistry(); - SystemConfiguration.setProperty("data.schemas", "none"); - SystemConfiguration.setProperty("query.schemas", "none"); - - Inputs.createSchemaFiles(null, false, null); - - dSchema = DataSchemaRegistry.get(Inputs.TEST_DATA_SCHEMA_NAME); - - // ProcessBuilder pAdd1 = new ProcessBuilder("curl", "-XPUT", indexTypeNum1, "-d", - // "{\"qname\":\"a.b.c.com\",\"date\":\"2016-02-20T23:29:05.000Z\",\"qtype\":[\"1\"]" - // + ",\"rcode\":\"0\",\"src_ip\":\"55.55.55.55\",\"dest_ip\":\"1.2.3.6\"" + ",\"ip\":[\"10.20.30.40\",\"10.20.30.60\"]}"); - // - doc = StringUtils.jsonStringToMapWritableWithArrayWritable(dataElementsJSON.get(0).toJSONString(), dSchema); - docWAW = StringUtils.jsonStringToMapWritableWithWritableArrayWritable(dataElementsJSON.get(0).toJSONString(), dSchema); - docMap = StringUtils.jsonStringToMap(dataElementsJSON.get(0).toJSONString(), dSchema); - } - - @AfterClass - public static void teardown() - { - // Reset the schema properties and registries - DataSchemaRegistry.clearRegistry(); - QuerySchemaRegistry.clearRegistry(); - SystemConfiguration.setProperty("data.schemas", "none"); - SystemConfiguration.setProperty("query.schemas", "none"); - } - - @Test - public void testSingleQuery() - { - String query1 = "?q=src_ip:55.55.55.55"; - assertTrue(QueryParserUtils.checkRecord(query1, doc, dSchema)); - assertTrue(QueryParserUtils.checkRecordWritableArrayWritable(query1, docWAW, dSchema)); - assertTrue(QueryParserUtils.checkRecord(query1, docMap, dSchema)); - - String query2 = "?q=qname:a.b.c.com"; - assertTrue(QueryParserUtils.checkRecord(query2, doc, dSchema)); - assertTrue(QueryParserUtils.checkRecordWritableArrayWritable(query2, docWAW, dSchema)); - assertTrue(QueryParserUtils.checkRecord(query2, docMap, dSchema)); - - String query3 = "?q=qname:d.b.c.com"; - assertFalse(QueryParserUtils.checkRecord(query3, doc, dSchema)); - assertFalse(QueryParserUtils.checkRecordWritableArrayWritable(query3, docWAW, dSchema)); - assertFalse(QueryParserUtils.checkRecord(query3, docMap, dSchema)); - } - - @Test - public void testQueryFieldDoesNotExist() - { - logger.info("running testQueryFieldDoesNotExist"); - - // Field does not exist, this should not be found - String query = "?q=nonexistent-field:*check*"; - assertFalse(QueryParserUtils.checkRecordWritableArrayWritable(query, docWAW, dSchema)); - assertFalse(QueryParserUtils.checkRecord(query, doc, dSchema)); - assertFalse(QueryParserUtils.checkRecord(query, docMap, dSchema)); - - // First field does not exist, but second should be found - String query2 = "?q=nonexistent-field:*check*+OR+qname:*a.b.c.com*"; - assertTrue(QueryParserUtils.checkRecordWritableArrayWritable(query2, docWAW, dSchema)); - assertTrue(QueryParserUtils.checkRecord(query2, doc, dSchema)); - assertTrue(QueryParserUtils.checkRecord(query2, docMap, dSchema)); - - // First field does not exist, second field does, but AND operator makes query false - String query3 = "?q=nonexistent-field:*check*+AND+qname:*a.b.c.com*"; - assertFalse(QueryParserUtils.checkRecordWritableArrayWritable(query3, docWAW, dSchema)); - assertFalse(QueryParserUtils.checkRecord(query3, doc, dSchema)); - assertFalse(QueryParserUtils.checkRecord(query3, docMap, dSchema)); - - logger.info("completed testQueryFieldDoesNotExist"); - } - - @Test - public void testIgnoreCase() - { - logger.info("running testIgnoreCase"); - - // with case sensitivity, should NOT be found - String query = "?q=qname:*A.b.c.com*"; - assertFalse(QueryParserUtils.checkRecord(query, doc, dSchema)); - assertFalse(QueryParserUtils.checkRecordWritableArrayWritable(query, docWAW, dSchema)); - assertFalse(QueryParserUtils.checkRecord(query, docMap, dSchema)); - - // with case sensitivity, should be found - String query2 = "?q=qname:*a.b.c.com*"; - assertTrue(QueryParserUtils.checkRecord(query2, doc, dSchema)); - assertTrue(QueryParserUtils.checkRecordWritableArrayWritable(query2, docWAW, dSchema)); - assertTrue(QueryParserUtils.checkRecord(query2, docMap, dSchema)); - - // adds @ flag = case insensitivity, thus should be found - String query3 = "?q=qname@:*A.b.c.com*"; - assertTrue(QueryParserUtils.checkRecord(query3, doc, dSchema)); - assertTrue(QueryParserUtils.checkRecordWritableArrayWritable(query3, docWAW, dSchema)); - assertTrue(QueryParserUtils.checkRecord(query3, docMap, dSchema)); - - logger.info("completed testIgnoreCase"); - } - - @Test - public void testSingleValueRangeQuery() - { - testSingleValueRangeQueryMapWritable(); - testSingleValueRangeQueryMap(); - testSingleValueRangeQueryMapWritableWAW(); - } - - private void testSingleValueRangeQueryMapWritable() - { - assertTrue(QueryParserUtils.checkRecord("?q=rcode:[0+TO+2]", doc, dSchema)); - assertTrue(QueryParserUtils.checkRecord("?q=rcode:{-1+TO+2}", doc, dSchema)); - assertTrue(QueryParserUtils.checkRecord("?q=rcode:[-1+TO+0]", doc, dSchema)); - assertFalse(QueryParserUtils.checkRecord("?q=rcode:{0+TO+3}", doc, dSchema)); - assertFalse(QueryParserUtils.checkRecord("?q=rcode:[3+TO+10]", doc, dSchema)); - } - - private void testSingleValueRangeQueryMap() - { - assertTrue(QueryParserUtils.checkRecord("?q=rcode:[0+TO+2]", docMap, dSchema)); - assertTrue(QueryParserUtils.checkRecord("?q=rcode:{-1+TO+2}", docMap, dSchema)); - assertTrue(QueryParserUtils.checkRecord("?q=rcode:[-1+TO+0]", docMap, dSchema)); - assertFalse(QueryParserUtils.checkRecord("?q=rcode:{0+TO+3}", docMap, dSchema)); - assertFalse(QueryParserUtils.checkRecord("?q=rcode:[3+TO+10]", docMap, dSchema)); - } - - private void testSingleValueRangeQueryMapWritableWAW() - { - assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=rcode:[0+TO+2]", docWAW, dSchema)); - assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=rcode:{-1+TO+2}", docWAW, dSchema)); - assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=rcode:[-1+TO+0]", docWAW, dSchema)); - assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=rcode:{0+TO+3}", docWAW, dSchema)); - assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=rcode:[3+TO+10]", docWAW, dSchema)); - } - - @Test - public void testIPRangeQuery() - { - testIPRangeQueryMapWritable(); - testIPRangeQueryMap(); - testIPRangeQueryMapWritableWAW(); - } - - public void testIPRangeQueryMapWritable() - { - // src_ip: 55.55.55.55 - // ip: 10.20.30.40,10.20.30.60 - assertTrue(QueryParserUtils.checkRecord("?q=src_ip:[55.55.55.0+TO+173.248.255.255]", doc, dSchema)); - assertTrue(QueryParserUtils.checkRecord("?q=src_ip:[55.55.55.0+TO+55.55.55.100]", doc, dSchema)); - assertTrue(QueryParserUtils.checkRecord("?q=src_ip:[55.55.55.2+TO+55.55.55.55]", doc, dSchema)); - assertTrue(QueryParserUtils.checkRecord("?q=src_ip:{55.55.55.2+TO+55.55.55.57}", doc, dSchema)); - - assertFalse(QueryParserUtils.checkRecord("?q=src_ip:{173.248.188.0+TO+173.248.188.10}", doc, dSchema)); - assertFalse(QueryParserUtils.checkRecord("?q=src_ip:{55.55.55.2+TO+55.55.55.55}", doc, dSchema)); - - assertTrue(QueryParserUtils.checkRecord("?q=ip:[10.20.30.50+TO+10.20.30.69]", doc, dSchema)); - } - - public void testIPRangeQueryMapWritableWAW() - { - // src_ip: 55.55.55.55 - // ip: 10.20.30.40,10.20.30.60 - assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=src_ip:[55.55.55.0+TO+173.248.255.255]", docWAW, dSchema)); - assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=src_ip:[55.55.55.0+TO+55.55.55.100]", docWAW, dSchema)); - assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=src_ip:[55.55.55.2+TO+55.55.55.55]", docWAW, dSchema)); - assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=src_ip:{55.55.55.2+TO+55.55.55.57}", docWAW, dSchema)); - - assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=src_ip:{173.248.188.0+TO+173.248.188.10}", docWAW, dSchema)); - assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=src_ip:{55.55.55.2+TO+55.55.55.55}", docWAW, dSchema)); - - assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=ip:[10.20.30.50+TO+10.20.30.69]", docWAW, dSchema)); - } - - public void testIPRangeQueryMap() - { - // src_ip: 55.55.55.55 - // ip: 10.20.30.40,10.20.30.60 - assertTrue(QueryParserUtils.checkRecord("?q=src_ip:[55.55.55.0+TO+173.248.255.255]", docMap, dSchema)); - assertTrue(QueryParserUtils.checkRecord("?q=src_ip:[55.55.55.0+TO+55.55.55.100]", docMap, dSchema)); - assertTrue(QueryParserUtils.checkRecord("?q=src_ip:[55.55.55.2+TO+55.55.55.55]", docMap, dSchema)); - assertTrue(QueryParserUtils.checkRecord("?q=src_ip:{55.55.55.2+TO+55.55.55.57}", docMap, dSchema)); - - assertFalse(QueryParserUtils.checkRecord("?q=src_ip:{173.248.188.0+TO+173.248.188.10}", docMap, dSchema)); - assertFalse(QueryParserUtils.checkRecord("?q=src_ip:{55.55.55.2+TO+55.55.55.55}", docMap, dSchema)); - - assertTrue(QueryParserUtils.checkRecord("?q=ip:[10.20.30.50+TO+10.20.30.69]", docMap, dSchema)); - } - - @Test - public void testDateRangeQuery() - { - testDateRangeQueryMapWritable(); - testDateRangeQueryMapWritableWAW(); - testDateRangeQueryMap(); - } - - private void testDateRangeQueryMapWritable() - { - // date: 2016-02-20T23:29:05.000Z - - assertTrue(QueryParserUtils.checkRecord("?q=date:[2014-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", doc, dSchema)); - assertTrue(QueryParserUtils.checkRecord("?q=date:[2015-05-05T20:33:07.000Z+TO+2016-04-20T23:29:05.000Z]", doc, dSchema)); - assertTrue(QueryParserUtils.checkRecord("?q=date:[2016-02-20T23:29:05.000Z+TO+2017-02-20T23:29:05.000Z]", doc, dSchema)); - assertTrue(QueryParserUtils.checkRecord("?q=date:{2015-06-05T20:33:07.000Z+TO+2016-02-20T23:30:05.000Z}", doc, dSchema)); - - assertFalse(QueryParserUtils.checkRecord("?q=date:[2013-05-05T20:33:07.000Z+TO+2014-07-05T20:33:07.000Z]", doc, dSchema)); - assertFalse(QueryParserUtils.checkRecord("?q=date:{2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z}", doc, dSchema)); - assertFalse(QueryParserUtils.checkRecord("?q=date:{2015-06-05T20:33:07.000Z+TO+2015-07-05T20:33:07.000Z}", doc, dSchema)); - } - - private void testDateRangeQueryMap() - { - // date: 2016-02-20T23:29:05.000Z - - assertTrue(QueryParserUtils.checkRecord("?q=date:[2014-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", docMap, dSchema)); - assertTrue(QueryParserUtils.checkRecord("?q=date:[2015-05-05T20:33:07.000Z+TO+2016-04-20T23:29:05.000Z]", docMap, dSchema)); - assertTrue(QueryParserUtils.checkRecord("?q=date:[2016-02-20T23:29:05.000Z+TO+2017-02-20T23:29:05.000Z]", docMap, dSchema)); - assertTrue(QueryParserUtils.checkRecord("?q=date:{2015-06-05T20:33:07.000Z+TO+2016-02-20T23:30:05.000Z}", docMap, dSchema)); - - assertFalse(QueryParserUtils.checkRecord("?q=date:[2013-05-05T20:33:07.000Z+TO+2014-07-05T20:33:07.000Z]", docMap, dSchema)); - assertFalse(QueryParserUtils.checkRecord("?q=date:{2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z}", docMap, dSchema)); - assertFalse(QueryParserUtils.checkRecord("?q=date:{2015-06-05T20:33:07.000Z+TO+2015-07-05T20:33:07.000Z}", docMap, dSchema)); - } - - private void testDateRangeQueryMapWritableWAW() - { - // date: 2016-02-20T23:29:05.000Z - - assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=date:[2014-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", docWAW, dSchema)); - assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=date:[2015-05-05T20:33:07.000Z+TO+2016-04-20T23:29:05.000Z]", docWAW, dSchema)); - assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=date:[2016-02-20T23:29:05.000Z+TO+2017-02-20T23:29:05.000Z]", docWAW, dSchema)); - assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=date:{2015-06-05T20:33:07.000Z+TO+2016-02-20T23:30:05.000Z}", docWAW, dSchema)); - - assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=date:[2013-05-05T20:33:07.000Z+TO+2014-07-05T20:33:07.000Z]", docWAW, dSchema)); - assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=date:{2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z}", docWAW, dSchema)); - assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=date:{2015-06-05T20:33:07.000Z+TO+2015-07-05T20:33:07.000Z}", docWAW, dSchema)); - } - - @Test - public void testBooleanQuery() - { - testBooleanQueryMapWritable(); - testBooleanQueryMapMapWritableWAW(); - testBooleanQueryMap(); - } - - private void testBooleanQueryMapWritable() - { - assertTrue(QueryParserUtils.checkRecord("?q=qtype:1+AND+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", doc, dSchema)); - assertTrue(QueryParserUtils.checkRecord("?q=qtype:5+OR+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", doc, dSchema)); - - assertTrue(QueryParserUtils.checkRecord("?q=qtype:1+AND+rcode:0+AND+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", doc, dSchema)); - - assertTrue(QueryParserUtils.checkRecord("?q=qtype:1+AND+rcode:0+OR+date:[2013-05-05T20:33:07.000Z+TO+2014-07-05T20:33:07.000Z]", doc, dSchema)); - assertTrue(QueryParserUtils.checkRecord("?q=qtype:1+AND+rcode:1+OR+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", doc, dSchema)); - - assertTrue(QueryParserUtils.checkRecord("?q=qtype:5+OR+qtype:2+OR+rcode:0", doc, dSchema)); - } - - private void testBooleanQueryMap() - { - assertTrue(QueryParserUtils.checkRecord("?q=qtype:1+AND+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", docMap, dSchema)); - assertTrue(QueryParserUtils.checkRecord("?q=qtype:5+OR+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", docMap, dSchema)); - - assertTrue(QueryParserUtils.checkRecord("?q=qtype:1+AND+rcode:0+AND+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", docMap, dSchema)); - - assertTrue(QueryParserUtils.checkRecord("?q=qtype:1+AND+rcode:0+OR+date:[2013-05-05T20:33:07.000Z+TO+2014-07-05T20:33:07.000Z]", docMap, dSchema)); - assertTrue(QueryParserUtils.checkRecord("?q=qtype:1+AND+rcode:1+OR+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", docMap, dSchema)); - - assertTrue(QueryParserUtils.checkRecord("?q=qtype:5+OR+qtype:2+OR+rcode:0", docMap, dSchema)); - } - - private void testBooleanQueryMapMapWritableWAW() - { - assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qtype:1+AND+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", docWAW, dSchema)); - assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qtype:5+OR+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", docWAW, dSchema)); - - assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qtype:1+AND+rcode:0+AND+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", - docWAW, dSchema)); - - assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qtype:1+AND+rcode:0+OR+date:[2013-05-05T20:33:07.000Z+TO+2014-07-05T20:33:07.000Z]", - docWAW, dSchema)); - assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qtype:1+AND+rcode:1+OR+date:[2015-05-05T20:33:07.000Z+TO+2016-02-20T23:29:05.000Z]", - docWAW, dSchema)); - - assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qtype:5+OR+qtype:2+OR+rcode:0", docWAW, dSchema)); - } - - @Test - public void testAllQuery() - { - assertTrue(QueryParserUtils.checkRecord("?q=*", doc, dSchema)); - assertTrue(QueryParserUtils.checkRecord("?q=*", docMap, dSchema)); - assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=*", docWAW, dSchema)); - } - - @Test - public void testWildcardQuery() - { - testWildcardQueryMapWritable(); - testWildcardQueryMap(); - testWildcardQueryMapWritableWAW(); - } - - private void testWildcardQueryMapWritable() - { - assertTrue(QueryParserUtils.checkRecord("?q=qname:*.com", doc, dSchema)); - assertTrue(QueryParserUtils.checkRecord("?q=qname:a.b.c.c*m", doc, dSchema)); - assertTrue(QueryParserUtils.checkRecord("?q=qname:a.b*", doc, dSchema)); - - assertFalse(QueryParserUtils.checkRecord("?q=qname:*.org", doc, dSchema)); - assertFalse(QueryParserUtils.checkRecord("?q=qname:mrtf*", doc, dSchema)); - assertFalse(QueryParserUtils.checkRecord("?q=qname:nedeljnik*.uk", doc, dSchema)); - - assertTrue(QueryParserUtils.checkRecord("?q=qname:a.b.c.c?m", doc, dSchema)); - assertTrue(QueryParserUtils.checkRecord("?q=qname:a.b.?.com", doc, dSchema)); - assertTrue(QueryParserUtils.checkRecord("?q=qname:?.b.c.com", doc, dSchema)); - - assertFalse(QueryParserUtils.checkRecord("?q=qname:medelj?ikafera.com", doc, dSchema)); - assertFalse(QueryParserUtils.checkRecord("?q=qname:nedeljntkafer?.com", doc, dSchema)); - assertFalse(QueryParserUtils.checkRecord("?q=qname:?edeljnikrfera.com", doc, dSchema)); - } - - private void testWildcardQueryMap() - { - assertTrue(QueryParserUtils.checkRecord("?q=qname:*.com", docMap, dSchema)); - assertTrue(QueryParserUtils.checkRecord("?q=qname:a.b.c.c*m", docMap, dSchema)); - assertTrue(QueryParserUtils.checkRecord("?q=qname:a.b*", docMap, dSchema)); - - assertFalse(QueryParserUtils.checkRecord("?q=qname:*.org", docMap, dSchema)); - assertFalse(QueryParserUtils.checkRecord("?q=qname:mrtf*", docMap, dSchema)); - assertFalse(QueryParserUtils.checkRecord("?q=qname:nedeljnik*.uk", docMap, dSchema)); - - assertTrue(QueryParserUtils.checkRecord("?q=qname:a.b.c.c?m", docMap, dSchema)); - assertTrue(QueryParserUtils.checkRecord("?q=qname:a.b.?.com", docMap, dSchema)); - assertTrue(QueryParserUtils.checkRecord("?q=qname:?.b.c.com", docMap, dSchema)); - - assertFalse(QueryParserUtils.checkRecord("?q=qname:medelj?ikafera.com", docMap, dSchema)); - assertFalse(QueryParserUtils.checkRecord("?q=qname:nedeljntkafer?.com", docMap, dSchema)); - assertFalse(QueryParserUtils.checkRecord("?q=qname:?edeljnikrfera.com", docMap, dSchema)); - } - - private void testWildcardQueryMapWritableWAW() - { - assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:*.com", docWAW, dSchema)); - assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:a.b.c.c*m", docWAW, dSchema)); - assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:a.b*", docWAW, dSchema)); - - assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:*.org", docWAW, dSchema)); - assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:mrtf*", docWAW, dSchema)); - assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:nedeljnik*.uk", docWAW, dSchema)); - - assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:a.b.c.c?m", docWAW, dSchema)); - assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:a.b.?.com", docWAW, dSchema)); - assertTrue(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:?.b.c.com", docWAW, dSchema)); - - assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:medelj?ikafera.com", docWAW, dSchema)); - assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:nedeljntkafer?.com", docWAW, dSchema)); - assertFalse(QueryParserUtils.checkRecordWritableArrayWritable("?q=qname:?edeljnikrfera.com", docWAW, dSchema)); - - } -} http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/9244df72/src/test/java/org/apache/pirk/test/schema/data/LoadDataSchemaTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/pirk/test/schema/data/LoadDataSchemaTest.java b/src/test/java/org/apache/pirk/test/schema/data/LoadDataSchemaTest.java deleted file mode 100644 index 9f52af3..0000000 --- a/src/test/java/org/apache/pirk/test/schema/data/LoadDataSchemaTest.java +++ /dev/null @@ -1,327 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pirk.test.schema.data; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.io.File; -import java.io.IOException; - -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.transform.Transformer; -import javax.xml.transform.TransformerFactory; -import javax.xml.transform.dom.DOMSource; -import javax.xml.transform.stream.StreamResult; - -import org.apache.pirk.schema.data.DataSchema; -import org.apache.pirk.schema.data.DataSchemaLoader; -import org.apache.pirk.schema.data.DataSchemaRegistry; -import org.apache.pirk.schema.data.partitioner.IPDataPartitioner; -import org.apache.pirk.schema.data.partitioner.PrimitiveTypePartitioner; -import org.apache.pirk.test.utils.TestUtils; -import org.apache.pirk.utils.SystemConfiguration; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.w3c.dom.Document; -import org.w3c.dom.Element; - -/** - * Test suite for LoadDataSchema and DataSchema - */ -public class LoadDataSchemaTest -{ - private static final Logger logger = LoggerFactory.getLogger(LoadDataSchemaTest.class); - - private String dataSchemaName = "fakeDataSchema"; - - private String element1 = "elementName1"; - private String element2 = "elementName2"; - private String element3 = "elementName3"; - - @Test - public void testGeneralSchemaLoad() throws Exception - { - // Pull off the property and reset upon completion - String schemasProp = SystemConfiguration.getProperty("data.schemas", "none"); - - // Write the schema file - try - { - createDataSchema("schemaFile"); - } catch (IOException e) - { - e.printStackTrace(); - fail(e.toString()); - } - - // Force the schema to load - DataSchemaLoader.initialize(); - - // Check the entries - DataSchema dSchema = DataSchemaRegistry.get(dataSchemaName); - - assertEquals(dataSchemaName, dSchema.getSchemaName()); - - assertEquals(3, dSchema.getElementNames().size()); - - // TODO: check Hadoop text names - - assertEquals(PrimitiveTypePartitioner.STRING, dSchema.getElementType(element1)); - assertEquals(PrimitiveTypePartitioner.INT, dSchema.getElementType(element2)); - assertEquals(PrimitiveTypePartitioner.STRING, dSchema.getElementType(element3)); - - assertEquals(PrimitiveTypePartitioner.class.getName(), dSchema.getPartitionerTypeName(element1)); - if (!(dSchema.getPartitionerForElement(element1) instanceof PrimitiveTypePartitioner)) - { - fail("Partitioner instance for element1 must be PrimitiveTypePartitioner"); - } - assertEquals(IPDataPartitioner.class.getName(), dSchema.getPartitionerTypeName(element3)); - if (!(dSchema.getPartitionerForElement(element3) instanceof IPDataPartitioner)) - { - fail("Partitioner instance for element3 must be IPDataPartitioner"); - } - - assertEquals(2, dSchema.getArrayElements().size()); - assertTrue(dSchema.getArrayElements().contains(element2)); - assertTrue(dSchema.getArrayElements().contains(element3)); - - assertEquals(1, dSchema.getNonArrayElements().size()); - assertTrue(dSchema.getNonArrayElements().contains(element1)); - - // Reset original data.schemas property - SystemConfiguration.setProperty("data.schemas", schemasProp); - - // Force the schema to load - if (!schemasProp.equals("none")) - { - DataSchemaLoader.initialize(); - } - } - - @Test - public void testIncorrectJavaType() throws Exception - { - // Pull off the property and reset upon completion - String schemasProp = SystemConfiguration.getProperty("data.schemas"); - - // Write the schema file - try - { - createDataSchemaIncorrectJavaType("wrongJavaType"); - } catch (IOException e) - { - e.printStackTrace(); - fail(e.toString()); - } - - try - { - // Force the schema to load - DataSchemaLoader.initialize(); - fail("DataSchemaLoader did not throw exception for incorrect javaType"); - } catch (Exception ignore) - {} - - // Reset original data.schemas property - SystemConfiguration.setProperty("data.schemas", schemasProp); - - // Force the schema to load - DataSchemaLoader.initialize(); - } - - @Test - public void testUnknownPartitioner() throws Exception - { - // Pull off the property and reset upon completion - String schemasProp = SystemConfiguration.getProperty("data.schemas"); - - // Write the schema file - try - { - createDataSchemaUnknownPartitioner("unknownPartitioner"); - } catch (IOException e) - { - e.printStackTrace(); - fail(e.toString()); - } - - try - { - // Force the schema to load - DataSchemaLoader.initialize(); - fail("DataSchemaLoader did not throw exception for unknown partitioner"); - } catch (Exception ignore) - {} - - // Reset original data.schemas property - SystemConfiguration.setProperty("data.schemas", schemasProp); - - // Force the schema to load - DataSchemaLoader.initialize(); - } - - // Create the file that contains an unknown partitioner - private void createDataSchemaUnknownPartitioner(String schemaFile) throws IOException - { - // Create a temporary file for the test schema, set in the properties - File file = File.createTempFile(schemaFile, ".xml"); - file.deleteOnExit(); - logger.info("file = " + file.toString()); - SystemConfiguration.setProperty("data.schemas", file.toString()); - - // Write to the file - try - { - DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance(); - DocumentBuilder dBuilder = dbFactory.newDocumentBuilder(); - Document doc = dBuilder.newDocument(); - - // root element - Element rootElement = doc.createElement("schema"); - doc.appendChild(rootElement); - - // Add the schemaName - Element schemaNameElement = doc.createElement("schemaName"); - schemaNameElement.appendChild(doc.createTextNode(dataSchemaName)); - rootElement.appendChild(schemaNameElement); - - // Add the element - unknown partitioner - TestUtils.addElement(doc, rootElement, element1, PrimitiveTypePartitioner.INT, "false", "fakePartitioner"); - - // Write to a xml file - TransformerFactory transformerFactory = TransformerFactory.newInstance(); - Transformer transformer = transformerFactory.newTransformer(); - DOMSource source = new DOMSource(doc); - StreamResult result = new StreamResult(file); - transformer.transform(source, result); - - // Output for testing - StreamResult consoleResult = new StreamResult(System.out); - transformer.transform(source, consoleResult); - System.out.println(); - } catch (Exception e) - { - e.printStackTrace(); - } - } - - // Create the test data schema file - private void createDataSchema(String schemaFile) throws IOException - { - // Create a temporary file for the test schema, set in the properties - File file = File.createTempFile(schemaFile, ".xml"); - file.deleteOnExit(); - logger.info("file = " + file.toString()); - SystemConfiguration.setProperty("data.schemas", file.toString()); - - // Write to the file - try - { - DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance(); - DocumentBuilder dBuilder = dbFactory.newDocumentBuilder(); - Document doc = dBuilder.newDocument(); - - // root element - Element rootElement = doc.createElement("schema"); - doc.appendChild(rootElement); - - // Add the schemaName - Element schemaNameElement = doc.createElement("schemaName"); - schemaNameElement.appendChild(doc.createTextNode(dataSchemaName)); - rootElement.appendChild(schemaNameElement); - - // Add the elements - // element1 -- single String - // TestUtils.addElement(doc, rootElement, element1, PrimitiveTypePartitioner.STRING, "false", PrimitiveTypePartitioner.class.getName()); - TestUtils.addElement(doc, rootElement, element1, PrimitiveTypePartitioner.STRING, "false", null); - - // element2 - -- array of Integers - TestUtils.addElement(doc, rootElement, element2, PrimitiveTypePartitioner.INT, "true", PrimitiveTypePartitioner.class.getName()); - - // element3 -- array of IP addresses - TestUtils.addElement(doc, rootElement, element3, PrimitiveTypePartitioner.STRING, "true", IPDataPartitioner.class.getName()); - - // Write to a xml file - TransformerFactory transformerFactory = TransformerFactory.newInstance(); - Transformer transformer = transformerFactory.newTransformer(); - DOMSource source = new DOMSource(doc); - StreamResult result = new StreamResult(file); - transformer.transform(source, result); - - // Output for testing - StreamResult consoleResult = new StreamResult(System.out); - transformer.transform(source, consoleResult); - System.out.println(); - - } catch (Exception e) - { - e.printStackTrace(); - } - } - - // Create the test schema file - private void createDataSchemaIncorrectJavaType(String schemaFile) throws IOException - { - // Create a temporary file for the test schema, set in the properties - File file = File.createTempFile(schemaFile, ".xml"); - file.deleteOnExit(); - logger.info("file = " + file.toString()); - SystemConfiguration.setProperty("data.schemas", file.toString()); - - // Write to the file - try - { - DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance(); - DocumentBuilder dBuilder = dbFactory.newDocumentBuilder(); - Document doc = dBuilder.newDocument(); - - // root element - Element rootElement = doc.createElement("schema"); - doc.appendChild(rootElement); - - // Add the schemaName - Element schemaNameElement = doc.createElement("schemaName"); - schemaNameElement.appendChild(doc.createTextNode(dataSchemaName)); - rootElement.appendChild(schemaNameElement); - - // Add the element - unknown Java type - TestUtils.addElement(doc, rootElement, element1, "bogus", "false", PrimitiveTypePartitioner.class.getName()); - - // Write to a xml file - TransformerFactory transformerFactory = TransformerFactory.newInstance(); - Transformer transformer = transformerFactory.newTransformer(); - DOMSource source = new DOMSource(doc); - StreamResult result = new StreamResult(file); - transformer.transform(source, result); - - // Output for testing - StreamResult consoleResult = new StreamResult(System.out); - transformer.transform(source, consoleResult); - System.out.println(); - - } catch (Exception e) - { - e.printStackTrace(); - } - } -}
