Repository: incubator-pirk Updated Branches: refs/heads/master 2d586b68f -> c12d47f1d
http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/src/main/java/org/apache/pirk/schema/data/DataSchemaLoader.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/schema/data/DataSchemaLoader.java b/src/main/java/org/apache/pirk/schema/data/DataSchemaLoader.java new file mode 100644 index 0000000..1a60d42 --- /dev/null +++ b/src/main/java/org/apache/pirk/schema/data/DataSchemaLoader.java @@ -0,0 +1,256 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pirk.schema.data; + +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.HashSet; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.Text; +import org.apache.pirk.schema.data.partitioner.DataPartitioner; +import org.apache.pirk.schema.data.partitioner.PrimitiveTypePartitioner; +import org.apache.pirk.utils.PIRException; +import org.apache.pirk.utils.SystemConfiguration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.SAXException; + +/** + * Class to load any data schemas specified in the properties file, 'data.schemas' + * <p> + * Schemas should be specified as follows; all items are treated in a case insensitive manner: + * + * <pre> + * {@code + * <schema> + * <schemaName> name of the schema </schemaName> + * <element> + * <name> element name /name> + * <type> class name or type name (if Java primitive type) of the element </type> + * <isArray> true or false -- whether or not the schema element is an array within the data </isArray> + * <partitioner> optional - Partitioner class for the element; defaults to primitive java type partitioner </partitioner> + * </element> + * </schema> + * } + * </pre> + * + * Primitive types must be one of the following: "byte", "short", "int", "long", "float", "double", "char", "string" + * <p> + */ +public class DataSchemaLoader +{ + private static final Logger logger = LoggerFactory.getLogger(DataSchemaLoader.class); + + private static HashSet<String> allowedPrimitiveJavaTypes = new HashSet<>(Arrays.asList(PrimitiveTypePartitioner.BYTE, PrimitiveTypePartitioner.SHORT, + PrimitiveTypePartitioner.INT, PrimitiveTypePartitioner.LONG, PrimitiveTypePartitioner.FLOAT, PrimitiveTypePartitioner.DOUBLE, + PrimitiveTypePartitioner.CHAR, PrimitiveTypePartitioner.STRING)); + + static + { + logger.info("Loading pre-configured data schemas: "); + + try + { + initialize(); + } catch (Exception e) + { + logger.error("Caught exception: "); + e.printStackTrace(); + } + } + + /** Kept for compatibility */ + public static void initialize() throws Exception + { + initialize(false, null); + } + + /** Kept for compatibility */ + public static void initialize(boolean hdfs, FileSystem fs) throws Exception + { + String dataSchemas = SystemConfiguration.getProperty("data.schemas", "none"); + if (!dataSchemas.equals("none")) + { + String[] dataSchemaFiles = dataSchemas.split(","); + for (String schemaFile : dataSchemaFiles) + { + logger.info("Loading schemaFile = " + schemaFile + " hdfs = " + hdfs); + + // Parse and load the schema file into a DataSchema object; place in the schemaMap + DataSchemaLoader loader = new DataSchemaLoader(); + InputStream is; + if (hdfs) + { + is = fs.open(new Path(schemaFile)); + logger.info("hdfs: filePath = " + schemaFile.toString()); + } + else + { + is = new FileInputStream(schemaFile); + logger.info("localFS: inputFile = " + schemaFile.toString()); + } + + try + { + DataSchema dataSchema = loader.loadSchemaFile(is); + DataSchemaRegistry.put(dataSchema); + } finally + { + is.close(); + } + } + } + } + + /* + * Default constructor + */ + public DataSchemaLoader() + {} + + public DataSchema loadSchemaFile(InputStream stream) throws IOException, PIRException + { + // Read in and parse the XML schema file + Document doc; + try + { + DocumentBuilder dBuilder = DocumentBuilderFactory.newInstance().newDocumentBuilder(); + doc = dBuilder.parse(stream); + } catch (ParserConfigurationException | SAXException e) + { + throw new PIRException("Data schema parsing error", e); + } + doc.getDocumentElement().normalize(); + logger.info("Root element: " + doc.getDocumentElement().getNodeName()); + + // Extract the schemaName + NodeList schemaNameList = doc.getElementsByTagName("schemaName"); + if (schemaNameList.getLength() != 1) + { + throw new PIRException("schemaNameList.getLength() = " + schemaNameList.getLength() + " -- should be one schema per xml file"); + } + String schemaName = schemaNameList.item(0).getTextContent().trim(); + logger.info("schemaName = " + schemaName); + + // Create the data schema holder + DataSchema dataSchema = new DataSchema(schemaName); + + // Extract and populate the elements + NodeList nList = doc.getElementsByTagName("element"); + for (int i = 0; i < nList.getLength(); i++) + { + Node nNode = nList.item(i); + if (nNode.getNodeType() == Node.ELEMENT_NODE) + { + parseElementNode((Element) nNode, dataSchema); + } + } + + return dataSchema; + } + + private void parseElementNode(Element eElement, DataSchema schema) throws PIRException + { + // Pull out the element name and type attributes. + String name = eElement.getElementsByTagName("name").item(0).getTextContent().trim(); + schema.getTextRep().put(name, new Text(name)); + + String type = eElement.getElementsByTagName("type").item(0).getTextContent().trim(); + schema.getTypeMap().put(name, type); + + // An absent isArray means false, and an empty isArray means true, otherwise take the value. + Node isArrayNode = eElement.getElementsByTagName("isArray").item(0); + if (isArrayNode != null) + { + String isArrayValue = isArrayNode.getTextContent().trim().toLowerCase(); + String isArray = isArrayValue.isEmpty() ? "true" : isArrayValue; + if (isArray.equals("true")) + { + schema.getArrayElements().add(name); + } + } + + // Pull and check the partitioner class -- if the partitioner tag doesn't exist, then + // it defaults to the PrimitiveTypePartitioner + String partitionerTypeName = PrimitiveTypePartitioner.class.getName(); + boolean isPrimitivePartitioner = true; + if (eElement.getElementsByTagName("partitioner").item(0) != null) + { + partitionerTypeName = eElement.getElementsByTagName("partitioner").item(0).getTextContent().trim(); + isPrimitivePartitioner = partitionerTypeName.equals(PrimitiveTypePartitioner.class.getName()); + } + + DataPartitioner partitioner; + if (isPrimitivePartitioner) + { + // Validate the primitive partitioner can only be used for primitive element types. + validateIsPrimitiveType(type); + partitioner = new PrimitiveTypePartitioner(); + } + else + { + partitioner = instantiatePartitioner(partitionerTypeName); + } + + // Place in the appropriate structures + schema.getPartitionerTypeMap().put(name, partitionerTypeName); + schema.getPartitionerInstances().put(partitionerTypeName, partitioner); + + logger.info("name = " + name + " javaType = " + type + " isArray = " + schema.getArrayElements().contains(name) + " partitioner " + partitionerTypeName); + } + + /* + * Checks the given type name is a supported Java primitive type, and throws a PIRException if not. + */ + void validateIsPrimitiveType(String typeName) throws PIRException + { + if (!allowedPrimitiveJavaTypes.contains(typeName.toLowerCase())) + { + throw new PIRException("javaType = " + typeName + " is not one of the allowed javaTypes: " + " byte, short, int, long, float, double, char, string"); + } + } + + DataPartitioner instantiatePartitioner(String partitionerTypeName) throws PIRException + { + Object obj; + try + { + @SuppressWarnings("unchecked") + Class<? extends DataPartitioner> c = (Class<? extends DataPartitioner>) Class.forName(partitionerTypeName); + obj = c.newInstance(); + } catch (ClassNotFoundException | InstantiationException | IllegalAccessException | ClassCastException e) + { + throw new PIRException("partitioner = " + partitionerTypeName + " cannot be instantiated or does not implement DataParitioner.", e); + } + + return (DataPartitioner) obj; + } +} http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/src/main/java/org/apache/pirk/schema/data/DataSchemaRegistry.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/schema/data/DataSchemaRegistry.java b/src/main/java/org/apache/pirk/schema/data/DataSchemaRegistry.java new file mode 100644 index 0000000..2cd6673 --- /dev/null +++ b/src/main/java/org/apache/pirk/schema/data/DataSchemaRegistry.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pirk.schema.data; + +import java.util.HashMap; +import java.util.Map; + +/** + * The data schema registry is a global location for data schema descriptors. + * + * @see DataSchema + * @see DataSchemaLoader + */ +public class DataSchemaRegistry +{ + // The registry. Maps schema name to data schema. + private static final Map<String,DataSchema> registry = new HashMap<>(); + + // Not designed to be instantiated. + DataSchemaRegistry() + { + + } + + /** + * Adds the given data schema to the registry. + * + * If there was an existing schema with the same name, it is replaced. + * + * @param schema + * The data schema to add. + * @return the previous schema registered at the same name, or <code>null</code> if there were none. + */ + public static DataSchema put(DataSchema schema) + { + return registry.put(schema.getSchemaName(), schema); + } + + /** + * Returns the data schema with the given name. + * + * @param schemaName + * The data schema name to be returned. + * @return The data schema, or <code>null</code> if no such schema. + */ + public static DataSchema get(String schemaName) + { + return registry.get(schemaName); + } +} http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/src/main/java/org/apache/pirk/schema/data/LoadDataSchemas.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/schema/data/LoadDataSchemas.java b/src/main/java/org/apache/pirk/schema/data/LoadDataSchemas.java deleted file mode 100644 index 60eaa66..0000000 --- a/src/main/java/org/apache/pirk/schema/data/LoadDataSchemas.java +++ /dev/null @@ -1,247 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.pirk.schema.data; - -import java.io.File; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Set; - -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; - -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.Text; -import org.apache.pirk.schema.data.partitioner.DataPartitioner; -import org.apache.pirk.schema.data.partitioner.PrimitiveTypePartitioner; -import org.apache.pirk.utils.SystemConfiguration; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; - -/** - * Class to load any data schemas specified in the properties file, 'data.schemas' - * <p> - * Schemas should be specified as follows; all items are treated in a case insensitive manner: - * - * <pre> - * {@code - * <schema> - * <schemaName> name of the schema </schemaName> - * <element> - * <name> element name /name> - * <type> class name or type name (if Java primitive type) of the element </type> - * <isArray> true or false -- whether or not the schema element is an array within the data </isArray> - * <partitioner> optional - Partitioner class for the element; defaults to primitive java type partitioner </partitioner> - * </element> - * </schema> - * } - * </pre> - * - * Primitive types must be one of the following: "byte", "short", "int", "long", "float", "double", "char", "string" - * <p> - */ -public class LoadDataSchemas -{ - private static final Logger logger = LoggerFactory.getLogger(LoadDataSchemas.class); - - private static HashMap<String,DataSchema> schemaMap; - - private static HashSet<String> allowedPrimitiveJavaTypes = new HashSet<>(Arrays.asList(PrimitiveTypePartitioner.BYTE, PrimitiveTypePartitioner.SHORT, - PrimitiveTypePartitioner.INT, PrimitiveTypePartitioner.LONG, PrimitiveTypePartitioner.FLOAT, PrimitiveTypePartitioner.DOUBLE, - PrimitiveTypePartitioner.CHAR, PrimitiveTypePartitioner.STRING)); - - static - { - logger.info("Loading data schemas: "); - - schemaMap = new HashMap<>(); - try - { - initialize(); - } catch (Exception e) - { - logger.error("Caught exception: "); - e.printStackTrace(); - } - } - - public static void initialize() throws Exception - { - initialize(false, null); - } - - public static void initialize(boolean hdfs, FileSystem fs) throws Exception - { - String dataSchemas = SystemConfiguration.getProperty("data.schemas", "none"); - if (!dataSchemas.equals("none")) - { - String[] dataSchemaFiles = dataSchemas.split(","); - for (String schemaFile : dataSchemaFiles) - { - logger.info("Loading schemaFile = " + schemaFile + " hdfs = " + hdfs); - - // Parse and load the schema file into a DataSchema object; place in the schemaMap - DataSchema dataSchema = loadDataSchemaFile(schemaFile, hdfs, fs); - schemaMap.put(dataSchema.getSchemaName(), dataSchema); - } - } - } - - public static HashMap<String,DataSchema> getSchemaMap() - { - return schemaMap; - } - - public static Set<String> getSchemaNames() - { - return schemaMap.keySet(); - } - - public static DataSchema getSchema(String schemaName) - { - return schemaMap.get(schemaName.toLowerCase()); - } - - private static DataSchema loadDataSchemaFile(String schemaFile, boolean hdfs, FileSystem fs) throws Exception - { - DataSchema dataSchema; - - // Initialize the elements needed to create the DataSchema - String schemaName; - HashMap<String,Text> textRep = new HashMap<>(); - HashSet<String> listRep = new HashSet<>(); - HashMap<String,String> typeMap = new HashMap<>(); - HashMap<String,String> partitionerMap = new HashMap<>(); - HashMap<String,Object> partitionerInstances = new HashMap<>(); - - DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance(); - DocumentBuilder dBuilder = dbFactory.newDocumentBuilder(); - - // Read in and parse the schema file - Document doc; - if (hdfs) - { - Path filePath = new Path(schemaFile); - logger.info("hdfs: filePath = " + filePath.toString()); - doc = dBuilder.parse(fs.open(filePath)); - } - else - { - File inputFile = new File(schemaFile); - logger.info("localFS: inputFile = " + inputFile.toString()); - doc = dBuilder.parse(inputFile); - } - doc.getDocumentElement().normalize(); - logger.info("Root element: " + doc.getDocumentElement().getNodeName()); - - logger.info("Root element: " + doc.getDocumentElement().getNodeName()); - - // Extract the schemaName - NodeList schemaNameList = doc.getElementsByTagName("schemaName"); - if (schemaNameList.getLength() > 1) - { - throw new Exception("schemaNameList.getLength() = " + schemaNameList.getLength() + " -- should be 1, " + "one schema per xml file"); - } - schemaName = schemaNameList.item(0).getTextContent().trim().toLowerCase(); - logger.info("schemaName = " + schemaName); - - // Extract the elements - NodeList nList = doc.getElementsByTagName("element"); - for (int i = 0; i < nList.getLength(); i++) - { - Node nNode = nList.item(i); - if (nNode.getNodeType() == Node.ELEMENT_NODE) - { - Element eElement = (Element) nNode; - - // Pull out the attributes - String name = eElement.getElementsByTagName("name").item(0).getTextContent().trim().toLowerCase(); - String type = eElement.getElementsByTagName("type").item(0).getTextContent().trim(); - - // An absent isArray means false, and an empty isArray means true, otherwise take the value. - String isArray = "false"; - Node isArrayNode = eElement.getElementsByTagName("isArray").item(0); - if (isArrayNode != null) - { - String isArrayValue = isArrayNode.getTextContent().trim().toLowerCase(); - isArray = isArrayValue.isEmpty() ? "true" : isArrayValue; - } - - // Pull and check the partitioner class -- if the partitioner tag doesn't exist, then - // it defaults to the PrimitiveTypePartitioner - String partitioner = null; - boolean primitivePartitioner = true; - if (eElement.getElementsByTagName("partitioner").item(0) != null) - { - partitioner = eElement.getElementsByTagName("partitioner").item(0).getTextContent().trim(); - if (!partitioner.equals(PrimitiveTypePartitioner.class.getName())) - { - primitivePartitioner = false; - } - } - if (primitivePartitioner) - { - partitioner = PrimitiveTypePartitioner.class.getName(); - partitionerInstances.put(PrimitiveTypePartitioner.class.getName(), new PrimitiveTypePartitioner()); - - // Validate the type since we are using the primitive partioner - if (!allowedPrimitiveJavaTypes.contains(type.toLowerCase())) - { - throw new Exception("javaType = " + type + " is not one of the allowed javaTypes: " + " byte, short, int, long, float, double, char, string"); - } - } - else - // If we have a non-primitive partitioner - { - Class c = Class.forName(partitioner); - Object obj = c.newInstance(); - if (!(obj instanceof DataPartitioner)) - { - throw new Exception("partitioner = " + partitioner + " DOES NOT implement the DataPartitioner interface"); - } - partitionerInstances.put(partitioner, obj); - } - - // Place in the appropriate structures - textRep.put(name, new Text(name)); - typeMap.put(name, type); - partitionerMap.put(name, partitioner); - - if (isArray.equals("true")) - { - listRep.add(name); - } - - logger.info("name = " + name + " javaType = " + type + " isArray = " + isArray + " partitioner " + partitioner); - } - } - - // Construct the DataSchema and set the partitionerInstances - dataSchema = new DataSchema(schemaName, textRep, listRep, typeMap, partitionerMap); - dataSchema.setPartitionerInstances(partitionerInstances); - - return dataSchema; - } -} http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/src/main/java/org/apache/pirk/schema/query/LoadQuerySchemas.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/schema/query/LoadQuerySchemas.java b/src/main/java/org/apache/pirk/schema/query/LoadQuerySchemas.java index fa00756..93ae66a 100644 --- a/src/main/java/org/apache/pirk/schema/query/LoadQuerySchemas.java +++ b/src/main/java/org/apache/pirk/schema/query/LoadQuerySchemas.java @@ -30,7 +30,7 @@ import javax.xml.parsers.DocumentBuilderFactory; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.pirk.schema.data.DataSchema; -import org.apache.pirk.schema.data.LoadDataSchemas; +import org.apache.pirk.schema.data.DataSchemaRegistry; import org.apache.pirk.schema.data.partitioner.DataPartitioner; import org.apache.pirk.utils.SystemConfiguration; import org.slf4j.Logger; @@ -119,7 +119,7 @@ public class LoadQuerySchemas public static QuerySchema getSchema(String schemaName) { - return schemaMap.get(schemaName.toLowerCase()); + return schemaMap.get(schemaName); } private static QuerySchema loadQuerySchemaFile(String schemaFile, boolean hdfs, FileSystem fs) throws Exception @@ -154,7 +154,7 @@ public class LoadQuerySchemas String dataSchemaName = extractValue(doc, "dataSchemaName"); logger.info("dataSchemaName = " + dataSchemaName); - DataSchema dataSchema = LoadDataSchemas.getSchema(dataSchemaName); + DataSchema dataSchema = DataSchemaRegistry.get(dataSchemaName); if (dataSchema == null) { throw new Exception("Loaded DataSchema does not exist for dataSchemaName = " + dataSchemaName); @@ -187,12 +187,12 @@ public class LoadQuerySchemas Element eElement = (Element) nNode; // Pull the name and add to the TreeSet - String name = eElement.getFirstChild().getNodeValue().trim().toLowerCase(); + String name = eElement.getFirstChild().getNodeValue().trim(); elementNames.add(name); // Compute the number of bits for this element logger.info("name = " + name); - logger.info("partitionerName = " + dataSchema.getPartitionerName(name)); + logger.info("partitionerName = " + dataSchema.getPartitionerTypeName(name)); if ((dataSchema.getPartitionerForElement(name)) == null) { logger.info("partitioner is null"); @@ -200,7 +200,7 @@ public class LoadQuerySchemas int bits = ((DataPartitioner) dataSchema.getPartitionerForElement(name)).getBits(dataSchema.getElementType(name)); // Multiply by the number of array elements allowed, if applicable - if (dataSchema.getListRep().contains(name)) + if (dataSchema.getArrayElements().contains(name)) { bits *= Integer.parseInt(SystemConfiguration.getProperty("pir.numReturnArrayElements")); } @@ -237,7 +237,7 @@ public class LoadQuerySchemas Element eElement = (Element) nNode; // Pull the name and add to the TreeSet - String name = eElement.getFirstChild().getNodeValue().trim().toLowerCase(); + String name = eElement.getFirstChild().getNodeValue().trim(); filterNamesSet.add(name); logger.info("filterName = " + name); @@ -263,7 +263,7 @@ public class LoadQuerySchemas { throw new Exception("itemList.getLength() = " + itemList.getLength() + " -- should be 1"); } - value = itemList.item(0).getTextContent().trim().toLowerCase(); + value = itemList.item(0).getTextContent().trim(); return value; } http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/src/main/java/org/apache/pirk/schema/query/filter/StopListFilter.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/schema/query/filter/StopListFilter.java b/src/main/java/org/apache/pirk/schema/query/filter/StopListFilter.java index 6c58d2c..018037d 100644 --- a/src/main/java/org/apache/pirk/schema/query/filter/StopListFilter.java +++ b/src/main/java/org/apache/pirk/schema/query/filter/StopListFilter.java @@ -56,16 +56,16 @@ public class StopListFilter implements DataFilter // If the data element contains a value on the stoplist (corresponding to a key in the filterSet), do not use for (String filterName : filterSet) { - if (dSchema.hasListRep(filterName)) + if (dSchema.isArrayElement(filterName)) { List<String> elementArray = null; - if (dataElement.get(dSchema.getTextElement(filterName)) instanceof WritableArrayWritable) + if (dataElement.get(dSchema.getTextName(filterName)) instanceof WritableArrayWritable) { - elementArray = Arrays.asList(((WritableArrayWritable) dataElement.get(dSchema.getTextElement(filterName))).toStrings()); + elementArray = Arrays.asList(((WritableArrayWritable) dataElement.get(dSchema.getTextName(filterName))).toStrings()); } - else if (dataElement.get(dSchema.getTextElement(filterName)) instanceof ArrayWritable) + else if (dataElement.get(dSchema.getTextName(filterName)) instanceof ArrayWritable) { - elementArray = Arrays.asList(((ArrayWritable) dataElement.get(dSchema.getTextElement(filterName))).toStrings()); + elementArray = Arrays.asList(((ArrayWritable) dataElement.get(dSchema.getTextName(filterName))).toStrings()); } for (String element : elementArray) @@ -79,7 +79,7 @@ public class StopListFilter implements DataFilter } else { - String element = dataElement.get(dSchema.getTextElement(filterName)).toString(); + String element = dataElement.get(dSchema.getTextName(filterName)).toString(); passFilter = StopListUtils.checkElement(element, stopList); } if (!passFilter) http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/src/main/java/org/apache/pirk/schema/response/QueryResponseJSON.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/schema/response/QueryResponseJSON.java b/src/main/java/org/apache/pirk/schema/response/QueryResponseJSON.java index 5f7e79c..d8928da 100644 --- a/src/main/java/org/apache/pirk/schema/response/QueryResponseJSON.java +++ b/src/main/java/org/apache/pirk/schema/response/QueryResponseJSON.java @@ -21,13 +21,12 @@ package org.apache.pirk.schema.response; import java.io.Serializable; import java.util.ArrayList; import java.util.HashMap; -import java.util.HashSet; import java.util.Set; import org.apache.hadoop.io.Text; import org.apache.pirk.query.wideskies.QueryInfo; import org.apache.pirk.schema.data.DataSchema; -import org.apache.pirk.schema.data.LoadDataSchemas; +import org.apache.pirk.schema.data.DataSchemaRegistry; import org.apache.pirk.schema.query.LoadQuerySchemas; import org.apache.pirk.schema.query.QuerySchema; import org.json.simple.JSONObject; @@ -77,7 +76,7 @@ public class QueryResponseJSON implements Serializable } QuerySchema qSchema = LoadQuerySchemas.getSchema(queryInfo.getQueryType()); - dSchema = LoadDataSchemas.getSchema(qSchema.getDataSchemaName()); + dSchema = DataSchemaRegistry.get(qSchema.getDataSchemaName()); jsonObj = new JSONObject(); setGeneralQueryResponseFields(queryInfo); @@ -123,12 +122,12 @@ public class QueryResponseJSON implements Serializable @SuppressWarnings("unchecked") private void initialize() { - HashSet<String> schemaStringRep = dSchema.getNonListRep(); + Set<String> schemaStringRep = dSchema.getNonArrayElements(); for (String key : schemaStringRep) { jsonObj.put(key, ""); } - HashSet<String> schemaListRep = dSchema.getListRep(); + Set<String> schemaListRep = dSchema.getArrayElements(); for (String key : schemaListRep) { jsonObj.put(key, new ArrayList<>()); @@ -148,7 +147,7 @@ public class QueryResponseJSON implements Serializable } else { - if (dSchema.getListRep().contains(key)) + if (dSchema.getArrayElements().contains(key)) { if (!(val instanceof ArrayList)) { @@ -171,7 +170,7 @@ public class QueryResponseJSON implements Serializable jsonObj.put(key, val); } } - else if (dSchema.getNonListRep().contains(key) || key.equals(SELECTOR)) + else if (dSchema.getNonArrayElements().contains(key) || key.equals(SELECTOR)) { jsonObj.put(key, val); } http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/src/main/java/org/apache/pirk/test/distributed/DistributedTestDriver.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/test/distributed/DistributedTestDriver.java b/src/main/java/org/apache/pirk/test/distributed/DistributedTestDriver.java index d822e81..c946e3b 100755 --- a/src/main/java/org/apache/pirk/test/distributed/DistributedTestDriver.java +++ b/src/main/java/org/apache/pirk/test/distributed/DistributedTestDriver.java @@ -22,7 +22,7 @@ import java.util.ArrayList; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; -import org.apache.pirk.schema.data.LoadDataSchemas; +import org.apache.pirk.schema.data.DataSchemaLoader; import org.apache.pirk.schema.query.LoadQuerySchemas; import org.apache.pirk.schema.query.filter.StopListFilter; import org.apache.pirk.test.distributed.testsuite.DistTestSuite; @@ -137,7 +137,7 @@ public class DistributedTestDriver // Force the query and data schemas to load their original values if (!dataSchemasProp.equals("none")) { - LoadDataSchemas.initialize(); + DataSchemaLoader.initialize(); } if (!querySchemasProp.equals("none")) http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/src/main/java/org/apache/pirk/test/utils/BaseTests.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/test/utils/BaseTests.java b/src/main/java/org/apache/pirk/test/utils/BaseTests.java index 1816783..a01df92 100644 --- a/src/main/java/org/apache/pirk/test/utils/BaseTests.java +++ b/src/main/java/org/apache/pirk/test/utils/BaseTests.java @@ -18,6 +18,8 @@ */ package org.apache.pirk.test.utils; +import static org.junit.Assert.fail; + import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; @@ -35,7 +37,6 @@ import org.json.simple.JSONArray; import org.json.simple.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.junit.Assert.fail; /** * Class to hold the base functional distributed tests http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/src/main/java/org/apache/pirk/test/utils/Inputs.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/test/utils/Inputs.java b/src/main/java/org/apache/pirk/test/utils/Inputs.java index b36ad70..826815f 100644 --- a/src/main/java/org/apache/pirk/test/utils/Inputs.java +++ b/src/main/java/org/apache/pirk/test/utils/Inputs.java @@ -35,7 +35,7 @@ import javax.xml.transform.stream.StreamResult; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.pirk.schema.data.LoadDataSchemas; +import org.apache.pirk.schema.data.DataSchemaLoader; import org.apache.pirk.schema.data.partitioner.IPDataPartitioner; import org.apache.pirk.schema.data.partitioner.ISO8601DatePartitioner; import org.apache.pirk.schema.data.partitioner.PrimitiveTypePartitioner; @@ -437,7 +437,7 @@ public class Inputs { createDataSchema(fs, true); } - LoadDataSchemas.initialize(); + DataSchemaLoader.initialize(); // Create and load the query schemas // DNS_HOSTNAME_QUERY http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/src/main/java/org/apache/pirk/test/utils/StandaloneQuery.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/test/utils/StandaloneQuery.java b/src/main/java/org/apache/pirk/test/utils/StandaloneQuery.java index 684d04d..d88daa9 100644 --- a/src/main/java/org/apache/pirk/test/utils/StandaloneQuery.java +++ b/src/main/java/org/apache/pirk/test/utils/StandaloneQuery.java @@ -18,6 +18,8 @@ */ package org.apache.pirk.test.utils; +import static org.junit.Assert.fail; + import java.io.File; import java.io.IOException; import java.util.ArrayList; @@ -42,7 +44,6 @@ import org.apache.pirk.utils.SystemConfiguration; import org.json.simple.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.junit.Assert.fail; public class StandaloneQuery { http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/src/main/java/org/apache/pirk/test/utils/TestUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/test/utils/TestUtils.java b/src/main/java/org/apache/pirk/test/utils/TestUtils.java index 57af5c4..a7e303a 100644 --- a/src/main/java/org/apache/pirk/test/utils/TestUtils.java +++ b/src/main/java/org/apache/pirk/test/utils/TestUtils.java @@ -190,17 +190,17 @@ public class TestUtils // Add the schemaName Element schemaNameElement = doc.createElement("schemaName"); - schemaNameElement.appendChild(doc.createTextNode(querySchemaName.toLowerCase())); + schemaNameElement.appendChild(doc.createTextNode(querySchemaName)); rootElement.appendChild(schemaNameElement); // Add the dataSchemaName Element dataSchemaNameElement = doc.createElement("dataSchemaName"); - dataSchemaNameElement.appendChild(doc.createTextNode(dataSchemaNameInput.toLowerCase())); + dataSchemaNameElement.appendChild(doc.createTextNode(dataSchemaNameInput)); rootElement.appendChild(dataSchemaNameElement); // Add the selectorName Element selectorNameElement = doc.createElement("selectorName"); - selectorNameElement.appendChild(doc.createTextNode(selectorNameInput.toLowerCase())); + selectorNameElement.appendChild(doc.createTextNode(selectorNameInput)); rootElement.appendChild(selectorNameElement); // Add the elementNames @@ -210,7 +210,7 @@ public class TestUtils { logger.info("elementName = " + elementName); Element name = doc.createElement("name"); - name.appendChild(doc.createTextNode(elementName.toLowerCase())); + name.appendChild(doc.createTextNode(elementName)); elements.appendChild(name); } @@ -228,7 +228,7 @@ public class TestUtils { logger.info("filterName = " + filterName); Element name = doc.createElement("name"); - name.appendChild(doc.createTextNode(filterName.toLowerCase())); + name.appendChild(doc.createTextNode(filterName)); filterNamesElement.appendChild(name); } } http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/src/main/java/org/apache/pirk/utils/QueryParserUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/utils/QueryParserUtils.java b/src/main/java/org/apache/pirk/utils/QueryParserUtils.java index 36892a3..6f27438 100644 --- a/src/main/java/org/apache/pirk/utils/QueryParserUtils.java +++ b/src/main/java/org/apache/pirk/utils/QueryParserUtils.java @@ -26,12 +26,10 @@ import java.util.regex.Pattern; import org.apache.hadoop.io.MapWritable; import org.apache.hadoop.io.Text; - import org.apache.pirk.inputformat.hadoop.TextArrayWritable; import org.apache.pirk.schema.data.DataSchema; import org.apache.pirk.schema.data.partitioner.IPDataPartitioner; import org.elasticsearch.hadoop.mr.WritableArrayWritable; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -687,7 +685,7 @@ public class QueryParserUtils logger.debug("query = " + query); // Special case for IPs - if (dataSchema.getPartitionerName(field).equals(IPDataPartitioner.class.getName())) // Doesn't handle arrays of IPs in the value right now... + if (dataSchema.getPartitionerTypeName(field).equals(IPDataPartitioner.class.getName())) // Doesn't handle arrays of IPs in the value right now... { logger.debug("Have IP Field"); http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/src/main/java/org/apache/pirk/utils/StringUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/utils/StringUtils.java b/src/main/java/org/apache/pirk/utils/StringUtils.java index c06e969..6d25251 100755 --- a/src/main/java/org/apache/pirk/utils/StringUtils.java +++ b/src/main/java/org/apache/pirk/utils/StringUtils.java @@ -27,13 +27,11 @@ import org.apache.hadoop.io.ArrayWritable; import org.apache.hadoop.io.MapWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.Writable; - import org.apache.pirk.schema.data.DataSchema; import org.elasticsearch.hadoop.mr.WritableArrayWritable; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -111,7 +109,7 @@ public class StringUtils if (jsonObj.get(key) != null) { logger.debug("key = " + key.toString()); - if (dataSchema.hasListRep((String) key)) + if (dataSchema.isArrayElement((String) key)) { WritableArrayWritable mapValue = StringUtils.jsonArrayStringToWritableArrayWritable(jsonObj.get(key).toString()); value.put(mapKey, mapValue); @@ -151,7 +149,7 @@ public class StringUtils if (jsonObj.get(key) != null) { logger.debug("key = " + key.toString()); - if (dataSchema.hasListRep((String) key)) + if (dataSchema.isArrayElement((String) key)) { ArrayWritable mapValue = StringUtils.jsonArrayStringtoArrayWritable(jsonObj.get(key).toString()); value.put(mapKey, mapValue); @@ -190,7 +188,7 @@ public class StringUtils String mapKey = key.toString(); if (jsonObj.get(key) != null) { - if (dataSchema.hasListRep((String) key)) + if (dataSchema.isArrayElement((String) key)) { ArrayList<String> mapValue = StringUtils.jsonArrayStringToArrayList(jsonObj.get(key).toString()); value.put(mapKey, mapValue); http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/src/main/java/org/apache/pirk/utils/SystemConfiguration.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/pirk/utils/SystemConfiguration.java b/src/main/java/org/apache/pirk/utils/SystemConfiguration.java index b529e8f..c10ecfc 100755 --- a/src/main/java/org/apache/pirk/utils/SystemConfiguration.java +++ b/src/main/java/org/apache/pirk/utils/SystemConfiguration.java @@ -24,7 +24,7 @@ import java.io.IOException; import java.io.InputStream; import java.util.Properties; -import org.apache.pirk.schema.data.LoadDataSchemas; +import org.apache.pirk.schema.data.DataSchemaLoader; import org.apache.pirk.schema.query.LoadQuerySchemas; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -61,10 +61,10 @@ public class SystemConfiguration // Load any data schema files indicated in the properties try { - LoadDataSchemas.class.newInstance(); + DataSchemaLoader.class.newInstance(); } catch (Exception e) { - logger.error("Issue when invoking LoadDataSchemas"); + logger.error("Issue when invoking DataSchemaLoader"); e.printStackTrace(); } @@ -74,7 +74,7 @@ public class SystemConfiguration LoadQuerySchemas.class.newInstance(); } catch (Exception e) { - logger.error("Issue when invoking LoadDataSchemas"); + logger.error("Issue when invoking DataSchemaLoader"); e.printStackTrace(); } } http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/src/main/resources/data-schema.xsd ---------------------------------------------------------------------- diff --git a/src/main/resources/data-schema.xsd b/src/main/resources/data-schema.xsd index d49e54e..7384409 100644 --- a/src/main/resources/data-schema.xsd +++ b/src/main/resources/data-schema.xsd @@ -22,16 +22,58 @@ <xs:element name="schema"> <xs:complexType> <xs:sequence> - <xs:element name="schemaName" type="xs:string" /> + + <xs:element name="schemaName" type="xs:string"> + <xs:annotation> + <xs:documentation>The name of the data schema. + The name omits leading and trailing + whitespace, and is case sensitive. + </xs:documentation> + </xs:annotation> + </xs:element> + <xs:element name="element" maxOccurs="unbounded"> <xs:complexType> <xs:sequence> - <xs:element name="name" type="xs:string" /> - <xs:element name="type" type="xs:string" /> + + <xs:element name="name" type="xs:string"> + <xs:annotation> + <xs:documentation>The name of the + data element. + The name omits + leading and trailing whitespace, + and is case sensitive. + </xs:documentation> + </xs:annotation> + </xs:element> + + <xs:element name="type" type="xs:string"> + <xs:annotation> + <xs:documentation>The type of the data element. + The type name is the fully qualified class name, + or the primitive Java type of the element. + </xs:documentation> + </xs:annotation> + </xs:element> + <xs:element name="isArray" type="xs:boolean" - default="true" minOccurs="0" /> + default="true" minOccurs="0"> + <xs:annotation> + <xs:documentation>Whether or not the schema element is + an array within the data. False if unspecified. + </xs:documentation> + </xs:annotation> + </xs:element> + <xs:element name="partitioner" - type="xs:string" minOccurs="0" /> + type="xs:string" minOccurs="0"> + <xs:annotation> + <xs:documentation>Partitioner type for the element. + The type name is the fully qualified class name. + Uses the primitive Java type partitioner if unspecified. + </xs:documentation> + </xs:annotation> + </xs:element> </xs:sequence> </xs:complexType> </xs:element> http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/src/test/java/test/general/ISO8601DateParserTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/test/general/ISO8601DateParserTest.java b/src/test/java/test/general/ISO8601DateParserTest.java index b3d41aa..02391d4 100644 --- a/src/test/java/test/general/ISO8601DateParserTest.java +++ b/src/test/java/test/general/ISO8601DateParserTest.java @@ -18,13 +18,14 @@ */ package test.general; +import static org.junit.Assert.assertEquals; + import java.text.ParseException; import org.apache.pirk.utils.ISO8601DateParser; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.junit.Assert.assertEquals; /** * Class to test basic functionality of ISO8601DateParser class http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/src/test/java/test/general/KeyedHashTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/test/general/KeyedHashTest.java b/src/test/java/test/general/KeyedHashTest.java index 131e18f..6f69b38 100644 --- a/src/test/java/test/general/KeyedHashTest.java +++ b/src/test/java/test/general/KeyedHashTest.java @@ -18,11 +18,12 @@ */ package test.general; +import static org.junit.Assert.assertEquals; + import org.apache.pirk.utils.KeyedHash; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.junit.Assert.assertEquals; /** * Basic functional tests for KeyedHash http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/src/test/java/test/general/PaillierTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/test/general/PaillierTest.java b/src/test/java/test/general/PaillierTest.java index ce86285..abb50fa 100644 --- a/src/test/java/test/general/PaillierTest.java +++ b/src/test/java/test/general/PaillierTest.java @@ -18,6 +18,9 @@ */ package test.general; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + import java.math.BigInteger; import java.util.Random; @@ -27,8 +30,6 @@ import org.apache.pirk.utils.SystemConfiguration; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; /** * Basic test functionality for Paillier library http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/src/test/java/test/general/PartitionUtilsTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/test/general/PartitionUtilsTest.java b/src/test/java/test/general/PartitionUtilsTest.java index 7dc97e2..f226cb5 100644 --- a/src/test/java/test/general/PartitionUtilsTest.java +++ b/src/test/java/test/general/PartitionUtilsTest.java @@ -18,6 +18,9 @@ */ package test.general; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + import java.math.BigInteger; import java.util.ArrayList; @@ -29,9 +32,6 @@ import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; - /** * Class to functionally test the bit conversion utils */ http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/src/test/java/test/general/QueryParserUtilsTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/test/general/QueryParserUtilsTest.java b/src/test/java/test/general/QueryParserUtilsTest.java index c57676e..16d73bb 100644 --- a/src/test/java/test/general/QueryParserUtilsTest.java +++ b/src/test/java/test/general/QueryParserUtilsTest.java @@ -18,12 +18,15 @@ */ package test.general; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + import java.util.ArrayList; import java.util.Map; import org.apache.hadoop.io.MapWritable; import org.apache.pirk.schema.data.DataSchema; -import org.apache.pirk.schema.data.LoadDataSchemas; +import org.apache.pirk.schema.data.DataSchemaRegistry; import org.apache.pirk.test.utils.Inputs; import org.apache.pirk.utils.QueryParserUtils; import org.apache.pirk.utils.StringUtils; @@ -32,9 +35,6 @@ import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - /** * Class for testing the QueryParser methods */ @@ -54,7 +54,7 @@ public class QueryParserUtilsTest Inputs.createSchemaFiles(null, false, null); - dSchema = LoadDataSchemas.getSchema(Inputs.TEST_DATA_SCHEMA_NAME); + dSchema = DataSchemaRegistry.get(Inputs.TEST_DATA_SCHEMA_NAME); // ProcessBuilder pAdd1 = new ProcessBuilder("curl", "-XPUT", indexTypeNum1, "-d", // "{\"qname\":\"a.b.c.com\",\"date\":\"2016-02-20T23:29:05.000Z\",\"qtype\":[\"1\"]" http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/src/test/java/test/schema/data/LoadDataSchemaTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/test/schema/data/LoadDataSchemaTest.java b/src/test/java/test/schema/data/LoadDataSchemaTest.java index 361442a..07167fc 100644 --- a/src/test/java/test/schema/data/LoadDataSchemaTest.java +++ b/src/test/java/test/schema/data/LoadDataSchemaTest.java @@ -18,6 +18,10 @@ */ package test.schema.data; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + import java.io.File; import java.io.IOException; @@ -29,7 +33,8 @@ import javax.xml.transform.dom.DOMSource; import javax.xml.transform.stream.StreamResult; import org.apache.pirk.schema.data.DataSchema; -import org.apache.pirk.schema.data.LoadDataSchemas; +import org.apache.pirk.schema.data.DataSchemaLoader; +import org.apache.pirk.schema.data.DataSchemaRegistry; import org.apache.pirk.schema.data.partitioner.IPDataPartitioner; import org.apache.pirk.schema.data.partitioner.PrimitiveTypePartitioner; import org.apache.pirk.test.utils.TestUtils; @@ -39,9 +44,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.w3c.dom.Document; import org.w3c.dom.Element; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; /** * Test suite for LoadDataSchema and DataSchema @@ -73,37 +75,38 @@ public class LoadDataSchemaTest } // Force the schema to load - LoadDataSchemas.initialize(); + DataSchemaLoader.initialize(); // Check the entries - DataSchema dSchema = LoadDataSchemas.getSchema(dataSchemaName); + DataSchema dSchema = DataSchemaRegistry.get(dataSchemaName); + + assertEquals(dataSchemaName, dSchema.getSchemaName()); - assertEquals(dataSchemaName.toLowerCase(), dSchema.getSchemaName()); + assertEquals(3, dSchema.getElementNames().size()); - assertEquals(3, dSchema.getTextRep().size()); + // TODO: check Hadoop text names - assertEquals(3, dSchema.getTypeMap().size()); - assertEquals(PrimitiveTypePartitioner.STRING, dSchema.getElementType(element1.toLowerCase())); - assertEquals(PrimitiveTypePartitioner.INT, dSchema.getElementType(element2.toLowerCase())); - assertEquals(PrimitiveTypePartitioner.STRING, dSchema.getElementType(element3.toLowerCase())); + assertEquals(PrimitiveTypePartitioner.STRING, dSchema.getElementType(element1)); + assertEquals(PrimitiveTypePartitioner.INT, dSchema.getElementType(element2)); + assertEquals(PrimitiveTypePartitioner.STRING, dSchema.getElementType(element3)); - assertEquals(PrimitiveTypePartitioner.class.getName(), dSchema.getPartitionerName(element1.toLowerCase())); - if (!(dSchema.getPartitionerForElement(element1.toLowerCase()) instanceof PrimitiveTypePartitioner)) + assertEquals(PrimitiveTypePartitioner.class.getName(), dSchema.getPartitionerTypeName(element1)); + if (!(dSchema.getPartitionerForElement(element1) instanceof PrimitiveTypePartitioner)) { fail("Partitioner instance for element1 must be PrimitiveTypePartitioner"); } - assertEquals(IPDataPartitioner.class.getName(), dSchema.getPartitionerName(element3.toLowerCase())); - if (!(dSchema.getPartitionerForElement(element3.toLowerCase()) instanceof IPDataPartitioner)) + assertEquals(IPDataPartitioner.class.getName(), dSchema.getPartitionerTypeName(element3)); + if (!(dSchema.getPartitionerForElement(element3) instanceof IPDataPartitioner)) { fail("Partitioner instance for element3 must be IPDataPartitioner"); } - assertEquals(2, dSchema.getListRep().size()); - assertTrue(dSchema.getListRep().contains(element2.toLowerCase())); - assertTrue(dSchema.getListRep().contains(element3.toLowerCase())); + assertEquals(2, dSchema.getArrayElements().size()); + assertTrue(dSchema.getArrayElements().contains(element2)); + assertTrue(dSchema.getArrayElements().contains(element3)); - assertEquals(1, dSchema.getNonListRep().size()); - assertTrue(dSchema.getNonListRep().contains(element1.toLowerCase())); + assertEquals(1, dSchema.getNonArrayElements().size()); + assertTrue(dSchema.getNonArrayElements().contains(element1)); // Reset original data.schemas property SystemConfiguration.setProperty("data.schemas", schemasProp); @@ -111,7 +114,7 @@ public class LoadDataSchemaTest // Force the schema to load if (!schemasProp.equals("none")) { - LoadDataSchemas.initialize(); + DataSchemaLoader.initialize(); } } @@ -134,8 +137,8 @@ public class LoadDataSchemaTest try { // Force the schema to load - LoadDataSchemas.initialize(); - fail("LoadDataSchemas did not throw exception for incorrect javaType"); + DataSchemaLoader.initialize(); + fail("DataSchemaLoader did not throw exception for incorrect javaType"); } catch (Exception ignore) {} @@ -143,7 +146,7 @@ public class LoadDataSchemaTest SystemConfiguration.setProperty("data.schemas", schemasProp); // Force the schema to load - LoadDataSchemas.initialize(); + DataSchemaLoader.initialize(); } @Test @@ -165,8 +168,8 @@ public class LoadDataSchemaTest try { // Force the schema to load - LoadDataSchemas.initialize(); - fail("LoadDataSchemas did not throw exception for unknown partitioner"); + DataSchemaLoader.initialize(); + fail("DataSchemaLoader did not throw exception for unknown partitioner"); } catch (Exception ignore) {} @@ -174,7 +177,7 @@ public class LoadDataSchemaTest SystemConfiguration.setProperty("data.schemas", schemasProp); // Force the schema to load - LoadDataSchemas.initialize(); + DataSchemaLoader.initialize(); } // Create the file that contains an unknown partitioner http://git-wip-us.apache.org/repos/asf/incubator-pirk/blob/c12d47f1/src/test/java/test/schema/query/LoadQuerySchemaTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/test/schema/query/LoadQuerySchemaTest.java b/src/test/java/test/schema/query/LoadQuerySchemaTest.java index dee8336..fb68c2a 100644 --- a/src/test/java/test/schema/query/LoadQuerySchemaTest.java +++ b/src/test/java/test/schema/query/LoadQuerySchemaTest.java @@ -18,6 +18,9 @@ */ package test.schema.query; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + import java.io.File; import java.io.IOException; import java.util.Arrays; @@ -31,7 +34,7 @@ import javax.xml.transform.TransformerFactory; import javax.xml.transform.dom.DOMSource; import javax.xml.transform.stream.StreamResult; -import org.apache.pirk.schema.data.LoadDataSchemas; +import org.apache.pirk.schema.data.DataSchemaLoader; import org.apache.pirk.schema.data.partitioner.IPDataPartitioner; import org.apache.pirk.schema.data.partitioner.PrimitiveTypePartitioner; import org.apache.pirk.schema.query.LoadQuerySchemas; @@ -45,9 +48,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.w3c.dom.Document; import org.w3c.dom.Element; -import test.schema.data.LoadDataSchemaTest; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; import test.schema.data.LoadDataSchemaTest; @@ -92,7 +92,7 @@ public class LoadQuerySchemaTest e.printStackTrace(); fail(e.toString()); } - LoadDataSchemas.initialize(); + DataSchemaLoader.initialize(); // Create the query schema used and force it to load try @@ -109,9 +109,9 @@ public class LoadQuerySchemaTest // Check the entries QuerySchema qSchema = LoadQuerySchemas.getSchema(querySchemaName); - assertEquals(querySchemaName.toLowerCase(), qSchema.getSchemaName()); - assertEquals(dataSchemaName.toLowerCase(), qSchema.getDataSchemaName()); - assertEquals(element4.toLowerCase(), qSchema.getSelectorName()); + assertEquals(querySchemaName, qSchema.getSchemaName()); + assertEquals(dataSchemaName, qSchema.getDataSchemaName()); + assertEquals(element4, qSchema.getSelectorName()); assertEquals(StopListFilter.class.getName(), qSchema.getFilter()); if (!(qSchema.getFilterInstance() instanceof StopListFilter)) @@ -122,18 +122,17 @@ public class LoadQuerySchemaTest assertEquals(3, qSchema.getElementNames().size()); for (String item : qSchema.getElementNames()) { - if (!(item.equals(element1.toLowerCase()) || item.equals(element2.toLowerCase()) || item.equals(element3.toLowerCase()))) + if (!(item.equals(element1) || item.equals(element2) || item.equals(element3))) { - fail("elementNames: item = " + item + " must equal one of: " + element1.toLowerCase() + ", " + element2.toLowerCase() + ", or " - + element3.toLowerCase()); + fail("elementNames: item = " + item + " must equal one of: " + element1 + ", " + element2 + ", or " + element3); } } assertEquals(1, qSchema.getFilterElementNames().size()); for (String item : qSchema.getFilterElementNames()) { - if (!item.equals(element2.toLowerCase())) + if (!item.equals(element2)) { - fail("filterElementNames: item = " + item + " must equal " + element2.toLowerCase()); + fail("filterElementNames: item = " + item + " must equal " + element2); } } @@ -151,7 +150,7 @@ public class LoadQuerySchemaTest // Force the query and data schemas to load their original values if (!dataSchemasProp.equals("none")) { - LoadDataSchemas.initialize(); + DataSchemaLoader.initialize(); } if (!querySchemasProp.equals("none")) @@ -178,7 +177,7 @@ public class LoadQuerySchemaTest e.printStackTrace(); fail(e.toString()); } - LoadDataSchemas.initialize(); + DataSchemaLoader.initialize(); // Create the query schema used and force it to load try @@ -204,7 +203,7 @@ public class LoadQuerySchemaTest // Force the query and data schemas to load their original values if (!dataSchemasProp.equals("none")) { - LoadDataSchemas.initialize(); + DataSchemaLoader.initialize(); } if (!querySchemasProp.equals("none")) @@ -268,7 +267,7 @@ public class LoadQuerySchemaTest e.printStackTrace(); fail(e.toString()); } - LoadDataSchemas.initialize(); + DataSchemaLoader.initialize(); // Create the query schema used and force it to load try @@ -295,7 +294,7 @@ public class LoadQuerySchemaTest // Force the query and data schemas to load their original values if (!dataSchemasProp.equals("none")) { - LoadDataSchemas.initialize(); + DataSchemaLoader.initialize(); } if (!querySchemasProp.equals("none"))
