http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/OGCFilter.java ---------------------------------------------------------------------- diff --git a/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/OGCFilter.java b/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/OGCFilter.java new file mode 100644 index 0000000..7bf622d --- /dev/null +++ b/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/OGCFilter.java @@ -0,0 +1,241 @@ +package ss.cloudbase.core.iterators.filter.ogc; + +import java.io.IOException; +import java.io.StringReader; +import java.util.HashMap; +import java.util.Map; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; + +import org.apache.log4j.Logger; +import org.w3c.dom.Document; +import org.w3c.dom.Node; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; + +import ss.cloudbase.core.iterators.filter.ogc.operation.IOperation; + +import cloudbase.core.data.Key; +import cloudbase.core.data.Value; +import cloudbase.core.iterators.filter.Filter; +import cloudbase.start.classloader.CloudbaseClassLoader; + +/** + * The OGCFilter class provides a basic implementation of the + * <a href="http://www.opengeospatial.org/standards/filter";>OGC Filter Encoding specification</a>. + * This allows for arbitrary queries to be passed via XML and executed in a distributed fashion across tablet servers. The following + * code sets up a basic FilteringIterator that uses this filter (note that this jar must be present in each + * of the tablet servers' classpaths to work): + * + * <code> + * <pre> + * cloudbase.core.client.Scanner reader; + * // set up the reader ... + * + * // we're going to parse a row formated like key:value|key:value|key:value and return all of the rows + * // where the key "city" starts with "new" + * String filter = "<PropertyIsLike><PropertyName>city</PropertyName><Literal>new*</Literal></PropertyIsLike>"; + * + * reader.setScanIterators(50, FilteringIterator.class.getName(), "myIterator"); + * reader.setScanIteratorOption("myIterator", "0", OGCFilter.class.getName()); + * reader.setScanIteratorOption("myIterator", "0." + OGCFilter.OPTION_PAIR_DELIMITER, "\\|"); + * reader.setScanIteratorOption("myIterator", "0." + OGCFilter.OPTION_VALUE_DELIMITER, ":"); + * reader.setScanIteratorOption("myIterator", "0." + OGCFilter.OPTION_FILTER, filter); + * <pre> + * </code> + * + * @author William Wall + */ +public class OGCFilter implements Filter { + private static final Logger logger = Logger.getLogger(OGCFilter.class); + + /** The string that separates the key/value pairs in the row value. */ + public static final String OPTION_PAIR_DELIMITER = "pairDelimiter"; + + /** The string that separates the key and the value(s) within a pair in the row value. */ + public static final String OPTION_VALUE_DELIMITER = "valueDelimiter"; + + /** The OGC Filter Encoding XML as a string */ + public static final String OPTION_FILTER = "filter"; + + /** Specifies the column name for the column family. If this option is not included, the column family is not included in the row. */ + public static final String OPTION_COLF_NAME = "colfName"; + + /** Specifies the column name for the column qualifier. If this option is not included, the column qualifier is not included in the row. */ + public static final String OPTION_COLQ_NAME = "colqName"; + + /** Specifies the compare type for the filter. Defaults to "auto". **/ + public static final String OPTION_COMPARE_TYPE = "compareType"; + + public static final String TYPE_NUMERIC = "numeric"; + + public static final String TYPE_STRING = "string"; + + public static final String TYPE_AUTO = "auto"; + + /** Contains the pair delimiter provided through the <code>OPTION_PAIR_DELIMITER</code> option. */ + protected String pairDelimiter; + + /** Contains the value delimiter provided through the <code>OPTION_VALUE_DELIMITER</code> option. */ + protected String valueDelimiter; + + /** Contains the column family column name provided through the <code>OPTION_COLF_NAME</code> option. */ + protected String colfName; + + /** Contains the column qualifier column name provided through the <code>OPTION_COLQ_NAME</code> option. */ + protected String colqName; + + /** The root operation of the query tree **/ + protected IOperation root; + + /** The compare type for the query tree **/ + protected String compareType; + + /** + * Whether or not to accept this key/value entry. A map of row keys and values is parsed and then sent off to the process function to be evaluated. + * @param key The cloudbase entry key + * @param value The cloudbase entry value + * @return True if the entry should be included in the results, false otherwise + */ + @Override + public boolean accept(Key key, Value value) { + if (root != null) { + Map<String, String> row = getRow(key, value); + if (root != null) { + return root.execute(row); + } + } + return false; + } + + public boolean accept(Map<String, String> record) { + if (root != null) { + return root.execute(record); + } + return false; + } + + /** + * Parses the cloudbase value into a map of key/value pairs. If the <code>OPTION_COLF_NAME</code> + * or <code>OPTION_COLQ_NAME</code> options were used, then they will also be added to the row map. + * By default, pairs are delimited by the first unicode character ("\u0000") and values by the last unicode + * character ("\uFFFD"). See the <code>OPTION_PAIR_DELIMITER</code> and <code>OPTION_VALUE_DELIMITER</code> + * options to change these values. + * + * @param cbKey The cloudbase entry key + * @param cbValue The cloudbase entry value + * @return A map that represents this row + */ + protected Map<String, String> getRow(Key cbKey, Value cbValue) { + //TODO: This should really be replaced by CBValueFormatter.parse(value.toString()), but I'm hesitant to require + // more jars (common-data and google-collections) to be in the cloudbase/lib directory. Also, what do we do with + // a field with multiple values? Should we just assume that if any value in that field matches then the row + // matches? Or should they all have to match? + + String value = cbValue.toString(); + Map<String, String> row = new HashMap<String, String>(); + + if (colfName != null) { + row.put(colfName, cbKey.getColumnFamily().toString()); + } + if (colqName != null) { + row.put(colqName, cbKey.getColumnQualifier().toString()); + } + + // Determine the start/end of the value. + int valueStartIndex = 0; + int valueEndIndex = value.length(); + + int vLen = valueDelimiter.length(); + int fLen = pairDelimiter.length(); + + // Parse each of the values from the row value. + while (valueStartIndex < valueEndIndex) { + int vIndex = value.indexOf(valueDelimiter, valueStartIndex); + + // If an "equals" sign was found, parse the key and value. + if (vIndex != -1) { + String key = value.substring(valueStartIndex, vIndex).trim(); + int v = value.indexOf(valueDelimiter, vIndex + vLen); + if (v == -1) { + v = valueEndIndex; + } + int f = value.indexOf(pairDelimiter, vIndex + vLen); + if (f == -1) { + f = valueEndIndex; + } + + int fIndex = Math.min(f,v); + String val = value.substring(vIndex + 1, fIndex).trim(); + valueStartIndex = f; + valueStartIndex += fLen; + row.put(key, val); + } + } + + return row; + } + + @Override + public void init(Map<String, String> options) { + pairDelimiter = options.get(OPTION_PAIR_DELIMITER); + if (pairDelimiter == null || pairDelimiter.length() == 0) { + pairDelimiter = "\u0000"; + } + + valueDelimiter = options.get(OPTION_VALUE_DELIMITER); + if (valueDelimiter == null || valueDelimiter.length() == 0) { + valueDelimiter = "\uFFFD"; + } + + compareType = options.get(OPTION_COMPARE_TYPE); + if (compareType == null || compareType.length() == 0) { + compareType = TYPE_AUTO; + } + + colfName = options.get(OPTION_COLF_NAME); + colqName = options.get(OPTION_COLQ_NAME); + + try { + DocumentBuilder builder = DocumentBuilderFactory.newInstance().newDocumentBuilder(); + InputSource is = new InputSource(); + is.setCharacterStream(new StringReader(options.get(OPTION_FILTER))); + Document doc = builder.parse(is); + Node filter = doc.getDocumentElement(); + if (filter.getNodeName().equalsIgnoreCase("filter")) { + filter = filter.getFirstChild(); + } + root = createOperationTree(filter); + } catch (IOException e) { + logger.error(e,e); + } catch (SAXException e) { + logger.error(e,e); + } catch (ParserConfigurationException e) { + logger.error(e,e); + } + } + + /** + * Creates the operation tree from the filter XML + * @param node The filter XML node to parse + * @return The root IOperation + */ + protected IOperation createOperationTree(Node node) { + try { + // instantiate the operation and initialize it + Class<? extends IOperation> clazz = CloudbaseClassLoader.loadClass(IOperation.class.getPackage().getName() + "." + node.getNodeName(), IOperation.class); + IOperation op = clazz.newInstance(); + op.init(node, compareType); + return op; + } catch (ClassNotFoundException e) { + logger.warn("Operation not supported: " + node.getNodeName()); + } catch (InstantiationException e) { + logger.error(e,e); + } catch (IllegalAccessException e) { + logger.error(e,e); + } + return null; + } +}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/AbstractComparisonOp.java ---------------------------------------------------------------------- diff --git a/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/AbstractComparisonOp.java b/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/AbstractComparisonOp.java new file mode 100644 index 0000000..46b1cf9 --- /dev/null +++ b/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/AbstractComparisonOp.java @@ -0,0 +1,80 @@ +package ss.cloudbase.core.iterators.filter.ogc.operation; + +import java.util.List; +import java.util.Map; + +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +import ss.cloudbase.core.iterators.filter.ogc.OGCFilter; + + + +/** + * This class provides a simple init method for setting up most of the variables + * needed to do a comparison operation between two values. + * + * @author William Wall + */ +public abstract class AbstractComparisonOp { + protected String name, literal, value; + protected boolean isNumeric = false; + protected double literalNum, valueNum; + protected String compareType; + + public void init(Node node, String compareType) { + this.compareType = compareType; + Node child; + NodeList children = node.getChildNodes(); + for (int i = 0; i < children.getLength(); i++) { + child = children.item(i); + if (child.getNodeName().equalsIgnoreCase("PropertyName")) { + name = child.getTextContent(); + } else { + literal = child.getTextContent(); + } + } + + if (compareType.equalsIgnoreCase(OGCFilter.TYPE_NUMERIC) || compareType.equalsIgnoreCase(OGCFilter.TYPE_AUTO)) { + literalNum = parseNumeric(literal); + isNumeric = !Double.isNaN(literalNum); + } + } + + public List<IOperation> getChildren() { + return null; + } + + protected boolean checkRowNumeric(String s) { + if (isNumeric) { + valueNum = parseNumeric(s); + return valueNum != Double.NaN; + } + return false; + } + + public String getValue(Map<String, String> row) { + String value = row.get(name); + + // nulls will be lexicographically equal to "" + if (value == null) { + value = ""; + } + return value; + } + + public static double parseNumeric(String s) { + // see if the string can be parsed as a double or an integer + double val = Double.NaN; + try { + val = Double.parseDouble(s); + } catch (Exception e) { + try { + val = new Double(Integer.parseInt(s)); + } catch (Exception e2) { + + } + } + return val; + } +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/AbstractLogicalOp.java ---------------------------------------------------------------------- diff --git a/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/AbstractLogicalOp.java b/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/AbstractLogicalOp.java new file mode 100644 index 0000000..0400f61 --- /dev/null +++ b/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/AbstractLogicalOp.java @@ -0,0 +1,40 @@ +package ss.cloudbase.core.iterators.filter.ogc.operation; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.log4j.Logger; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +import cloudbase.start.classloader.CloudbaseClassLoader; + +public class AbstractLogicalOp { + private static final Logger logger = Logger.getLogger(AbstractLogicalOp.class); + + List<IOperation> children = new ArrayList<IOperation>(); + + public void init(Node node, String compareType) { + Node child; + NodeList children = node.getChildNodes(); + for (int i = 0; i < children.getLength(); i++) { + child = children.item(i); + try { + Class<? extends IOperation> clazz = CloudbaseClassLoader.loadClass(IOperation.class.getPackage().getName() + "." + child.getNodeName(), IOperation.class); + IOperation op = clazz.newInstance(); + op.init(child, compareType); + this.children.add(op); + } catch (ClassNotFoundException e) { + logger.warn("Operation not supported: " + node.getNodeName()); + } catch (InstantiationException e) { + logger.error(e,e); + } catch (IllegalAccessException e) { + logger.error(e,e); + } + } + } + + public List<IOperation> getChildren() { + return children; + } +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/And.java ---------------------------------------------------------------------- diff --git a/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/And.java b/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/And.java new file mode 100644 index 0000000..b192b19 --- /dev/null +++ b/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/And.java @@ -0,0 +1,29 @@ +package ss.cloudbase.core.iterators.filter.ogc.operation; + +import java.util.Map; + +/** + * Executes a logical AND on all the child operations. + * + * <code> + * <pre> + * <And> + * <PropertyIsEqualTo>... + * <PropertyIsLessThan>... + * </And> + * </pre> + * </code> + * + * @author William Wall + */ +public class And extends AbstractLogicalOp implements IOperation { + @Override + public boolean execute(Map<String, String> row) { + boolean result = true; + for (int i = 0; i < children.size(); i++) { + result = children.get(i).execute(row); + if (!result) break; + } + return result; + } +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/BBOX.java ---------------------------------------------------------------------- diff --git a/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/BBOX.java b/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/BBOX.java new file mode 100644 index 0000000..8596338 --- /dev/null +++ b/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/BBOX.java @@ -0,0 +1,125 @@ +package ss.cloudbase.core.iterators.filter.ogc.operation; + +import java.awt.Rectangle; +import java.awt.Shape; +import java.awt.geom.Point2D; +import java.util.List; +import java.util.Map; + +import org.apache.log4j.Logger; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + + +/** + * Tests a row to see if it falls within the given shape. The shape should be + * defined in degrees. There is no need to send a property name since all the + * rows contain either lonself/latself or lon/lat fields. + * + * Example: + * <pre> + * <BBOX> + * <gml:Envelope> + * <!-- coordinates are in lon/lat order --> + * <gml:LowerCorner>13.09 31.5899</gml:LowerCorner> + * <gml:UpperCorner>35.725 42.8153</gml:UpperCorner> + * </gml:Envelope> + * </BBOX> + * </pre> + * + * @author William Wall + */ +public class BBOX implements IOperation { + private static final Logger logger = Logger.getLogger(BBOX.class); + + Shape shape; + + // longitude column names in order of priority + protected static final String[] LON_NAMES = { + "lonself", + "lon", + "long", + "longitude" + }; + + // latitude column names in order of priority + protected static final String[] LAT_NAMES = { + "latself", + "lat", + "latitude" + }; + + @Override + public boolean execute(Map<String, String> row) { + Point2D p = BBOX.getPoint(row); + + if (p != null && shape != null) { + if (shape.contains(p)) { + return true; + } else { + // attempt to normalize the point into the shape in the event that the shape + // bounds are outside of -180 to 180 + Rectangle bounds = shape.getBounds(); + while (p.getX() < bounds.getMinX()) { + p.setLocation(p.getX() + 360, p.getY()); + } + while (p.getX() > bounds.getMaxX()) { + p.setLocation(p.getX() - 360, p.getY()); + } + + return shape.contains(p); + } + } + + return false; + } + + @Override + public List<IOperation> getChildren() { + return null; + } + + @Override + public void init(Node node, String compareType) { + NodeList children = node.getChildNodes(); + for (int i = 0; i < children.getLength(); i++) { + shape = ShapeFactory.getShape(children.item(i)); + if (shape != null) { + break; + } + } + + } + + /** + * Gets a point that represents the location of this row. See + * @param row + * @return The point object as (x - Longitude, y - Latitude) + */ + protected static Point2D getPoint(Map<String, String> row) { + Point2D.Double p = new Point2D.Double(); + p.x = getDegree(row, LON_NAMES); + p.y = getDegree(row, LAT_NAMES); + return p; + } + + protected static double getDegree(Map<String, String> row, String[] cols) { + double num = Double.NaN; + String value; + for (int i = 0; i < cols.length; i++) { + if (row.containsKey(cols[i])) { + value = row.get(cols[i]); + if (value != null && !value.equals("-")) { + try { + num = Double.parseDouble(value); + break; + } catch (NumberFormatException e) { + logger.warn("Could not parse degree value from " + cols[i] + " = " + value); + } + } + } + } + + return num; + } +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/IOperation.java ---------------------------------------------------------------------- diff --git a/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/IOperation.java b/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/IOperation.java new file mode 100644 index 0000000..6ad8ebe --- /dev/null +++ b/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/IOperation.java @@ -0,0 +1,30 @@ +package ss.cloudbase.core.iterators.filter.ogc.operation; + +import java.util.List; +import java.util.Map; + +import org.w3c.dom.Node; + +public interface IOperation { + + /** + * Sets up the operation from the filter XML node + * @param node The node + * @param compareType The compare type. Defaults to "auto", but you can force it to be "numeric" or "string". + */ + public void init(Node node, String compareType); + + /** + * Executes the operation indicated by the given node in the query + * tree. + * @param row The key/value pairs for the current row + * @return The boolean evaluation of the operation + */ + public boolean execute(Map<String, String> row); + + /** + * Returns the nodes children. This is only applicable to logical + * operations (AND, OR, NOT). + */ + public List<IOperation> getChildren(); +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/Not.java ---------------------------------------------------------------------- diff --git a/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/Not.java b/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/Not.java new file mode 100644 index 0000000..74826a8 --- /dev/null +++ b/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/Not.java @@ -0,0 +1,35 @@ +package ss.cloudbase.core.iterators.filter.ogc.operation; + +import java.util.Map; + +/** + * Executes a logical NOT on the child operations. If there is a single child, then + * the operation is NOT. If more than one child exists, this operation defaults to + * NOR behavior. For NAND behavior, make a single AND child of NOT. + * + * <code> + * <pre> + * <Not> + * <PropertyIsEqualTo>... + * </Not> + * </pre> + * </code> + * + * @author William Wall + * + */ +public class Not extends AbstractLogicalOp implements IOperation { + + @Override + public boolean execute(Map<String, String> row) { + // For typical NOT behavior, a NOT group should have one child. If it has more than one child, it behaves + // like NOR. NAND/NOR behavior can be implemented by giving the Not group a child group of AND/OR. + boolean result = true; + for (int i = 0; i < children.size(); i++) { + result = !children.get(i).execute(row); + // in the case that there are multiple children, treat them as NOR + if (!result) break; + } + return result; + } +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/Or.java ---------------------------------------------------------------------- diff --git a/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/Or.java b/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/Or.java new file mode 100644 index 0000000..0a3dd6e --- /dev/null +++ b/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/Or.java @@ -0,0 +1,29 @@ +package ss.cloudbase.core.iterators.filter.ogc.operation; + +import java.util.Map; + +/** + * Executes a logical OR on the child operations. + * + * <code> + * <pre> + * <Or> + * <PropertyIsEqualTo>... + * <PropertyIsLessThan>... + * </Or> + * </pre> + * </code> + * + * @author William Wall + */ +public class Or extends AbstractLogicalOp implements IOperation { + @Override + public boolean execute(Map<String, String> row) { + boolean result = false; + for (int i = 0; i < children.size(); i++) { + result = children.get(i).execute(row); + if (result) break; + } + return result; + } +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/PropertyIsBetween.java ---------------------------------------------------------------------- diff --git a/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/PropertyIsBetween.java b/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/PropertyIsBetween.java new file mode 100644 index 0000000..2c9d86c --- /dev/null +++ b/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/PropertyIsBetween.java @@ -0,0 +1,76 @@ +package ss.cloudbase.core.iterators.filter.ogc.operation; + +import java.util.List; +import java.util.Map; + +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +import ss.cloudbase.core.iterators.filter.ogc.OGCFilter; + + +/** + * An operation that determines if the row's value is between the given + * boundary values. + * + * Example: + * <pre> + * <PropertyIsBetween> + * <PropertyName>height</PropertyName> + * <LowerBoundary><Literal>180</Literal></LowerBoundary> + * <UpperBoundary><Literal>200</Literal></UpperBoundary> + * </PropertyIsBetween> + * </pre> + * + * @author William Wall + */ +public class PropertyIsBetween implements IOperation { + String name; + String lower, upper, value; + double lowerNum, upperNum, valueNum; + boolean isNumeric = false; + + @Override + public boolean execute(Map<String, String> row) { + if (isNumeric) { + valueNum = AbstractComparisonOp.parseNumeric(row.get(name)); + if (valueNum != Double.NaN) { + return lowerNum <= valueNum && valueNum <= upperNum; + } + } + + value = row.get(name); + if (value == null) { + value = ""; + } + + return value.compareTo(lower) > -1 && value.compareTo(upper) < 1; + } + + @Override + public List<IOperation> getChildren() { + return null; + } + + @Override + public void init(Node node, String compareType) { + Node child; + NodeList children = node.getChildNodes(); + for (int i = 0; i < children.getLength(); i++) { + child = children.item(i); + if (child.getNodeName().equalsIgnoreCase("PropertyName")) { + name = child.getTextContent(); + } else if (child.getNodeName().equalsIgnoreCase("LowerBoundary")) { + lower = child.getTextContent(); + } else if (child.getNodeName().equalsIgnoreCase("UpperBoundary")) { + upper = child.getTextContent(); + } + } + + if (compareType.equalsIgnoreCase(OGCFilter.TYPE_NUMERIC) || compareType.equalsIgnoreCase(OGCFilter.TYPE_AUTO)) { + upperNum = AbstractComparisonOp.parseNumeric(upper); + lowerNum = AbstractComparisonOp.parseNumeric(lower); + isNumeric = !Double.isNaN(upperNum) && !Double.isNaN(lowerNum); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/PropertyIsEqualTo.java ---------------------------------------------------------------------- diff --git a/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/PropertyIsEqualTo.java b/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/PropertyIsEqualTo.java new file mode 100644 index 0000000..93fa3f5 --- /dev/null +++ b/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/PropertyIsEqualTo.java @@ -0,0 +1,30 @@ +package ss.cloudbase.core.iterators.filter.ogc.operation; + +import java.util.Map; + +/** + * An operation to see whether the values are equal or not. + * + * Example: + * <pre> + * <PropertyIsEqualTo> + * <PropertyName>user</PropertyIsEqualTo> + * <Literal>CmdrTaco</Literal> + * </PropertyIsEqualTo> + * </pre> + * + * @author William Wall + */ +public class PropertyIsEqualTo extends AbstractComparisonOp implements IOperation { + + @Override + public boolean execute(Map<String, String> row) { + value = getValue(row); + + if (checkRowNumeric(value)) { + return valueNum == literalNum; + } + + return value.equals(literal); + } +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/PropertyIsGreaterThan.java ---------------------------------------------------------------------- diff --git a/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/PropertyIsGreaterThan.java b/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/PropertyIsGreaterThan.java new file mode 100644 index 0000000..8eb9ec0 --- /dev/null +++ b/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/PropertyIsGreaterThan.java @@ -0,0 +1,29 @@ +package ss.cloudbase.core.iterators.filter.ogc.operation; + +import java.util.Map; + +/** + * An operation to see if the row value is greater than the given value. + * + * Example: + * <pre> + * <PropertyIsGreaterThan> + * <PropertyName>height</PropertyName> + * <Literal>200</Literal> + * </PropertyIsGreaterThan> + * </pre> + * @author William Wall + */ +public class PropertyIsGreaterThan extends AbstractComparisonOp implements IOperation { + + @Override + public boolean execute(Map<String, String> row) { + value = getValue(row); + + if (checkRowNumeric(value)) { + return valueNum > literalNum; + } + + return value.compareTo(literal) > 0; + } +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/PropertyIsGreaterThanOrEqualTo.java ---------------------------------------------------------------------- diff --git a/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/PropertyIsGreaterThanOrEqualTo.java b/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/PropertyIsGreaterThanOrEqualTo.java new file mode 100644 index 0000000..17fb1dc --- /dev/null +++ b/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/PropertyIsGreaterThanOrEqualTo.java @@ -0,0 +1,29 @@ +package ss.cloudbase.core.iterators.filter.ogc.operation; + +import java.util.Map; + +/** + * An operation to see if the row value is greater than or equal to the given value. + * + * Example: + * <pre> + * <PropertyIsGreaterThanOrEqualTo> + * <PropertyName>height</PropertyName> + * <Literal>100</Literal> + * </PropertyIsGreaterThanOrEqualTo> + * </pre> + * @author William Wall + */ +public class PropertyIsGreaterThanOrEqualTo extends AbstractComparisonOp implements IOperation { + + @Override + public boolean execute(Map<String, String> row) { + value = getValue(row); + + if (checkRowNumeric(value)) { + return valueNum >= literalNum; + } + + return value.compareTo(literal) > -1; + } +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/PropertyIsLessThan.java ---------------------------------------------------------------------- diff --git a/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/PropertyIsLessThan.java b/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/PropertyIsLessThan.java new file mode 100644 index 0000000..f094c99 --- /dev/null +++ b/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/PropertyIsLessThan.java @@ -0,0 +1,31 @@ +package ss.cloudbase.core.iterators.filter.ogc.operation; + +import java.util.Map; + +/** + * An operation to see if the row value is less than the given value. + * + * Example: + * <pre> + * <PropertyIsLessThan> + * <PropertyName>height</PropertyName> + * <Literal>180</Literal> + * </PropertyIsLessThan> + * </pre> + * + * @author William Wall + */ +public class PropertyIsLessThan extends AbstractComparisonOp implements IOperation { + + @Override + public boolean execute(Map<String, String> row) { + value = getValue(row); + + if (checkRowNumeric(value)) { + return valueNum < literalNum; + } + + return value.compareTo(literal) < 0; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/PropertyIsLessThanOrEqualTo.java ---------------------------------------------------------------------- diff --git a/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/PropertyIsLessThanOrEqualTo.java b/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/PropertyIsLessThanOrEqualTo.java new file mode 100644 index 0000000..9b01aae --- /dev/null +++ b/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/PropertyIsLessThanOrEqualTo.java @@ -0,0 +1,29 @@ +package ss.cloudbase.core.iterators.filter.ogc.operation; + +import java.util.Map; + +/** + * An operation to see if the row value is less than or equal to the given value. + * + * <pre> + * <PropertyIsLessThanOrEqualTo> + * <PropertyName>height</PropertyName> + * <Literal>100</Literal> + * </PropertyIsLessThanOrEqualTo> + * </pre> + * + * @author William Wall + */ +public class PropertyIsLessThanOrEqualTo extends AbstractComparisonOp implements IOperation { + + @Override + public boolean execute(Map<String, String> row) { + value = getValue(row); + + if (checkRowNumeric(value)) { + return valueNum <= literalNum; + } + + return value.compareTo(literal) < 1; + } +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/PropertyIsLike.java ---------------------------------------------------------------------- diff --git a/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/PropertyIsLike.java b/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/PropertyIsLike.java new file mode 100644 index 0000000..ad38951 --- /dev/null +++ b/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/PropertyIsLike.java @@ -0,0 +1,144 @@ +package ss.cloudbase.core.iterators.filter.ogc.operation; + +import java.util.List; +import java.util.Map; + +import org.w3c.dom.NamedNodeMap; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +/** + * An operation that determines if the row value is like the given value. This + * operation supports wildcards (*). + * + * Example: + * + * <pre> + * <PropertyIsLike> + * <PropertyName>city</PropertyName> + * <Literal>new*</Literal> + * </PropertyIsLike> + * </pre> + * + * @author William Wall + * + */ +public class PropertyIsLike implements IOperation { + String pattern; + String name; + + @Override + public boolean execute(Map<String, String> row) { + String value = row.get(name); + if (value == null) { + value = ""; + } + + return value.matches(pattern); + } + + @Override + public List<IOperation> getChildren() { + return null; + } + + @Override + public void init(Node node, String compareType) { + Node child; + NodeList children = node.getChildNodes(); + for (int i = 0; i < children.getLength(); i++) { + child = children.item(i); + if (child.getNodeName().equalsIgnoreCase("PropertyName")) { + name = child.getTextContent(); + } else { + pattern = child.getTextContent(); + } + } + + pattern = convertToRegex(node, pattern); + } + + /** + * Converts the pattern, wild card, single and escape characters to the + * regular expression equivalents. Everything else in the pattern is treated + * as a regex literal. + * + * @param node The PropertyIsLike node + * @param likePattern The initial like pattern + * + * @return the equivalent regular expression string. + */ + public String convertToRegex(Node node, String likePattern) { + // Convert the pattern to a regular expression. + StringBuilder regex = new StringBuilder(); + + NamedNodeMap attr = node.getAttributes(); + + String wildCard = "*"; + if (attr.getNamedItem("wildCard") != null) { + wildCard = attr.getNamedItem("wildCard").toString(); + } + + String escapeChar = "\\"; + if (attr.getNamedItem("escapeChar") != null) { + escapeChar = attr.getNamedItem("escapeChar").toString(); + } + + String singleChar = "."; + if (attr.getNamedItem("singleChar") != null) { + singleChar = attr.getNamedItem("singleChar").toString(); + } + + int escapeCharIndex = likePattern.indexOf(escapeChar); + + // These are required in WFS but we'll handle null values here. + int wildCardIndex = wildCard == null ? -1 : likePattern.indexOf(wildCard); + int singleCharIndex = singleChar == null ? -1 : likePattern.indexOf(singleChar); + for (int index = 0; index < likePattern.length(); index++) { + char ch = likePattern.charAt(index); + if (index == escapeCharIndex) { + escapeCharIndex = likePattern.indexOf(escapeChar, escapeCharIndex + escapeChar.length()); + + // If there are consecutive escape characters, skip to the + // next one to save it in the regex. + if (index + 1 == escapeCharIndex) { + escapeCharIndex = likePattern.indexOf(escapeChar, escapeCharIndex + escapeChar.length()); + } else if (index + 1 == wildCardIndex) { + wildCardIndex = likePattern.indexOf(wildCard, wildCardIndex + wildCard.length()); + } else if (index + 1 == singleCharIndex) { + singleCharIndex = likePattern.indexOf(singleChar, singleCharIndex + singleChar.length()); + } else { + // This is an undefined condition, just skip the escape + // character. + } + } + + // Insert the regular expression equivalent of a wild card. + else if (index == wildCardIndex) { + regex.append(".*"); + index += wildCard.length() - 1; + wildCardIndex = likePattern.indexOf(wildCard, wildCardIndex + wildCard.length()); + } + + // Insert the regular expression equivalent of the single char. + else if (index == singleCharIndex) { + regex.append("."); + index += singleChar.length() - 1; + singleCharIndex = likePattern.indexOf(singleChar, singleCharIndex + singleChar.length()); + } + + // Handle certain characters in a special manner. + else if (('[' == ch) || (']' == ch) || ('\\' == ch) || ('^' == ch)) { + regex.append('\\').append(ch); + } + + // Force everything else to be literals. + else { + regex.append('[').append(ch).append(']'); + } + } + + // add case insensitive flag and start match at beginning of the string + return "(?i)^" + regex.toString(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/PropertyIsNotEqualTo.java ---------------------------------------------------------------------- diff --git a/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/PropertyIsNotEqualTo.java b/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/PropertyIsNotEqualTo.java new file mode 100644 index 0000000..b62e6c6 --- /dev/null +++ b/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/PropertyIsNotEqualTo.java @@ -0,0 +1,30 @@ +package ss.cloudbase.core.iterators.filter.ogc.operation; + +import java.util.Map; + +/** + * An operation that determines if the row value is not equal to the given value. + * + * Example: + * <pre> + * <PropertyIsNotEqualTo> + * <PropertyName>weather</PropertyName> + * <Literal>rainy</Literal> + * </PropertyIsNotEqualTo> + * </pre> + * + * @author William Wall + * + */ +public class PropertyIsNotEqualTo extends AbstractComparisonOp implements IOperation { + @Override + public boolean execute(Map<String, String> row) { + value = getValue(row); + + if (checkRowNumeric(value)) { + return valueNum != literalNum; + } + + return !value.equals(literal); + } +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/PropertyIsNull.java ---------------------------------------------------------------------- diff --git a/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/PropertyIsNull.java b/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/PropertyIsNull.java new file mode 100644 index 0000000..d5d67c2 --- /dev/null +++ b/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/PropertyIsNull.java @@ -0,0 +1,38 @@ +package ss.cloudbase.core.iterators.filter.ogc.operation; + +import java.util.List; +import java.util.Map; + +import org.w3c.dom.Node; + +/** + * An operation to determine if the row value is null. Nulls and empty strings will both match. + * + * Example: + * <pre> + * <PropertyIsNull> + * <PropertyName>socialSkills</PropertyName> + * </PropertyIsNull> + * </pre> + * + * @author William Wall + */ +public class PropertyIsNull implements IOperation { + String name; + + @Override + public boolean execute(Map<String, String> row) { + String value = row.get(name); + return value == null || value.length() == 0; + } + + @Override + public List<IOperation> getChildren() { + return null; + } + + @Override + public void init(Node node, String compareType) { + name = node.getTextContent(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/ShapeFactory.java ---------------------------------------------------------------------- diff --git a/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/ShapeFactory.java b/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/ShapeFactory.java new file mode 100644 index 0000000..10fea78 --- /dev/null +++ b/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/operation/ShapeFactory.java @@ -0,0 +1,133 @@ +package ss.cloudbase.core.iterators.filter.ogc.operation; + +import java.awt.Rectangle; +import java.awt.Shape; +import java.awt.geom.Ellipse2D; +import java.awt.geom.Path2D; +import java.awt.geom.Point2D; + +import org.apache.log4j.Logger; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +import ss.cloudbase.core.iterators.filter.ogc.util.GeoUtil; + + +public class ShapeFactory { + private static final Logger logger = Logger.getLogger(ShapeFactory.class); + + public static Shape getShape(Node node) { + if (node.getNodeName().equalsIgnoreCase("gml:Envelope")) { + return parseEnvelope(node); + } else if (node.getNodeName().equalsIgnoreCase("gml:Polygon")) { + return parsePolygon(node); + } else if (node.getNodeName().equalsIgnoreCase("gml:CircleByCenterPoint")) { + return parseCircle(node); + } + + logger.warn("No parser implemented for: " + node.getLocalName()); + return null; + } + + protected static Shape parseEnvelope(Node node) { + Rectangle rect = null; + + Node child; + NodeList children = node.getChildNodes(); + for (int i = 0; i < children.getLength(); i++) { + child = children.item(i); + String[] parts = child.getTextContent().split("\\s"); + + if (parts.length == 2) { + double lon = Double.parseDouble(parts[0]); + double lat = Double.parseDouble(parts[1]); + + if (rect == null) { + rect = new Rectangle(); + rect.setFrame(lon, lat, 0, 0); + } else { + rect.add(lon, lat); + } + } + } + + // If the rectangle width is greater than 180 degrees, the user most likely + // meant to use the inverse BBOX (where the east value is less than the west). + // This is for clients that wrap coordinates rather than use absolute coordinates. + if (rect.getWidth() > 180) { + rect.setFrame(rect.getMaxX(), rect.getMaxY(), 360 - rect.getWidth(), rect.getHeight()); + } + + return rect; + } + + protected static Shape parsePolygon(Node node) { + Path2D poly = null; + + String text = node.getTextContent(); + String[] list = text.split("\\s"); + + for (int i = 1; i < list.length; i += 2) { + double lon = Double.parseDouble(list[i-1]); + double lat = Double.parseDouble(list[i]); + if (poly == null) { + poly = new Path2D.Double(); + poly.moveTo(lon, lat); + } else { + poly.lineTo(lon, lat); + } + } + + return poly; + } + + protected static Shape parseCircle(Node node) { + Ellipse2D circle = null; + + double radius = Double.NaN, lon = Double.NaN, lat = Double.NaN; + String units = null; + + Node child; + NodeList children = node.getChildNodes(); + try { + for (int i = 0; i < children.getLength(); i++) { + child = children.item(i); + if (child.getNodeName().equalsIgnoreCase("gml:radius")) { + radius = Double.parseDouble(child.getTextContent()); + units = child.getAttributes().getNamedItem("uom").getTextContent(); + } else { + String[] list = child.getTextContent().split("\\s"); + lon = Double.parseDouble(list[0]); + lat = Double.parseDouble(list[1]); + } + } + + radius = convertToKM(radius, units); + Point2D center = new Point2D.Double(lon, lat); + Point2D end = GeoUtil.calculateEndLocation(center, radius, lat > 0 ? 180: 0); + + radius = Math.abs(end.getY() - lat); + circle = new Ellipse2D.Double(); + circle.setFrameFromCenter(center, new Point2D.Double(center.getX() + radius, center.getY() + radius)); + } catch (NumberFormatException e) { + + } catch (ArrayIndexOutOfBoundsException e) { + + } + + return circle; + } + + private static double convertToKM(double radius, String units) { + if (units.equalsIgnoreCase("km")) { + return radius; + } else if (units.equalsIgnoreCase("m")) { + return radius / 1000; + } else if (units.equalsIgnoreCase("mi")) { + return 0.621371192 * radius; + } else if (units.equalsIgnoreCase("ft")) { + return radius / 3280.8399; + } + return radius; + } +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/util/GeoUtil.java ---------------------------------------------------------------------- diff --git a/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/util/GeoUtil.java b/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/util/GeoUtil.java new file mode 100644 index 0000000..95259fa --- /dev/null +++ b/partition/common-query/src/main/java/ss/cloudbase/core/iterators/filter/ogc/util/GeoUtil.java @@ -0,0 +1,32 @@ +package ss.cloudbase.core.iterators.filter.ogc.util; + +import java.awt.geom.Point2D; + +public class GeoUtil { + /** + * Calculates an ending location from a point, distance, and bearing + * @param point The start point + * @param distance The distance from the start point in kilometers + * @param bearing The bearing (in degrees) where north is 0 + * @return The resulting point + */ + public static Point2D calculateEndLocation(Point2D point, double distance, double bearing) { + double r = 6371; // earth's mean radius in km + + double lon1 = Math.toRadians(point.getX()); + double lat1 = Math.toRadians(point.getY()); + bearing = Math.toRadians(bearing); + + double lat2 = Math.asin( Math.sin(lat1) * Math.cos(distance/r) + Math.cos(lat1) * Math.sin(distance/r) * Math.cos(bearing) ); + double lon2 = lon1 + Math.atan2(Math.sin(bearing) * Math.sin(distance/r) * Math.cos(lat1), Math.cos(distance/r) - Math.sin(lat1) * Math.sin(lat2)); + + lon2 = (lon2+Math.PI)%(2*Math.PI) - Math.PI; // normalise to -180...+180 + + if (Double.isNaN(lat2) || Double.isNaN(lon2)) return null; + + lon2 = Math.toDegrees(lon2); + lat2 = Math.toDegrees(lat2); + + return new Point2D.Double(lon2, lat2); + } +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/partition/common-query/src/test/java/GVDateFilterTest.java ---------------------------------------------------------------------- diff --git a/partition/common-query/src/test/java/GVDateFilterTest.java b/partition/common-query/src/test/java/GVDateFilterTest.java new file mode 100644 index 0000000..8ea5578 --- /dev/null +++ b/partition/common-query/src/test/java/GVDateFilterTest.java @@ -0,0 +1,156 @@ +/* + * To change this template, choose Tools | Templates + * and open the template in the editor. + */ + +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.Map.Entry; + +import org.apache.hadoop.io.Text; +import org.junit.Test; + +import ss.cloudbase.core.iterators.GMDenIntersectingIterator; +import ss.cloudbase.core.iterators.filter.general.GVDateFilter; + +import cloudbase.core.client.Connector; +import cloudbase.core.client.Scanner; +import cloudbase.core.client.TableNotFoundException; +import cloudbase.core.data.Key; +import cloudbase.core.data.Range; +import cloudbase.core.data.Value; +import cloudbase.core.iterators.FilteringIterator; +import cloudbase.core.security.Authorizations; + +/** + * + * @author rashah + */ +public class GVDateFilterTest +{ + + private Connector cellLevelConn; + private Connector serializedConn; + private static final String TABLE = "partition"; + private static final Authorizations AUTHS = new Authorizations("ALPHA,BETA,GAMMA".split(",")); + + + + protected Connector getSerializedConnector() + { + if (serializedConn == null) + { + serializedConn = SampleGVData.initConnector(); + SampleGVData.writeDenSerialized(serializedConn, SampleGVData.sampleData()); + } + return serializedConn; + } + + + + protected Scanner getSerializedScanner() + { + Connector c = getSerializedConnector(); + try + { + return c.createScanner(TABLE, AUTHS); + } + catch (TableNotFoundException e) + { + return null; + } + } + + protected Scanner setUpGVDFFilter(Scanner s, String timesta) + { + try + { + + s.setScanIterators(50, FilteringIterator.class.getName(), "gvdf"); + s.setScanIteratorOption("gvdf", "0", GVDateFilter.class.getName()); + s.setScanIteratorOption("gvdf", "0." + GVDateFilter.OPTIONInTimestamp, timesta); + + } + catch (IOException e) + { + // TODO Auto-generated catch block + e.printStackTrace(); + } + return s; + } + + protected String checkSerialized(Scanner s) + { + StringBuilder sb = new StringBuilder(); + boolean first = true; + for (Entry<Key, Value> e : s) + { + + if (!first) + { + sb.append(","); + } + else + { + first = false; + } + + String colq = e.getKey().getColumnQualifier().toString(); + + sb.append(colq); + } + return sb.toString(); + } + + + @Test + public void testNoResults() + { + + Scanner s = setUpGVDFFilter(getSerializedScanner(), "2008-03-03T20:44:28.633Z"); + s.setRange(new Range()); + + assertTrue(checkSerialized(s).equals("")); + } + + + @Test + public void testOneResult() + { + + Scanner s = setUpGVDFFilter(getSerializedScanner(), "2011-03-03T20:44:28.633Z"); + s.setRange(new Range()); + + System.out.println(checkSerialized(s)); + + assertTrue(checkSerialized(s).equals("03")); + } + + @Test + public void testTwoResults() + { + + Scanner s = setUpGVDFFilter(getSerializedScanner(), "2009-03-03T20:44:28.633Z"); + s.setRange(new Range()); + + assertTrue(checkSerialized(s).equals("04,01")); + } + + @Test + public void testThreeResults() + { + + Scanner s = setUpGVDFFilter(getSerializedScanner(), "2010-03-01T20:44:28.633Z"); + s.setRange(new Range()); + + assertTrue(checkSerialized(s).equals("04,01,03")); + } + + @Test + public void testDummyTest() + { + assertTrue(true); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/partition/common-query/src/test/java/GVFrequencyFilterTest.java ---------------------------------------------------------------------- diff --git a/partition/common-query/src/test/java/GVFrequencyFilterTest.java b/partition/common-query/src/test/java/GVFrequencyFilterTest.java new file mode 100644 index 0000000..25c602a --- /dev/null +++ b/partition/common-query/src/test/java/GVFrequencyFilterTest.java @@ -0,0 +1,144 @@ +/* + * To change this template, choose Tools | Templates + * and open the template in the editor. + */ + +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.Map.Entry; + +import org.apache.hadoop.io.Text; +import org.junit.Test; + +import ss.cloudbase.core.iterators.GMDenIntersectingIterator; +import ss.cloudbase.core.iterators.filter.general.GVFrequencyFilter; + +import cloudbase.core.client.Connector; +import cloudbase.core.client.Scanner; +import cloudbase.core.client.TableNotFoundException; +import cloudbase.core.data.Key; +import cloudbase.core.data.Range; +import cloudbase.core.data.Value; +import cloudbase.core.iterators.FilteringIterator; +import cloudbase.core.security.Authorizations; + +/** + * + * @author rashah + */ +public class GVFrequencyFilterTest +{ + + private Connector cellLevelConn; + private Connector serializedConn; + private static final String TABLE = "partition"; + private static final Authorizations AUTHS = new Authorizations("ALPHA,BETA,GAMMA".split(",")); + + + + protected Connector getSerializedConnector() + { + if (serializedConn == null) + { + serializedConn = SampleGVData.initConnector(); + SampleGVData.writeDenSerialized(serializedConn, SampleGVData.sampleData()); + } + return serializedConn; + } + + + + protected Scanner getSerializedScanner() + { + Connector c = getSerializedConnector(); + try + { + return c.createScanner(TABLE, AUTHS); + } + catch (TableNotFoundException e) + { + return null; + } + } + + protected Scanner setUpGVDFFilter(Scanner s, String Frequency) + { + try + { + s.clearScanIterators(); + + s.setScanIterators(50, FilteringIterator.class.getName(), "gvff"); + s.setScanIteratorOption("gvff", "0", GVFrequencyFilter.class.getName()); + s.setScanIteratorOption("gvff", "0." + GVFrequencyFilter.OPTIONFrequency, Frequency); + + } + catch (IOException e) + { + // TODO Auto-generated catch block + e.printStackTrace(); + } + return s; + } + + protected String checkSerialized(Scanner s) + { + StringBuilder sb = new StringBuilder(); + boolean first = true; + for (Entry<Key, Value> e : s) + { + + if (!first) + { + sb.append(","); + } + else + { + first = false; + } + + String colq = e.getKey().getColumnQualifier().toString(); + + //System.out.println(e.getKey()+"\t"+e.getValue()); + + sb.append(colq); + } + return sb.toString(); + } + + @Test + public void testNoMatch() + { + + Scanner s = setUpGVDFFilter(getSerializedScanner(), "2000000000"); + s.setRange(new Range()); + + assertTrue(checkSerialized(s).isEmpty()); + } + + @Test + public void testSingleMatch() + { + Scanner s = setUpGVDFFilter(getSerializedScanner(), "1500000000"); + s.setRange(new Range()); + + assertTrue(checkSerialized(s).equals("01")); + } + + + @Test + public void testDoubleMatch() + { + Scanner s = setUpGVDFFilter(getSerializedScanner(), "1200000000"); + s.setRange(new Range()); + + assertTrue(checkSerialized(s).equals("01,03")); + } + + @Test + public void testDummyTest() + { + assertTrue(true); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/partition/common-query/src/test/java/IteratorTest.java ---------------------------------------------------------------------- diff --git a/partition/common-query/src/test/java/IteratorTest.java b/partition/common-query/src/test/java/IteratorTest.java new file mode 100644 index 0000000..1b5cf14 --- /dev/null +++ b/partition/common-query/src/test/java/IteratorTest.java @@ -0,0 +1,554 @@ +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.hadoop.io.Text; +import org.junit.Test; + +import ss.cloudbase.core.iterators.CellLevelFilteringIterator; +import ss.cloudbase.core.iterators.CellLevelRecordIterator; +import ss.cloudbase.core.iterators.ConversionIterator; +import ss.cloudbase.core.iterators.GMDenIntersectingIterator; +import ss.cloudbase.core.iterators.SortedMinIterator; +import ss.cloudbase.core.iterators.SortedRangeIterator; +import ss.cloudbase.core.iterators.UniqueIterator; +import ss.cloudbase.core.iterators.filter.CBConverter; +import cloudbase.core.client.Connector; +import cloudbase.core.client.Scanner; +import cloudbase.core.client.TableNotFoundException; +import cloudbase.core.data.Key; +import cloudbase.core.data.PartialKey; +import cloudbase.core.data.Range; +import cloudbase.core.data.Value; +import cloudbase.core.security.Authorizations; + +public class IteratorTest { + private Connector cellLevelConn; + private Connector serializedConn; + + private static final String TABLE = "partition"; + private static final Authorizations AUTHS = new Authorizations("ALPHA,BETA,GAMMA".split(",")); + + public IteratorTest() { + + } + + protected Connector getCellLevelConnector() { + if (cellLevelConn == null) { + cellLevelConn = SampleData.initConnector(); + SampleData.writeDenCellLevel(cellLevelConn, SampleData.sampleData()); + } + return cellLevelConn; + } + + protected Connector getSerializedConnector() { + if (serializedConn == null) { + serializedConn = SampleData.initConnector(); + SampleData.writeDenSerialized(serializedConn, SampleData.sampleData()); + SampleData.writeDenProvenance(serializedConn); + SampleData.writeMinIndexes(serializedConn); + } + return serializedConn; + } + + protected Scanner getProvenanceScanner() { + Connector c = getSerializedConnector(); + try { + return c.createScanner("provenance", AUTHS); + } catch (TableNotFoundException e) { + return null; + } + } + + protected Scanner getCellLevelScanner() { + Connector c = getCellLevelConnector(); + try { + return c.createScanner(TABLE, AUTHS); + } catch (TableNotFoundException e) { + return null; + } + } + + protected Scanner getSerializedScanner() { + Connector c = getSerializedConnector(); + try { + return c.createScanner(TABLE, AUTHS); + } catch (TableNotFoundException e) { + return null; + } + } + + protected Scanner setUpIntersectingIterator(Scanner s, Text[] terms, boolean multiDoc) { + try { + s.setScanIterators(50, GMDenIntersectingIterator.class.getName(), "ii"); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + s.setScanIteratorOption("ii", GMDenIntersectingIterator.indexFamilyOptionName, "index"); + s.setScanIteratorOption("ii", GMDenIntersectingIterator.docFamilyOptionName, "event"); + s.setScanIteratorOption("ii", GMDenIntersectingIterator.OPTION_MULTI_DOC, "" + multiDoc); + s.setScanIteratorOption("ii", GMDenIntersectingIterator.columnFamiliesOptionName, GMDenIntersectingIterator.encodeColumns(terms)); + return s; + } + + protected String checkSerialized(Scanner s) { + StringBuilder sb = new StringBuilder(); + boolean first = true; + for (Entry<Key, Value> e: s) { + if (!first) { + sb.append(","); + } else { + first = false; + } + + String colq = e.getKey().getColumnQualifier().toString(); + + sb.append(colq); + } + return sb.toString(); + } + + protected String checkCellLevel(Scanner s) { + StringBuilder sb = new StringBuilder(); + boolean first = true; + for (Entry<Key, Value> e: s) { + String colq = e.getKey().getColumnQualifier().toString(); + int i = colq.indexOf("\u0000"); + if (i > -1) { + if (!first) { + sb.append(","); + } else { + first = false; + } + sb.append(colq.substring(0, i)); + sb.append("."); + sb.append(colq.substring(i + 1)); + sb.append("="); + sb.append(e.getValue().toString()); + } + } + return sb.toString(); + } + + @Test + public void testSerializedSingleDuplicate() { + Text[] terms = new Text[] { + new Text("A"), + new Text("A") + }; + + String test = "01"; + Scanner s = setUpIntersectingIterator(getSerializedScanner(), terms, false); + s.setRange(new Range()); + assertTrue(test.equals(checkSerialized(s))); + } + + @Test + public void testCellLevelSingleDuplicate() { + Text[] terms = new Text[] { + new Text("A"), + new Text("A") + }; + String test = "01.field0=A,01.field1=B,01.field2=C,01.field3=D,01.field4=E"; + Scanner s = setUpIntersectingIterator(getCellLevelScanner(), terms, true); + s.setRange(new Range()); + assertTrue(test.equals(checkCellLevel(s))); + } + + @Test + public void testSerializedTwoTerms() { + Text[] terms = new Text[] { + new Text("C"), + new Text("D") + }; + // all the evens will come first + String test = "02,01,03"; + Scanner s = setUpIntersectingIterator(getSerializedScanner(), terms, false); + s.setRange(new Range()); + assertTrue(test.equals(checkSerialized(s))); + } + + @Test + public void testCellLevelTwoTerms() { + Text[] terms = new Text[] { + new Text("C"), + new Text("D") + }; + + String test = "02.field0=B,02.field1=C,02.field2=D,02.field3=E,02.field4=F," + + "01.field0=A,01.field1=B,01.field2=C,01.field3=D,01.field4=E," + + "03.field0=C,03.field1=D,03.field2=E,03.field3=F,03.field4=G"; + Scanner s = setUpIntersectingIterator(getCellLevelScanner(), terms, true); + s.setRange(new Range()); + assertTrue(test.equals(checkCellLevel(s))); + } + + @Test + public void testSerializedTwoTermsWithRange() { + Text[] terms = new Text[] { + new Text("C"), + new Text("D") + }; + + String test = "02"; + Scanner s = setUpIntersectingIterator(getSerializedScanner(), terms, false); + s.setRange(new Range(new Key(new Text("0")), true, new Key(new Text("1")), false)); + assertTrue(test.equals(checkSerialized(s))); + } + + @Test + public void testCellLevelTwoTermsWithRange() { + Text[] terms = new Text[] { + new Text("C"), + new Text("D") + }; + + String test = "02.field0=B,02.field1=C,02.field2=D,02.field3=E,02.field4=F"; + Scanner s = setUpIntersectingIterator(getCellLevelScanner(), terms, true); + s.setRange(new Range(new Key(new Text("0")), true, new Key(new Text("1")), false)); + assertTrue(test.equals(checkCellLevel(s))); + } + + @Test + public void testSerializedSingleRange() { + Text[] terms = new Text[] { + new Text(GMDenIntersectingIterator.getRangeTerm("index", "A", true, "B", true)), + new Text(GMDenIntersectingIterator.getRangeTerm("index", "A", true, "B", true)) + }; + + String test = "02,01"; + Scanner s = setUpIntersectingIterator(getSerializedScanner(), terms, false); + s.setRange(new Range()); + assertTrue(test.equals(checkSerialized(s))); + } + + @Test + public void testSerializedMultiRange() { + Text[] terms = new Text[] { + new Text(GMDenIntersectingIterator.getRangeTerm("index", "A", true, "B", true)), + new Text(GMDenIntersectingIterator.getRangeTerm("index", "B", true, "C", true)) + }; + + String test = "02,01"; + Scanner s = setUpIntersectingIterator(getSerializedScanner(), terms, false); + s.setRange(new Range()); + assertTrue(test.equals(checkSerialized(s))); + } + + @Test + public void testSerializedTermAndRange() { + Text[] terms = new Text[] { + new Text("B"), + new Text(GMDenIntersectingIterator.getRangeTerm("index", "A", true, "E", true)) + }; + + String test = "02,01"; + Scanner s = setUpIntersectingIterator(getSerializedScanner(), terms, false); + s.setRange(new Range()); + assertTrue(test.equals(checkSerialized(s))); + } + + protected Scanner setUpSortedRangeIterator(Scanner s, boolean multiDoc) { + try { + s.setScanIterators(50, SortedRangeIterator.class.getName(), "ri"); + s.setScanIteratorOption("ri", SortedRangeIterator.OPTION_COLF, "index"); + s.setScanIteratorOption("ri", SortedRangeIterator.OPTION_DOC_COLF, "event"); + s.setScanIteratorOption("ri", SortedRangeIterator.OPTION_LOWER_BOUND, "A"); + s.setScanIteratorOption("ri", SortedRangeIterator.OPTION_UPPER_BOUND, "C"); + s.setScanIteratorOption("ri", SortedRangeIterator.OPTION_START_INCLUSIVE, "true"); + s.setScanIteratorOption("ri", SortedRangeIterator.OPTION_END_INCLUSIVE, "true"); + s.setScanIteratorOption("ri", SortedRangeIterator.OPTION_MULTI_DOC, "" + multiDoc); + return s; + } catch (IOException e) { + e.printStackTrace(); + return null; + } + } + + @Test + public void testSerializedSortedRangeIterator() { + Scanner s = setUpSortedRangeIterator(getSerializedScanner(), false); + String test = "02,01,03"; + s.setRange(new Range()); + assertTrue(test.equals(checkSerialized(s))); + } + + @Test + public void testCellLevelSortedRangeIterator() { + Scanner s = setUpSortedRangeIterator(getCellLevelScanner(), true); + String test = "02.field0=B,02.field1=C,02.field2=D,02.field3=E,02.field4=F," + + "01.field0=A,01.field1=B,01.field2=C,01.field3=D,01.field4=E," + + "03.field0=C,03.field1=D,03.field2=E,03.field3=F,03.field4=G"; + s.setRange(new Range()); + assertTrue(test.equals(checkCellLevel(s))); + } + + @Test + public void testUniqueIterator() { + Scanner s = getProvenanceScanner(); + try { + s.setScanIterators(50, UniqueIterator.class.getName(), "skipper"); + Key start = new Key(new Text("sid1")); + s.setRange(new Range(start, start.followingKey(PartialKey.ROW))); + + int count = 0; + for (Entry<Key, Value> e: s) { + count++; + } + + assertEquals(count, 3); + } catch (IOException e) { + e.printStackTrace(); + } + } + + protected Scanner setUpConversionIterator(Scanner s) { + String[] conversions = new String[] { + "field0 + 10", + "field1 - 10", + "field2 * 10", + "field3 / 10", + "field4 % 10" + }; + + try { + s.setScanIterators(50, ConversionIterator.class.getName(), "ci"); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + s.setScanIteratorOption("ci", ConversionIterator.OPTION_CONVERSIONS, ConversionIterator.encodeConversions(conversions)); + Key start = new Key(new Text("1"), new Text("event"), new Text("01")); + s.setRange(new Range(start, true, start.followingKey(PartialKey.ROW_COLFAM_COLQUAL), false)); + + return s; + } + + @Test + public void testConversionIteratorSerialized() { + Scanner s = getSerializedScanner(); + s = setUpConversionIterator(s); + + CBConverter c = new CBConverter(); + + boolean test = true; + Map<String, Double> expected = new HashMap<String, Double>(); + + expected.put("field0", 20.0); + expected.put("field1", 1.0); + expected.put("field2", 120.0); + expected.put("field3", 1.3); + expected.put("field4", 4.0); + + Map<String, String> record; + + for (Entry<Key, Value> e: s) { + record = c.toMap(e.getKey(), e.getValue()); + + for (Entry<String, String> pair: record.entrySet()) { + test = test && expected.get(pair.getKey()).equals(new Double(Double.parseDouble(record.get(pair.getKey())))); + } + } + + assertTrue(test); + } + + @Test + public void testConversionIteratorCellLevel() { + Scanner s = getCellLevelScanner(); + s = setUpConversionIterator(s); + s.setScanIteratorOption("ci", ConversionIterator.OPTION_MULTI_DOC, "true"); + + boolean test = true; + Map<String, Double> expected = new HashMap<String, Double>(); + + expected.put("field0", 20.0); + expected.put("field1", 1.0); + expected.put("field2", 120.0); + expected.put("field3", 1.3); + expected.put("field4", 4.0); + + for (Entry<Key, Value> e: s) { + String field = getField(e.getKey()); + if (field != null) { + test = test && expected.get(field).equals(new Double(Double.parseDouble(e.getValue().toString()))); + } + } + + assertTrue(test); + } + + protected String getField(Key key) { + String colq = key.getColumnQualifier().toString(); + int start = colq.indexOf("\u0000"); + if (start == -1) { + return null; + } + + int end = colq.indexOf("\u0000", start + 1); + if (end == -1) { + end = colq.length(); + } + + return colq.substring(start + 1, end); + } + + @Test + public void testCellLevelOGCFilter() { + Scanner s = getCellLevelScanner(); + s.fetchColumnFamily(new Text("event")); + + try { + s.setScanIterators(60, CellLevelFilteringIterator.class.getName(), "fi"); + } catch (IOException e) { + e.printStackTrace(); + } + + s.setScanIteratorOption("fi", CellLevelFilteringIterator.OPTION_FILTER, "<PropertyIsBetween><PropertyName>field0</PropertyName>" + + "<LowerBoundary><Literal>A</Literal></LowerBoundary>" + + "<UpperBoundary><Literal>C</Literal></UpperBoundary>" + + "</PropertyIsBetween>"); + + String test = "02.field0=B,02.field1=C,02.field2=D,02.field3=E,02.field4=F," + + "01.field0=A,01.field1=B,01.field2=C,01.field3=D,01.field4=E," + + "03.field0=C,03.field1=D,03.field2=E,03.field3=F,03.field4=G"; + assertTrue(test.equals(checkCellLevel(s))); + } + + @Test + public void testMultiLevelIterator() { + Scanner s = getCellLevelScanner(); + Text[] terms = new Text[] { + new Text("C"), + new Text("D") + }; + + s = setUpIntersectingIterator(s, terms, true); + + try { + s.setScanIterators(60, CellLevelFilteringIterator.class.getName(), "fi"); + } catch (IOException e) { + e.printStackTrace(); + } + + s.setScanIteratorOption("fi", CellLevelFilteringIterator.OPTION_FILTER, "<PropertyIsEqualTo><PropertyName>field0</PropertyName>" + + "<Literal>A</Literal>" + + "</PropertyIsEqualTo>"); + + String test = "01.field0=A,01.field1=B,01.field2=C,01.field3=D,01.field4=E"; + assertTrue(test.equals(checkCellLevel(s))); + } + + @Test + public void testMultiLevelIterator2() { + Scanner s = getCellLevelScanner(); + s = setUpSortedRangeIterator(s, true); + try { + s.setScanIterators(60, CellLevelFilteringIterator.class.getName(), "fi"); + } catch (IOException e) { + e.printStackTrace(); + } + s.setScanIteratorOption("fi", CellLevelFilteringIterator.OPTION_FILTER, "<PropertyIsEqualTo><PropertyName>field0</PropertyName>" + + "<Literal>A</Literal>" + + "</PropertyIsEqualTo>"); + + String test = "01.field0=A,01.field1=B,01.field2=C,01.field3=D,01.field4=E"; + assertTrue(test.equals(checkCellLevel(s))); + } + + @Test + public void testCellLevelRecordIterator() { + Scanner s = getCellLevelScanner(); + s = setUpSortedRangeIterator(s, true); + try { + s.setScanIterators(60, CellLevelRecordIterator.class.getName(), "recordItr"); + } catch (IOException e) { + e.printStackTrace(); + } + +// for (Entry<Key, Value> e: s) { +// String v = e.getValue().toString(); +// v = v.replaceAll("\\u0000", ","); +// v = v.replaceAll("\\uFFFD", "="); +// System.out.println(e.getKey() + "\t" + v); +// } + String test = "02,01,03"; + assertTrue(test.equals(checkSerialized(s))); + } + + @Test + public void testIntersectionWithoutDocLookup() { + Text[] terms = new Text[] { + new Text("C"), + new Text("D") + }; + // all the evens will come first + String test = "\u000002,\u000001,\u000003"; + Scanner s = setUpIntersectingIterator(getSerializedScanner(), terms, false); + s.setScanIteratorOption("ii", GMDenIntersectingIterator.OPTION_DOC_LOOKUP, "false"); + s.setRange(new Range()); + assertTrue(test.equals(checkSerialized(s))); + } + + @Test + public void testSimpleNot() { + Text[] terms = new Text[] { + new Text("B"), + new Text("F") + }; + + boolean[] nots = new boolean[] { + false, + true + }; + + String test="01"; + Scanner s = setUpIntersectingIterator(getSerializedScanner(), terms, false); + s.setScanIteratorOption("ii", GMDenIntersectingIterator.notFlagOptionName, GMDenIntersectingIterator.encodeBooleans(nots)); + s.setRange(new Range()); + + assertTrue(test.equals(checkSerialized(s))); + } + + @Test + public void testRangeNot() { + Text[] terms = new Text[] { + new Text("B"), + new Text(GMDenIntersectingIterator.getRangeTerm("index", "F", true, "H", true)) + }; + + boolean[] nots = new boolean[] { + false, + true + }; + + String test = "01"; + Scanner s = setUpIntersectingIterator(getSerializedScanner(), terms, false); + s.setScanIteratorOption("ii", GMDenIntersectingIterator.notFlagOptionName, GMDenIntersectingIterator.encodeBooleans(nots)); + s.setRange(new Range()); + + assertTrue(test.equals(checkSerialized(s))); + } + + @Test + public void testMinIteratorOnLastKeys() { + Scanner s = getSerializedScanner(); + try { + s.setScanIterators(50, SortedMinIterator.class.getName(), "min"); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + s.setScanIteratorOption("min", SortedMinIterator.OPTION_PREFIX, "z"); + s.setRange(new Range()); + + String test = "02,04,06,08,10,01,03,05,07,09"; + assertTrue(test.equals(checkSerialized(s))); + } +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/partition/common-query/src/test/java/JTSFilterTest.java ---------------------------------------------------------------------- diff --git a/partition/common-query/src/test/java/JTSFilterTest.java b/partition/common-query/src/test/java/JTSFilterTest.java new file mode 100644 index 0000000..8224f64 --- /dev/null +++ b/partition/common-query/src/test/java/JTSFilterTest.java @@ -0,0 +1,181 @@ +/* + * To change this template, choose Tools | Templates + * and open the template in the editor. + */ + +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.Map.Entry; + +import org.apache.hadoop.io.Text; +import org.junit.Test; + +import ss.cloudbase.core.iterators.GMDenIntersectingIterator; +import ss.cloudbase.core.iterators.filter.jts.JTSFilter; + +import cloudbase.core.client.Connector; +import cloudbase.core.client.Scanner; +import cloudbase.core.client.TableNotFoundException; +import cloudbase.core.data.Key; +import cloudbase.core.data.Range; +import cloudbase.core.data.Value; +import cloudbase.core.iterators.FilteringIterator; +import cloudbase.core.security.Authorizations; + +/** + * + * @author rashah + */ +public class JTSFilterTest +{ + + private Connector cellLevelConn; + private Connector serializedConn; + private static final String TABLE = "partition"; + private static final Authorizations AUTHS = new Authorizations("ALPHA,BETA,GAMMA".split(",")); + + + + protected Connector getSerializedConnector() + { + if (serializedConn == null) + { + serializedConn = SampleJTSData.initConnector(); + SampleJTSData.writeDenSerialized(serializedConn, SampleJTSData.sampleData()); + } + return serializedConn; + } + + + + protected Scanner getSerializedScanner() + { + Connector c = getSerializedConnector(); + try + { + return c.createScanner(TABLE, AUTHS); + } + catch (TableNotFoundException e) + { + return null; + } + } + + protected Scanner setUpJTSFilter(Scanner s, String latitude, String longitude, boolean change_name) + { + try + { + + s.setScanIterators(50, FilteringIterator.class.getName(), "gvdf"); + s.setScanIteratorOption("gvdf", "0", JTSFilter.class.getName()); + s.setScanIteratorOption("gvdf", "0." + JTSFilter.OPTIONCenterPointLat, latitude); + s.setScanIteratorOption("gvdf", "0." + JTSFilter.OPTIONCenterPointLon, longitude); + if (change_name) + s.setScanIteratorOption("gvdf", "0." + JTSFilter.OPTIONGeometryKeyName, "beam-footprint"); + + + } + catch (IOException e) + { + // TODO Auto-generated catch block + e.printStackTrace(); + } + return s; + } + + protected String checkSerialized(Scanner s) + { + StringBuilder sb = new StringBuilder(); + boolean first = true; + for (Entry<Key, Value> e : s) + { + + if (!first) + { + sb.append(","); + } + else + { + first = false; + } + + String colq = e.getKey().getColumnQualifier().toString(); + + sb.append(colq); + } + return sb.toString(); + } + + + @Test + public void testNoResults() + { + //London is in niether - 51°30'0.00"N 0° 7'0.00"W + String latitude = "51.5"; + String longitude = "0.11"; + + Scanner s = setUpJTSFilter(getSerializedScanner(), latitude, longitude, false); + s.setRange(new Range()); + +// System.out.println("{" + checkSerialized(s) + "}"); + assertTrue(checkSerialized(s).isEmpty()); + } + + + @Test + public void testOneResultAmerica() + { + //This is North America + //Points 39°44'21.00"N 104°59'3.00"W (Denver) are in the footprint + String latitude = "33"; + String longitude = "-93.0"; + + Scanner s = setUpJTSFilter(getSerializedScanner(), latitude, longitude, false); + s.setRange(new Range()); + + System.out.println("{" + checkSerialized(s) + "}"); + assertTrue(checkSerialized(s).equals("02")); + } + + + @Test + public void testOneResultAustralia() + { + //This is Australia + //Points like 22S 135E are in the beam + String latitude = "-9"; + String longitude = "100.0"; + + Scanner s = setUpJTSFilter(getSerializedScanner(), latitude, longitude, false); + s.setRange(new Range()); + + System.out.println("{" + checkSerialized(s) + "}"); + assertTrue(checkSerialized(s).equals("01")); + } + + @Test + public void testOneResultHawaii() + { + // -164 40 - somewhere near hawaii + + //This is Australia + //Points like 22S 135E are in the beam + String latitude = "40"; + String longitude = "-164.0"; + + Scanner s = setUpJTSFilter(getSerializedScanner(), latitude, longitude, true); + s.setRange(new Range()); + + System.out.println("{" + checkSerialized(s) + "}"); + assertTrue(checkSerialized(s).equals("03")); + } + + + @Test + public void testDummyTest() + { + assertTrue(true); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/partition/common-query/src/test/java/OGCFilterTest.java ---------------------------------------------------------------------- diff --git a/partition/common-query/src/test/java/OGCFilterTest.java b/partition/common-query/src/test/java/OGCFilterTest.java new file mode 100644 index 0000000..fd54945 --- /dev/null +++ b/partition/common-query/src/test/java/OGCFilterTest.java @@ -0,0 +1,163 @@ +import java.util.HashMap; +import java.util.Map; + +import org.apache.hadoop.io.Text; +import org.junit.Test; + +import cloudbase.core.data.Key; +import cloudbase.core.data.Value; +import ss.cloudbase.core.iterators.filter.ogc.OGCFilter; + +import static org.junit.Assert.*; + +public class OGCFilterTest { + private Key testKey = new Key(new Text("row"), new Text("colf"), new Text("colq")); + private Value testValue = new Value("uuid~event\uFFFDmy-event-hash-1\u0000date\uFFFD20100819\u0000time~dss\uFFFD212706.000\u0000frequency\uFFFD3.368248181443644E8\u0000latitude\uFFFD48.74571142707959\u0000longitude\uFFFD13.865561564126812\u0000altitude\uFFFD1047.0\u0000datetime\uFFFD2010-08-19T21:27:06.000Z\u0000test~key\uFFFD\u0000key\uFFFDa\uFFFDb".getBytes()); + + public OGCFilterTest() { + + } + + private OGCFilter getFilter(String filter) { + OGCFilter f = new OGCFilter(); + Map<String, String> options = new HashMap<String, String>(); + options.put(OGCFilter.OPTION_FILTER, filter); + f.init(options); + return f; + } + + @Test + public void testBBOX() { + OGCFilter f = getFilter("<BBOX><gml:Envelope>" + + "<gml:LowerCorner>13 48</gml:LowerCorner>" + + "<gml:UpperCorner>14 49</gml:UpperCorner>" + + "</gml:Envelope></BBOX>"); + assertTrue(f.accept(testKey, testValue)); + } + + @Test + public void testBetweenStr() { + OGCFilter f = getFilter("<PropertyIsBetween><PropertyName>datetime</PropertyName>" + + "<LowerBoundary><Literal>2010-08-19</Literal></LowerBoundary>" + + "<UpperBoundary><Literal>2010-08-20</Literal></UpperBoundary>" + + "</PropertyIsBetween>"); + + assertTrue(f.accept(testKey, testValue)); + } + + @Test + public void testBetweenNum() { + OGCFilter f = getFilter("<PropertyIsBetween><PropertyName>frequency</PropertyName>" + + "<LowerBoundary><Literal>330000000</Literal></LowerBoundary>" + + "<UpperBoundary><Literal>340000000</Literal></UpperBoundary>" + + "</PropertyIsBetween>"); + + assertTrue(f.accept(testKey, testValue)); + } + + @Test + public void testEqualStr() { + OGCFilter f = getFilter("<PropertyIsEqualTo><PropertyName>uuid~event</PropertyName><Literal>my-event-hash-1</Literal></PropertyIsEqualTo>"); + assertTrue(f.accept(testKey, testValue)); + } + + @Test + public void testEqualNum() { + OGCFilter f = getFilter("<PropertyIsEqualTo><PropertyName>altitude</PropertyName><Literal>1047</Literal></PropertyIsEqualTo>"); + assertTrue(f.accept(testKey, testValue)); + } + + @Test + public void testGreaterThanStr() { + OGCFilter f = getFilter("<PropertyIsGreaterThan><PropertyName>datetime</PropertyName><Literal>2010-08-15</Literal></PropertyIsGreaterThan>"); + assertTrue(f.accept(testKey, testValue)); + } + + @Test + public void testGreaterThanNum() { + OGCFilter f = getFilter("<PropertyIsGreaterThan><PropertyName>altitude</PropertyName><Literal>1000</Literal></PropertyIsGreaterThan>"); + assertTrue(f.accept(testKey, testValue)); + } + + @Test + public void testLessThanStr() { + OGCFilter f = getFilter("<PropertyIsLessThan><PropertyName>datetime</PropertyName><Literal>2010-08-20</Literal></PropertyIsLessThan>"); + assertTrue(f.accept(testKey, testValue)); + } + + @Test + public void testLessThanNum() { + OGCFilter f = getFilter("<PropertyIsLessThan><PropertyName>altitude</PropertyName><Literal>1200</Literal></PropertyIsLessThan>"); + assertTrue(f.accept(testKey, testValue)); + } + + @Test + public void testLike() { + OGCFilter f = getFilter("<PropertyIsLike><PropertyName>uuid~event</PropertyName><Literal>*event*</Literal></PropertyIsLike>"); + assertTrue(f.accept(testKey, testValue)); + } + + @Test + public void testNotEqualNum() { + OGCFilter f = getFilter("<PropertyIsNotEqualTo><PropertyName>altitude</PropertyName><Literal>1046</Literal></PropertyIsNotEqualTo>"); + assertTrue(f.accept(testKey, testValue)); + } + + @Test + public void testNull() { + OGCFilter f = getFilter("<PropertyIsNull><PropertyName>test~key</PropertyName></PropertyIsNull>"); + assertTrue(f.accept(testKey, testValue)); + } + + @Test + public void testNot() { + OGCFilter f = getFilter("<Not><PropertyIsEqualTo><PropertyName>altitude</PropertyName><Literal>1047</Literal></PropertyIsEqualTo></Not>"); + assertFalse(f.accept(testKey, testValue)); + } + + @Test + public void testAnd() { + OGCFilter f = getFilter("<And>" + + "<PropertyIsEqualTo><PropertyName>altitude</PropertyName><Literal>1047</Literal></PropertyIsEqualTo>" + + "<PropertyIsNull><PropertyName>test~key</PropertyName></PropertyIsNull>" + + "</And>"); + + assertTrue(f.accept(testKey, testValue)); + } + + @Test + public void testOr() { + OGCFilter f = getFilter("<Or>" + + "<PropertyIsLike><PropertyName>uuid~event</PropertyName><Literal>*event*</Literal></PropertyIsLike>" + + "<PropertyIsNull><PropertyName>uuid~event</PropertyName></PropertyIsNull>" + + "</Or>"); + + assertTrue(f.accept(testKey, testValue)); + } + + @Test + public void testNand() { + OGCFilter f = getFilter("<Not><And>" + + "<PropertyIsNull><PropertyName>uuid~event</PropertyName></PropertyIsNull>" + + "<PropertyIsNull><PropertyName>test~key</PropertyName></PropertyIsNull>" + + "</And></Not>"); + + assertTrue(f.accept(testKey, testValue)); + } + + @Test + public void testNor() { + OGCFilter f = getFilter("<Not>" + + "<PropertyIsNull><PropertyName>uuid~event</PropertyName></PropertyIsNull>" + + "<PropertyIsNull><PropertyName>altitude</PropertyName></PropertyIsNull>" + + "</Not>"); + + assertTrue(f.accept(testKey, testValue)); + } + + @Test + public void testParse() { + OGCFilter f = getFilter("<PropertyIsEqualTo><PropertyName>key</PropertyName><Literal>a</Literal></PropertyIsEqualTo>"); + assertTrue(f.accept(testKey, testValue)); + } +}
