http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/osgi/sesame-runtime-osgi/pom.xml ---------------------------------------------------------------------- diff --git a/osgi/sesame-runtime-osgi/pom.xml b/osgi/sesame-runtime-osgi/pom.xml new file mode 100644 index 0000000..db1a4bb --- /dev/null +++ b/osgi/sesame-runtime-osgi/pom.xml @@ -0,0 +1,116 @@ +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>mvm.rya</groupId> + <artifactId>rya.osgi</artifactId> + <version>3.0.4-SNAPSHOT</version> + </parent> + <artifactId>sesame-runtime-osgi</artifactId> + <version>2.6.4</version> + <packaging>pom</packaging> + <name>${project.groupId}.${project.artifactId}</name> + <dependencies> + <dependency> + <groupId>org.openrdf.sesame</groupId> + <artifactId>sesame-runtime-osgi</artifactId> + </dependency> + <dependency> + <groupId>biz.aQute</groupId> + <artifactId>bnd</artifactId> + <version>0.0.397</version> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <executions> + <execution> + <id>copy</id> + <phase>generate-resources</phase> + <goals> + <goal>copy</goal> + </goals> + <configuration> + <artifactItems> + <artifactItem> + <groupId>org.openrdf.sesame</groupId> + <artifactId>sesame-runtime-osgi</artifactId> + <version>${project.version}</version> + <outputDirectory>${project.build.directory}</outputDirectory> + <destFileName>sesame-runtime-osgi.jar</destFileName> + </artifactItem> + <artifactItem> + <groupId>biz.aQute</groupId> + <artifactId>bnd</artifactId> + <version>0.0.397</version> + <outputDirectory>${project.build.directory}</outputDirectory> + <destFileName>bnd.jar</destFileName> + </artifactItem> + </artifactItems> + <!-- other configurations here --> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <version>1.1</version> + <executions> + <execution> + <phase>process-resources</phase> + <goals> + <goal>exec</goal> + </goals> + </execution> + </executions> + <configuration> + <executable>java</executable> + <arguments> + <argument>-jar</argument> + <argument>target/bnd.jar</argument> + <argument>build</argument> + <argument>openrdf-sesame-osgi.bnd</argument> + </arguments> + </configuration> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>build-helper-maven-plugin + </artifactId> + <version>1.7</version> + <executions> + <execution> + <id>attach-artifacts</id> + <phase>package</phase> + <goals> + <goal>attach-artifact</goal> + </goals> + <configuration> + <artifacts> + <artifact> + <file>${project.build.directory}/sesame-runtime-osgi-${project.version}.jar</file> + </artifact> + </artifacts> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + <repositories> + <repository> + <releases> + <enabled>true</enabled> + </releases> + <snapshots> + <enabled>true</enabled> + </snapshots> + <id>bndrepo</id> + <name>aQute BND Repo</name> + <url>http://www.aqute.biz/repo</url> + </repository> + </repositories> +</project>
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/partition/common-query-ext/pom.xml ---------------------------------------------------------------------- diff --git a/partition/common-query-ext/pom.xml b/partition/common-query-ext/pom.xml new file mode 100644 index 0000000..4fb0aee --- /dev/null +++ b/partition/common-query-ext/pom.xml @@ -0,0 +1,71 @@ +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <!--<parent>--> + <!--<groupId>sitestore</groupId>--> + <!--<artifactId>sitestore</artifactId>--> + <!--<version>2.0.0-SNAPSHOT</version>--> + <!--</parent>--> + + <parent> + <groupId>mvm.rya</groupId> + <artifactId>parent</artifactId> + <version>2.0.0-SNAPSHOT</version> + </parent> + + <groupId>sitestore.common</groupId> + <artifactId>common-query-ext</artifactId> + <name>common-query (${project.version})</name> + <version>1.0.0-SNAPSHOT</version> + <description>A set of filters and iterators for cloudbase queries</description> + + <properties> + <skipTests>true</skipTests> + </properties> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-source-plugin</artifactId> + <version>2.1.2</version> + <executions> + <execution> + <id>attach-sources</id> + <phase>install</phase> + <goals> + <goal>jar</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <version>2.7.2</version> + <configuration> + <skipTests>${skipTests}</skipTests> + </configuration> + </plugin> + </plugins> + </build> + <!--<scm>--> + <!--<connection>${scmLocation}/tto/ss/common/trunk/common-query</connection>--> + <!--</scm>--> + <dependencies> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + </dependency> + <dependency> + <groupId>sitestore.common</groupId> + <artifactId>common-query</artifactId> + <version>2.0.0-SNAPSHOT</version> + </dependency> + <dependency> + <groupId>cloudbase</groupId> + <artifactId>cloudbase-core</artifactId> + </dependency> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/partition/common-query-ext/src/main/java/ss/cloudbase/core/iterators/ext/EncodedSortedRangeIterator.java ---------------------------------------------------------------------- diff --git a/partition/common-query-ext/src/main/java/ss/cloudbase/core/iterators/ext/EncodedSortedRangeIterator.java b/partition/common-query-ext/src/main/java/ss/cloudbase/core/iterators/ext/EncodedSortedRangeIterator.java new file mode 100644 index 0000000..fb59102 --- /dev/null +++ b/partition/common-query-ext/src/main/java/ss/cloudbase/core/iterators/ext/EncodedSortedRangeIterator.java @@ -0,0 +1,44 @@ +package ss.cloudbase.core.iterators.ext; + +import cloudbase.core.data.Key; +import cloudbase.core.data.Value; +import cloudbase.core.iterators.IteratorEnvironment; +import cloudbase.core.iterators.SortedKeyValueIterator; +import org.apache.commons.codec.binary.Base64; +import org.apache.hadoop.io.Text; +import ss.cloudbase.core.iterators.SortedRangeIterator; + +import java.io.IOException; +import java.util.Map; + +/** + * Class EncodedSortedRangeIterator + * Date: Aug 10, 2011 + * Time: 10:37:28 AM + */ +public class EncodedSortedRangeIterator extends SortedRangeIterator { + + @Override + public void init(SortedKeyValueIterator<Key, Value> source, Map<String, String> options, IteratorEnvironment env) throws IOException { + super.init(source, options, env); + if (options.containsKey(OPTION_LOWER_BOUND)) { + lower = new Text(decode(options.get(OPTION_LOWER_BOUND))); + } else { + lower = new Text("\u0000"); + } + + if (options.containsKey(OPTION_UPPER_BOUND)) { + upper = new Text(decode(options.get(OPTION_UPPER_BOUND))); + } else { + upper = new Text("\u0000"); + } + } + + public static String encode(String str) { + return new String(Base64.encodeBase64(str.getBytes())); + } + + public static String decode(String str) { + return new String(Base64.decodeBase64(str.getBytes())); + } +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/partition/common-query-ext/src/test/java/GVDateFilterTest.java ---------------------------------------------------------------------- diff --git a/partition/common-query-ext/src/test/java/GVDateFilterTest.java b/partition/common-query-ext/src/test/java/GVDateFilterTest.java new file mode 100644 index 0000000..8ea5578 --- /dev/null +++ b/partition/common-query-ext/src/test/java/GVDateFilterTest.java @@ -0,0 +1,156 @@ +/* + * To change this template, choose Tools | Templates + * and open the template in the editor. + */ + +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.Map.Entry; + +import org.apache.hadoop.io.Text; +import org.junit.Test; + +import ss.cloudbase.core.iterators.GMDenIntersectingIterator; +import ss.cloudbase.core.iterators.filter.general.GVDateFilter; + +import cloudbase.core.client.Connector; +import cloudbase.core.client.Scanner; +import cloudbase.core.client.TableNotFoundException; +import cloudbase.core.data.Key; +import cloudbase.core.data.Range; +import cloudbase.core.data.Value; +import cloudbase.core.iterators.FilteringIterator; +import cloudbase.core.security.Authorizations; + +/** + * + * @author rashah + */ +public class GVDateFilterTest +{ + + private Connector cellLevelConn; + private Connector serializedConn; + private static final String TABLE = "partition"; + private static final Authorizations AUTHS = new Authorizations("ALPHA,BETA,GAMMA".split(",")); + + + + protected Connector getSerializedConnector() + { + if (serializedConn == null) + { + serializedConn = SampleGVData.initConnector(); + SampleGVData.writeDenSerialized(serializedConn, SampleGVData.sampleData()); + } + return serializedConn; + } + + + + protected Scanner getSerializedScanner() + { + Connector c = getSerializedConnector(); + try + { + return c.createScanner(TABLE, AUTHS); + } + catch (TableNotFoundException e) + { + return null; + } + } + + protected Scanner setUpGVDFFilter(Scanner s, String timesta) + { + try + { + + s.setScanIterators(50, FilteringIterator.class.getName(), "gvdf"); + s.setScanIteratorOption("gvdf", "0", GVDateFilter.class.getName()); + s.setScanIteratorOption("gvdf", "0." + GVDateFilter.OPTIONInTimestamp, timesta); + + } + catch (IOException e) + { + // TODO Auto-generated catch block + e.printStackTrace(); + } + return s; + } + + protected String checkSerialized(Scanner s) + { + StringBuilder sb = new StringBuilder(); + boolean first = true; + for (Entry<Key, Value> e : s) + { + + if (!first) + { + sb.append(","); + } + else + { + first = false; + } + + String colq = e.getKey().getColumnQualifier().toString(); + + sb.append(colq); + } + return sb.toString(); + } + + + @Test + public void testNoResults() + { + + Scanner s = setUpGVDFFilter(getSerializedScanner(), "2008-03-03T20:44:28.633Z"); + s.setRange(new Range()); + + assertTrue(checkSerialized(s).equals("")); + } + + + @Test + public void testOneResult() + { + + Scanner s = setUpGVDFFilter(getSerializedScanner(), "2011-03-03T20:44:28.633Z"); + s.setRange(new Range()); + + System.out.println(checkSerialized(s)); + + assertTrue(checkSerialized(s).equals("03")); + } + + @Test + public void testTwoResults() + { + + Scanner s = setUpGVDFFilter(getSerializedScanner(), "2009-03-03T20:44:28.633Z"); + s.setRange(new Range()); + + assertTrue(checkSerialized(s).equals("04,01")); + } + + @Test + public void testThreeResults() + { + + Scanner s = setUpGVDFFilter(getSerializedScanner(), "2010-03-01T20:44:28.633Z"); + s.setRange(new Range()); + + assertTrue(checkSerialized(s).equals("04,01,03")); + } + + @Test + public void testDummyTest() + { + assertTrue(true); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/partition/common-query-ext/src/test/java/GVFrequencyFilterTest.java ---------------------------------------------------------------------- diff --git a/partition/common-query-ext/src/test/java/GVFrequencyFilterTest.java b/partition/common-query-ext/src/test/java/GVFrequencyFilterTest.java new file mode 100644 index 0000000..25c602a --- /dev/null +++ b/partition/common-query-ext/src/test/java/GVFrequencyFilterTest.java @@ -0,0 +1,144 @@ +/* + * To change this template, choose Tools | Templates + * and open the template in the editor. + */ + +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.Map.Entry; + +import org.apache.hadoop.io.Text; +import org.junit.Test; + +import ss.cloudbase.core.iterators.GMDenIntersectingIterator; +import ss.cloudbase.core.iterators.filter.general.GVFrequencyFilter; + +import cloudbase.core.client.Connector; +import cloudbase.core.client.Scanner; +import cloudbase.core.client.TableNotFoundException; +import cloudbase.core.data.Key; +import cloudbase.core.data.Range; +import cloudbase.core.data.Value; +import cloudbase.core.iterators.FilteringIterator; +import cloudbase.core.security.Authorizations; + +/** + * + * @author rashah + */ +public class GVFrequencyFilterTest +{ + + private Connector cellLevelConn; + private Connector serializedConn; + private static final String TABLE = "partition"; + private static final Authorizations AUTHS = new Authorizations("ALPHA,BETA,GAMMA".split(",")); + + + + protected Connector getSerializedConnector() + { + if (serializedConn == null) + { + serializedConn = SampleGVData.initConnector(); + SampleGVData.writeDenSerialized(serializedConn, SampleGVData.sampleData()); + } + return serializedConn; + } + + + + protected Scanner getSerializedScanner() + { + Connector c = getSerializedConnector(); + try + { + return c.createScanner(TABLE, AUTHS); + } + catch (TableNotFoundException e) + { + return null; + } + } + + protected Scanner setUpGVDFFilter(Scanner s, String Frequency) + { + try + { + s.clearScanIterators(); + + s.setScanIterators(50, FilteringIterator.class.getName(), "gvff"); + s.setScanIteratorOption("gvff", "0", GVFrequencyFilter.class.getName()); + s.setScanIteratorOption("gvff", "0." + GVFrequencyFilter.OPTIONFrequency, Frequency); + + } + catch (IOException e) + { + // TODO Auto-generated catch block + e.printStackTrace(); + } + return s; + } + + protected String checkSerialized(Scanner s) + { + StringBuilder sb = new StringBuilder(); + boolean first = true; + for (Entry<Key, Value> e : s) + { + + if (!first) + { + sb.append(","); + } + else + { + first = false; + } + + String colq = e.getKey().getColumnQualifier().toString(); + + //System.out.println(e.getKey()+"\t"+e.getValue()); + + sb.append(colq); + } + return sb.toString(); + } + + @Test + public void testNoMatch() + { + + Scanner s = setUpGVDFFilter(getSerializedScanner(), "2000000000"); + s.setRange(new Range()); + + assertTrue(checkSerialized(s).isEmpty()); + } + + @Test + public void testSingleMatch() + { + Scanner s = setUpGVDFFilter(getSerializedScanner(), "1500000000"); + s.setRange(new Range()); + + assertTrue(checkSerialized(s).equals("01")); + } + + + @Test + public void testDoubleMatch() + { + Scanner s = setUpGVDFFilter(getSerializedScanner(), "1200000000"); + s.setRange(new Range()); + + assertTrue(checkSerialized(s).equals("01,03")); + } + + @Test + public void testDummyTest() + { + assertTrue(true); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/partition/common-query-ext/src/test/java/IteratorTest.java ---------------------------------------------------------------------- diff --git a/partition/common-query-ext/src/test/java/IteratorTest.java b/partition/common-query-ext/src/test/java/IteratorTest.java new file mode 100644 index 0000000..1b5cf14 --- /dev/null +++ b/partition/common-query-ext/src/test/java/IteratorTest.java @@ -0,0 +1,554 @@ +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.hadoop.io.Text; +import org.junit.Test; + +import ss.cloudbase.core.iterators.CellLevelFilteringIterator; +import ss.cloudbase.core.iterators.CellLevelRecordIterator; +import ss.cloudbase.core.iterators.ConversionIterator; +import ss.cloudbase.core.iterators.GMDenIntersectingIterator; +import ss.cloudbase.core.iterators.SortedMinIterator; +import ss.cloudbase.core.iterators.SortedRangeIterator; +import ss.cloudbase.core.iterators.UniqueIterator; +import ss.cloudbase.core.iterators.filter.CBConverter; +import cloudbase.core.client.Connector; +import cloudbase.core.client.Scanner; +import cloudbase.core.client.TableNotFoundException; +import cloudbase.core.data.Key; +import cloudbase.core.data.PartialKey; +import cloudbase.core.data.Range; +import cloudbase.core.data.Value; +import cloudbase.core.security.Authorizations; + +public class IteratorTest { + private Connector cellLevelConn; + private Connector serializedConn; + + private static final String TABLE = "partition"; + private static final Authorizations AUTHS = new Authorizations("ALPHA,BETA,GAMMA".split(",")); + + public IteratorTest() { + + } + + protected Connector getCellLevelConnector() { + if (cellLevelConn == null) { + cellLevelConn = SampleData.initConnector(); + SampleData.writeDenCellLevel(cellLevelConn, SampleData.sampleData()); + } + return cellLevelConn; + } + + protected Connector getSerializedConnector() { + if (serializedConn == null) { + serializedConn = SampleData.initConnector(); + SampleData.writeDenSerialized(serializedConn, SampleData.sampleData()); + SampleData.writeDenProvenance(serializedConn); + SampleData.writeMinIndexes(serializedConn); + } + return serializedConn; + } + + protected Scanner getProvenanceScanner() { + Connector c = getSerializedConnector(); + try { + return c.createScanner("provenance", AUTHS); + } catch (TableNotFoundException e) { + return null; + } + } + + protected Scanner getCellLevelScanner() { + Connector c = getCellLevelConnector(); + try { + return c.createScanner(TABLE, AUTHS); + } catch (TableNotFoundException e) { + return null; + } + } + + protected Scanner getSerializedScanner() { + Connector c = getSerializedConnector(); + try { + return c.createScanner(TABLE, AUTHS); + } catch (TableNotFoundException e) { + return null; + } + } + + protected Scanner setUpIntersectingIterator(Scanner s, Text[] terms, boolean multiDoc) { + try { + s.setScanIterators(50, GMDenIntersectingIterator.class.getName(), "ii"); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + s.setScanIteratorOption("ii", GMDenIntersectingIterator.indexFamilyOptionName, "index"); + s.setScanIteratorOption("ii", GMDenIntersectingIterator.docFamilyOptionName, "event"); + s.setScanIteratorOption("ii", GMDenIntersectingIterator.OPTION_MULTI_DOC, "" + multiDoc); + s.setScanIteratorOption("ii", GMDenIntersectingIterator.columnFamiliesOptionName, GMDenIntersectingIterator.encodeColumns(terms)); + return s; + } + + protected String checkSerialized(Scanner s) { + StringBuilder sb = new StringBuilder(); + boolean first = true; + for (Entry<Key, Value> e: s) { + if (!first) { + sb.append(","); + } else { + first = false; + } + + String colq = e.getKey().getColumnQualifier().toString(); + + sb.append(colq); + } + return sb.toString(); + } + + protected String checkCellLevel(Scanner s) { + StringBuilder sb = new StringBuilder(); + boolean first = true; + for (Entry<Key, Value> e: s) { + String colq = e.getKey().getColumnQualifier().toString(); + int i = colq.indexOf("\u0000"); + if (i > -1) { + if (!first) { + sb.append(","); + } else { + first = false; + } + sb.append(colq.substring(0, i)); + sb.append("."); + sb.append(colq.substring(i + 1)); + sb.append("="); + sb.append(e.getValue().toString()); + } + } + return sb.toString(); + } + + @Test + public void testSerializedSingleDuplicate() { + Text[] terms = new Text[] { + new Text("A"), + new Text("A") + }; + + String test = "01"; + Scanner s = setUpIntersectingIterator(getSerializedScanner(), terms, false); + s.setRange(new Range()); + assertTrue(test.equals(checkSerialized(s))); + } + + @Test + public void testCellLevelSingleDuplicate() { + Text[] terms = new Text[] { + new Text("A"), + new Text("A") + }; + String test = "01.field0=A,01.field1=B,01.field2=C,01.field3=D,01.field4=E"; + Scanner s = setUpIntersectingIterator(getCellLevelScanner(), terms, true); + s.setRange(new Range()); + assertTrue(test.equals(checkCellLevel(s))); + } + + @Test + public void testSerializedTwoTerms() { + Text[] terms = new Text[] { + new Text("C"), + new Text("D") + }; + // all the evens will come first + String test = "02,01,03"; + Scanner s = setUpIntersectingIterator(getSerializedScanner(), terms, false); + s.setRange(new Range()); + assertTrue(test.equals(checkSerialized(s))); + } + + @Test + public void testCellLevelTwoTerms() { + Text[] terms = new Text[] { + new Text("C"), + new Text("D") + }; + + String test = "02.field0=B,02.field1=C,02.field2=D,02.field3=E,02.field4=F," + + "01.field0=A,01.field1=B,01.field2=C,01.field3=D,01.field4=E," + + "03.field0=C,03.field1=D,03.field2=E,03.field3=F,03.field4=G"; + Scanner s = setUpIntersectingIterator(getCellLevelScanner(), terms, true); + s.setRange(new Range()); + assertTrue(test.equals(checkCellLevel(s))); + } + + @Test + public void testSerializedTwoTermsWithRange() { + Text[] terms = new Text[] { + new Text("C"), + new Text("D") + }; + + String test = "02"; + Scanner s = setUpIntersectingIterator(getSerializedScanner(), terms, false); + s.setRange(new Range(new Key(new Text("0")), true, new Key(new Text("1")), false)); + assertTrue(test.equals(checkSerialized(s))); + } + + @Test + public void testCellLevelTwoTermsWithRange() { + Text[] terms = new Text[] { + new Text("C"), + new Text("D") + }; + + String test = "02.field0=B,02.field1=C,02.field2=D,02.field3=E,02.field4=F"; + Scanner s = setUpIntersectingIterator(getCellLevelScanner(), terms, true); + s.setRange(new Range(new Key(new Text("0")), true, new Key(new Text("1")), false)); + assertTrue(test.equals(checkCellLevel(s))); + } + + @Test + public void testSerializedSingleRange() { + Text[] terms = new Text[] { + new Text(GMDenIntersectingIterator.getRangeTerm("index", "A", true, "B", true)), + new Text(GMDenIntersectingIterator.getRangeTerm("index", "A", true, "B", true)) + }; + + String test = "02,01"; + Scanner s = setUpIntersectingIterator(getSerializedScanner(), terms, false); + s.setRange(new Range()); + assertTrue(test.equals(checkSerialized(s))); + } + + @Test + public void testSerializedMultiRange() { + Text[] terms = new Text[] { + new Text(GMDenIntersectingIterator.getRangeTerm("index", "A", true, "B", true)), + new Text(GMDenIntersectingIterator.getRangeTerm("index", "B", true, "C", true)) + }; + + String test = "02,01"; + Scanner s = setUpIntersectingIterator(getSerializedScanner(), terms, false); + s.setRange(new Range()); + assertTrue(test.equals(checkSerialized(s))); + } + + @Test + public void testSerializedTermAndRange() { + Text[] terms = new Text[] { + new Text("B"), + new Text(GMDenIntersectingIterator.getRangeTerm("index", "A", true, "E", true)) + }; + + String test = "02,01"; + Scanner s = setUpIntersectingIterator(getSerializedScanner(), terms, false); + s.setRange(new Range()); + assertTrue(test.equals(checkSerialized(s))); + } + + protected Scanner setUpSortedRangeIterator(Scanner s, boolean multiDoc) { + try { + s.setScanIterators(50, SortedRangeIterator.class.getName(), "ri"); + s.setScanIteratorOption("ri", SortedRangeIterator.OPTION_COLF, "index"); + s.setScanIteratorOption("ri", SortedRangeIterator.OPTION_DOC_COLF, "event"); + s.setScanIteratorOption("ri", SortedRangeIterator.OPTION_LOWER_BOUND, "A"); + s.setScanIteratorOption("ri", SortedRangeIterator.OPTION_UPPER_BOUND, "C"); + s.setScanIteratorOption("ri", SortedRangeIterator.OPTION_START_INCLUSIVE, "true"); + s.setScanIteratorOption("ri", SortedRangeIterator.OPTION_END_INCLUSIVE, "true"); + s.setScanIteratorOption("ri", SortedRangeIterator.OPTION_MULTI_DOC, "" + multiDoc); + return s; + } catch (IOException e) { + e.printStackTrace(); + return null; + } + } + + @Test + public void testSerializedSortedRangeIterator() { + Scanner s = setUpSortedRangeIterator(getSerializedScanner(), false); + String test = "02,01,03"; + s.setRange(new Range()); + assertTrue(test.equals(checkSerialized(s))); + } + + @Test + public void testCellLevelSortedRangeIterator() { + Scanner s = setUpSortedRangeIterator(getCellLevelScanner(), true); + String test = "02.field0=B,02.field1=C,02.field2=D,02.field3=E,02.field4=F," + + "01.field0=A,01.field1=B,01.field2=C,01.field3=D,01.field4=E," + + "03.field0=C,03.field1=D,03.field2=E,03.field3=F,03.field4=G"; + s.setRange(new Range()); + assertTrue(test.equals(checkCellLevel(s))); + } + + @Test + public void testUniqueIterator() { + Scanner s = getProvenanceScanner(); + try { + s.setScanIterators(50, UniqueIterator.class.getName(), "skipper"); + Key start = new Key(new Text("sid1")); + s.setRange(new Range(start, start.followingKey(PartialKey.ROW))); + + int count = 0; + for (Entry<Key, Value> e: s) { + count++; + } + + assertEquals(count, 3); + } catch (IOException e) { + e.printStackTrace(); + } + } + + protected Scanner setUpConversionIterator(Scanner s) { + String[] conversions = new String[] { + "field0 + 10", + "field1 - 10", + "field2 * 10", + "field3 / 10", + "field4 % 10" + }; + + try { + s.setScanIterators(50, ConversionIterator.class.getName(), "ci"); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + s.setScanIteratorOption("ci", ConversionIterator.OPTION_CONVERSIONS, ConversionIterator.encodeConversions(conversions)); + Key start = new Key(new Text("1"), new Text("event"), new Text("01")); + s.setRange(new Range(start, true, start.followingKey(PartialKey.ROW_COLFAM_COLQUAL), false)); + + return s; + } + + @Test + public void testConversionIteratorSerialized() { + Scanner s = getSerializedScanner(); + s = setUpConversionIterator(s); + + CBConverter c = new CBConverter(); + + boolean test = true; + Map<String, Double> expected = new HashMap<String, Double>(); + + expected.put("field0", 20.0); + expected.put("field1", 1.0); + expected.put("field2", 120.0); + expected.put("field3", 1.3); + expected.put("field4", 4.0); + + Map<String, String> record; + + for (Entry<Key, Value> e: s) { + record = c.toMap(e.getKey(), e.getValue()); + + for (Entry<String, String> pair: record.entrySet()) { + test = test && expected.get(pair.getKey()).equals(new Double(Double.parseDouble(record.get(pair.getKey())))); + } + } + + assertTrue(test); + } + + @Test + public void testConversionIteratorCellLevel() { + Scanner s = getCellLevelScanner(); + s = setUpConversionIterator(s); + s.setScanIteratorOption("ci", ConversionIterator.OPTION_MULTI_DOC, "true"); + + boolean test = true; + Map<String, Double> expected = new HashMap<String, Double>(); + + expected.put("field0", 20.0); + expected.put("field1", 1.0); + expected.put("field2", 120.0); + expected.put("field3", 1.3); + expected.put("field4", 4.0); + + for (Entry<Key, Value> e: s) { + String field = getField(e.getKey()); + if (field != null) { + test = test && expected.get(field).equals(new Double(Double.parseDouble(e.getValue().toString()))); + } + } + + assertTrue(test); + } + + protected String getField(Key key) { + String colq = key.getColumnQualifier().toString(); + int start = colq.indexOf("\u0000"); + if (start == -1) { + return null; + } + + int end = colq.indexOf("\u0000", start + 1); + if (end == -1) { + end = colq.length(); + } + + return colq.substring(start + 1, end); + } + + @Test + public void testCellLevelOGCFilter() { + Scanner s = getCellLevelScanner(); + s.fetchColumnFamily(new Text("event")); + + try { + s.setScanIterators(60, CellLevelFilteringIterator.class.getName(), "fi"); + } catch (IOException e) { + e.printStackTrace(); + } + + s.setScanIteratorOption("fi", CellLevelFilteringIterator.OPTION_FILTER, "<PropertyIsBetween><PropertyName>field0</PropertyName>" + + "<LowerBoundary><Literal>A</Literal></LowerBoundary>" + + "<UpperBoundary><Literal>C</Literal></UpperBoundary>" + + "</PropertyIsBetween>"); + + String test = "02.field0=B,02.field1=C,02.field2=D,02.field3=E,02.field4=F," + + "01.field0=A,01.field1=B,01.field2=C,01.field3=D,01.field4=E," + + "03.field0=C,03.field1=D,03.field2=E,03.field3=F,03.field4=G"; + assertTrue(test.equals(checkCellLevel(s))); + } + + @Test + public void testMultiLevelIterator() { + Scanner s = getCellLevelScanner(); + Text[] terms = new Text[] { + new Text("C"), + new Text("D") + }; + + s = setUpIntersectingIterator(s, terms, true); + + try { + s.setScanIterators(60, CellLevelFilteringIterator.class.getName(), "fi"); + } catch (IOException e) { + e.printStackTrace(); + } + + s.setScanIteratorOption("fi", CellLevelFilteringIterator.OPTION_FILTER, "<PropertyIsEqualTo><PropertyName>field0</PropertyName>" + + "<Literal>A</Literal>" + + "</PropertyIsEqualTo>"); + + String test = "01.field0=A,01.field1=B,01.field2=C,01.field3=D,01.field4=E"; + assertTrue(test.equals(checkCellLevel(s))); + } + + @Test + public void testMultiLevelIterator2() { + Scanner s = getCellLevelScanner(); + s = setUpSortedRangeIterator(s, true); + try { + s.setScanIterators(60, CellLevelFilteringIterator.class.getName(), "fi"); + } catch (IOException e) { + e.printStackTrace(); + } + s.setScanIteratorOption("fi", CellLevelFilteringIterator.OPTION_FILTER, "<PropertyIsEqualTo><PropertyName>field0</PropertyName>" + + "<Literal>A</Literal>" + + "</PropertyIsEqualTo>"); + + String test = "01.field0=A,01.field1=B,01.field2=C,01.field3=D,01.field4=E"; + assertTrue(test.equals(checkCellLevel(s))); + } + + @Test + public void testCellLevelRecordIterator() { + Scanner s = getCellLevelScanner(); + s = setUpSortedRangeIterator(s, true); + try { + s.setScanIterators(60, CellLevelRecordIterator.class.getName(), "recordItr"); + } catch (IOException e) { + e.printStackTrace(); + } + +// for (Entry<Key, Value> e: s) { +// String v = e.getValue().toString(); +// v = v.replaceAll("\\u0000", ","); +// v = v.replaceAll("\\uFFFD", "="); +// System.out.println(e.getKey() + "\t" + v); +// } + String test = "02,01,03"; + assertTrue(test.equals(checkSerialized(s))); + } + + @Test + public void testIntersectionWithoutDocLookup() { + Text[] terms = new Text[] { + new Text("C"), + new Text("D") + }; + // all the evens will come first + String test = "\u000002,\u000001,\u000003"; + Scanner s = setUpIntersectingIterator(getSerializedScanner(), terms, false); + s.setScanIteratorOption("ii", GMDenIntersectingIterator.OPTION_DOC_LOOKUP, "false"); + s.setRange(new Range()); + assertTrue(test.equals(checkSerialized(s))); + } + + @Test + public void testSimpleNot() { + Text[] terms = new Text[] { + new Text("B"), + new Text("F") + }; + + boolean[] nots = new boolean[] { + false, + true + }; + + String test="01"; + Scanner s = setUpIntersectingIterator(getSerializedScanner(), terms, false); + s.setScanIteratorOption("ii", GMDenIntersectingIterator.notFlagOptionName, GMDenIntersectingIterator.encodeBooleans(nots)); + s.setRange(new Range()); + + assertTrue(test.equals(checkSerialized(s))); + } + + @Test + public void testRangeNot() { + Text[] terms = new Text[] { + new Text("B"), + new Text(GMDenIntersectingIterator.getRangeTerm("index", "F", true, "H", true)) + }; + + boolean[] nots = new boolean[] { + false, + true + }; + + String test = "01"; + Scanner s = setUpIntersectingIterator(getSerializedScanner(), terms, false); + s.setScanIteratorOption("ii", GMDenIntersectingIterator.notFlagOptionName, GMDenIntersectingIterator.encodeBooleans(nots)); + s.setRange(new Range()); + + assertTrue(test.equals(checkSerialized(s))); + } + + @Test + public void testMinIteratorOnLastKeys() { + Scanner s = getSerializedScanner(); + try { + s.setScanIterators(50, SortedMinIterator.class.getName(), "min"); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + s.setScanIteratorOption("min", SortedMinIterator.OPTION_PREFIX, "z"); + s.setRange(new Range()); + + String test = "02,04,06,08,10,01,03,05,07,09"; + assertTrue(test.equals(checkSerialized(s))); + } +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/partition/common-query-ext/src/test/java/JTSFilterTest.java ---------------------------------------------------------------------- diff --git a/partition/common-query-ext/src/test/java/JTSFilterTest.java b/partition/common-query-ext/src/test/java/JTSFilterTest.java new file mode 100644 index 0000000..8224f64 --- /dev/null +++ b/partition/common-query-ext/src/test/java/JTSFilterTest.java @@ -0,0 +1,181 @@ +/* + * To change this template, choose Tools | Templates + * and open the template in the editor. + */ + +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.Map.Entry; + +import org.apache.hadoop.io.Text; +import org.junit.Test; + +import ss.cloudbase.core.iterators.GMDenIntersectingIterator; +import ss.cloudbase.core.iterators.filter.jts.JTSFilter; + +import cloudbase.core.client.Connector; +import cloudbase.core.client.Scanner; +import cloudbase.core.client.TableNotFoundException; +import cloudbase.core.data.Key; +import cloudbase.core.data.Range; +import cloudbase.core.data.Value; +import cloudbase.core.iterators.FilteringIterator; +import cloudbase.core.security.Authorizations; + +/** + * + * @author rashah + */ +public class JTSFilterTest +{ + + private Connector cellLevelConn; + private Connector serializedConn; + private static final String TABLE = "partition"; + private static final Authorizations AUTHS = new Authorizations("ALPHA,BETA,GAMMA".split(",")); + + + + protected Connector getSerializedConnector() + { + if (serializedConn == null) + { + serializedConn = SampleJTSData.initConnector(); + SampleJTSData.writeDenSerialized(serializedConn, SampleJTSData.sampleData()); + } + return serializedConn; + } + + + + protected Scanner getSerializedScanner() + { + Connector c = getSerializedConnector(); + try + { + return c.createScanner(TABLE, AUTHS); + } + catch (TableNotFoundException e) + { + return null; + } + } + + protected Scanner setUpJTSFilter(Scanner s, String latitude, String longitude, boolean change_name) + { + try + { + + s.setScanIterators(50, FilteringIterator.class.getName(), "gvdf"); + s.setScanIteratorOption("gvdf", "0", JTSFilter.class.getName()); + s.setScanIteratorOption("gvdf", "0." + JTSFilter.OPTIONCenterPointLat, latitude); + s.setScanIteratorOption("gvdf", "0." + JTSFilter.OPTIONCenterPointLon, longitude); + if (change_name) + s.setScanIteratorOption("gvdf", "0." + JTSFilter.OPTIONGeometryKeyName, "beam-footprint"); + + + } + catch (IOException e) + { + // TODO Auto-generated catch block + e.printStackTrace(); + } + return s; + } + + protected String checkSerialized(Scanner s) + { + StringBuilder sb = new StringBuilder(); + boolean first = true; + for (Entry<Key, Value> e : s) + { + + if (!first) + { + sb.append(","); + } + else + { + first = false; + } + + String colq = e.getKey().getColumnQualifier().toString(); + + sb.append(colq); + } + return sb.toString(); + } + + + @Test + public void testNoResults() + { + //London is in niether - 51°30'0.00"N 0° 7'0.00"W + String latitude = "51.5"; + String longitude = "0.11"; + + Scanner s = setUpJTSFilter(getSerializedScanner(), latitude, longitude, false); + s.setRange(new Range()); + +// System.out.println("{" + checkSerialized(s) + "}"); + assertTrue(checkSerialized(s).isEmpty()); + } + + + @Test + public void testOneResultAmerica() + { + //This is North America + //Points 39°44'21.00"N 104°59'3.00"W (Denver) are in the footprint + String latitude = "33"; + String longitude = "-93.0"; + + Scanner s = setUpJTSFilter(getSerializedScanner(), latitude, longitude, false); + s.setRange(new Range()); + + System.out.println("{" + checkSerialized(s) + "}"); + assertTrue(checkSerialized(s).equals("02")); + } + + + @Test + public void testOneResultAustralia() + { + //This is Australia + //Points like 22S 135E are in the beam + String latitude = "-9"; + String longitude = "100.0"; + + Scanner s = setUpJTSFilter(getSerializedScanner(), latitude, longitude, false); + s.setRange(new Range()); + + System.out.println("{" + checkSerialized(s) + "}"); + assertTrue(checkSerialized(s).equals("01")); + } + + @Test + public void testOneResultHawaii() + { + // -164 40 - somewhere near hawaii + + //This is Australia + //Points like 22S 135E are in the beam + String latitude = "40"; + String longitude = "-164.0"; + + Scanner s = setUpJTSFilter(getSerializedScanner(), latitude, longitude, true); + s.setRange(new Range()); + + System.out.println("{" + checkSerialized(s) + "}"); + assertTrue(checkSerialized(s).equals("03")); + } + + + @Test + public void testDummyTest() + { + assertTrue(true); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/partition/common-query-ext/src/test/java/OGCFilterTest.java ---------------------------------------------------------------------- diff --git a/partition/common-query-ext/src/test/java/OGCFilterTest.java b/partition/common-query-ext/src/test/java/OGCFilterTest.java new file mode 100644 index 0000000..fd54945 --- /dev/null +++ b/partition/common-query-ext/src/test/java/OGCFilterTest.java @@ -0,0 +1,163 @@ +import java.util.HashMap; +import java.util.Map; + +import org.apache.hadoop.io.Text; +import org.junit.Test; + +import cloudbase.core.data.Key; +import cloudbase.core.data.Value; +import ss.cloudbase.core.iterators.filter.ogc.OGCFilter; + +import static org.junit.Assert.*; + +public class OGCFilterTest { + private Key testKey = new Key(new Text("row"), new Text("colf"), new Text("colq")); + private Value testValue = new Value("uuid~event\uFFFDmy-event-hash-1\u0000date\uFFFD20100819\u0000time~dss\uFFFD212706.000\u0000frequency\uFFFD3.368248181443644E8\u0000latitude\uFFFD48.74571142707959\u0000longitude\uFFFD13.865561564126812\u0000altitude\uFFFD1047.0\u0000datetime\uFFFD2010-08-19T21:27:06.000Z\u0000test~key\uFFFD\u0000key\uFFFDa\uFFFDb".getBytes()); + + public OGCFilterTest() { + + } + + private OGCFilter getFilter(String filter) { + OGCFilter f = new OGCFilter(); + Map<String, String> options = new HashMap<String, String>(); + options.put(OGCFilter.OPTION_FILTER, filter); + f.init(options); + return f; + } + + @Test + public void testBBOX() { + OGCFilter f = getFilter("<BBOX><gml:Envelope>" + + "<gml:LowerCorner>13 48</gml:LowerCorner>" + + "<gml:UpperCorner>14 49</gml:UpperCorner>" + + "</gml:Envelope></BBOX>"); + assertTrue(f.accept(testKey, testValue)); + } + + @Test + public void testBetweenStr() { + OGCFilter f = getFilter("<PropertyIsBetween><PropertyName>datetime</PropertyName>" + + "<LowerBoundary><Literal>2010-08-19</Literal></LowerBoundary>" + + "<UpperBoundary><Literal>2010-08-20</Literal></UpperBoundary>" + + "</PropertyIsBetween>"); + + assertTrue(f.accept(testKey, testValue)); + } + + @Test + public void testBetweenNum() { + OGCFilter f = getFilter("<PropertyIsBetween><PropertyName>frequency</PropertyName>" + + "<LowerBoundary><Literal>330000000</Literal></LowerBoundary>" + + "<UpperBoundary><Literal>340000000</Literal></UpperBoundary>" + + "</PropertyIsBetween>"); + + assertTrue(f.accept(testKey, testValue)); + } + + @Test + public void testEqualStr() { + OGCFilter f = getFilter("<PropertyIsEqualTo><PropertyName>uuid~event</PropertyName><Literal>my-event-hash-1</Literal></PropertyIsEqualTo>"); + assertTrue(f.accept(testKey, testValue)); + } + + @Test + public void testEqualNum() { + OGCFilter f = getFilter("<PropertyIsEqualTo><PropertyName>altitude</PropertyName><Literal>1047</Literal></PropertyIsEqualTo>"); + assertTrue(f.accept(testKey, testValue)); + } + + @Test + public void testGreaterThanStr() { + OGCFilter f = getFilter("<PropertyIsGreaterThan><PropertyName>datetime</PropertyName><Literal>2010-08-15</Literal></PropertyIsGreaterThan>"); + assertTrue(f.accept(testKey, testValue)); + } + + @Test + public void testGreaterThanNum() { + OGCFilter f = getFilter("<PropertyIsGreaterThan><PropertyName>altitude</PropertyName><Literal>1000</Literal></PropertyIsGreaterThan>"); + assertTrue(f.accept(testKey, testValue)); + } + + @Test + public void testLessThanStr() { + OGCFilter f = getFilter("<PropertyIsLessThan><PropertyName>datetime</PropertyName><Literal>2010-08-20</Literal></PropertyIsLessThan>"); + assertTrue(f.accept(testKey, testValue)); + } + + @Test + public void testLessThanNum() { + OGCFilter f = getFilter("<PropertyIsLessThan><PropertyName>altitude</PropertyName><Literal>1200</Literal></PropertyIsLessThan>"); + assertTrue(f.accept(testKey, testValue)); + } + + @Test + public void testLike() { + OGCFilter f = getFilter("<PropertyIsLike><PropertyName>uuid~event</PropertyName><Literal>*event*</Literal></PropertyIsLike>"); + assertTrue(f.accept(testKey, testValue)); + } + + @Test + public void testNotEqualNum() { + OGCFilter f = getFilter("<PropertyIsNotEqualTo><PropertyName>altitude</PropertyName><Literal>1046</Literal></PropertyIsNotEqualTo>"); + assertTrue(f.accept(testKey, testValue)); + } + + @Test + public void testNull() { + OGCFilter f = getFilter("<PropertyIsNull><PropertyName>test~key</PropertyName></PropertyIsNull>"); + assertTrue(f.accept(testKey, testValue)); + } + + @Test + public void testNot() { + OGCFilter f = getFilter("<Not><PropertyIsEqualTo><PropertyName>altitude</PropertyName><Literal>1047</Literal></PropertyIsEqualTo></Not>"); + assertFalse(f.accept(testKey, testValue)); + } + + @Test + public void testAnd() { + OGCFilter f = getFilter("<And>" + + "<PropertyIsEqualTo><PropertyName>altitude</PropertyName><Literal>1047</Literal></PropertyIsEqualTo>" + + "<PropertyIsNull><PropertyName>test~key</PropertyName></PropertyIsNull>" + + "</And>"); + + assertTrue(f.accept(testKey, testValue)); + } + + @Test + public void testOr() { + OGCFilter f = getFilter("<Or>" + + "<PropertyIsLike><PropertyName>uuid~event</PropertyName><Literal>*event*</Literal></PropertyIsLike>" + + "<PropertyIsNull><PropertyName>uuid~event</PropertyName></PropertyIsNull>" + + "</Or>"); + + assertTrue(f.accept(testKey, testValue)); + } + + @Test + public void testNand() { + OGCFilter f = getFilter("<Not><And>" + + "<PropertyIsNull><PropertyName>uuid~event</PropertyName></PropertyIsNull>" + + "<PropertyIsNull><PropertyName>test~key</PropertyName></PropertyIsNull>" + + "</And></Not>"); + + assertTrue(f.accept(testKey, testValue)); + } + + @Test + public void testNor() { + OGCFilter f = getFilter("<Not>" + + "<PropertyIsNull><PropertyName>uuid~event</PropertyName></PropertyIsNull>" + + "<PropertyIsNull><PropertyName>altitude</PropertyName></PropertyIsNull>" + + "</Not>"); + + assertTrue(f.accept(testKey, testValue)); + } + + @Test + public void testParse() { + OGCFilter f = getFilter("<PropertyIsEqualTo><PropertyName>key</PropertyName><Literal>a</Literal></PropertyIsEqualTo>"); + assertTrue(f.accept(testKey, testValue)); + } +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/partition/common-query-ext/src/test/java/SampleData.java ---------------------------------------------------------------------- diff --git a/partition/common-query-ext/src/test/java/SampleData.java b/partition/common-query-ext/src/test/java/SampleData.java new file mode 100644 index 0000000..071076b --- /dev/null +++ b/partition/common-query-ext/src/test/java/SampleData.java @@ -0,0 +1,228 @@ +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.hadoop.io.Text; + +import cloudbase.core.client.BatchWriter; +import cloudbase.core.client.CBException; +import cloudbase.core.client.CBSecurityException; +import cloudbase.core.client.Connector; +import cloudbase.core.client.Instance; +import cloudbase.core.client.MultiTableBatchWriter; +import cloudbase.core.client.TableExistsException; +import cloudbase.core.client.TableNotFoundException; +import cloudbase.core.client.mock.MockInstance; +import cloudbase.core.data.Mutation; +import cloudbase.core.security.Authorizations; + + +public class SampleData { + public static int NUM_PARTITIONS = 2; + public static int NUM_SAMPLES = 10; + + public static Connector initConnector() { + Instance instance = new MockInstance(); + + try { + Connector connector = instance.getConnector("root", "password".getBytes()); + + // set up table + connector.tableOperations().create("partition"); + connector.tableOperations().create("provenance"); + + // set up root's auths + connector.securityOperations().changeUserAuthorizations("root", new Authorizations("ALPHA,BETA,GAMMA".split(","))); + + return connector; + } catch (CBException e) { + e.printStackTrace(); + } catch (CBSecurityException e) { + e.printStackTrace(); + } catch (TableExistsException e) { + e.printStackTrace(); + } + + return null; + } + + public static Collection<Map<String, String>> sampleData() { + List<Map<String, String>> list = new ArrayList<Map<String, String>>(); + Map<String, String> item; + + for (int i = 0; i < NUM_SAMPLES; i++) { + item = new HashMap<String, String>(); + for (int j = 0; j < 5; j++) { + item.put("field" + j , new String(new char[] {(char) ('A' + ((j + i) % 26))})); + } + list.add(item); + } + return list; + } + + public static void writeDenCellLevel(Connector connector, Collection<Map<String, String>> data) { + // write sample data + MultiTableBatchWriter mtbw = connector.createMultiTableBatchWriter(200000, 10000, 1); + try { + BatchWriter writer; + if (mtbw != null) { + writer = mtbw.getBatchWriter("partition"); + } else { + writer = connector.createBatchWriter("partition", 200000, 10000, 1); + } + int count = 0; + Mutation m; + for (Map<String, String> object: data) { + count++; + String id = (count < 10 ? "0" + count: "" + count); + Text partition = new Text("" + (count % NUM_PARTITIONS)); + + // write dummy record + m = new Mutation(partition); + m.put("event", id, ""); + writer.addMutation(m); + + for (Entry<String, String> entry: object.entrySet()) { + // write the event mutation + m = new Mutation(partition); + m.put("event", id + "\u0000" + entry.getKey(), entry.getValue()); + writer.addMutation(m); + + // write the general index mutation + m = new Mutation(partition); + m.put("index", entry.getValue() + "\u0000" + id, ""); + writer.addMutation(m); + + // write the specific index mutation + m = new Mutation(partition); + m.put("index", entry.getKey() + "//" + entry.getValue() + "\u0000" + id, ""); + writer.addMutation(m); + } + } + writer.close(); + } catch (CBException e) { + e.printStackTrace(); + } catch (CBSecurityException e) { + e.printStackTrace(); + } catch (TableNotFoundException e) { + e.printStackTrace(); + } + } + + public static void writeDenSerialized(Connector connector, Collection<Map<String, String>> data) { + // write sample data + MultiTableBatchWriter mtbw = connector.createMultiTableBatchWriter(200000, 10000, 1); + try { + BatchWriter writer; + if (mtbw != null) { + writer = mtbw.getBatchWriter("partition"); + } else { + writer = connector.createBatchWriter("partition", 200000, 10000, 1); + } + int count = 0; + Mutation m; + for (Map<String, String> object: data) { + count++; + String id = (count < 10 ? "0" + count: "" + count); + Text partition = new Text("" + (count % NUM_PARTITIONS)); + + StringBuilder value = new StringBuilder(); + boolean first = true; + for (Entry<String, String> entry: object.entrySet()) { + if (!first) { + value.append("\u0000"); + } else { + first = false; + } + value.append(entry.getKey()); + value.append("\uFFFD"); + value.append(entry.getValue()); + + // write the general index mutation + m = new Mutation(partition); + m.put("index", entry.getValue() + "\u0000" + id, ""); + writer.addMutation(m); + + // write the specific index mutation + m = new Mutation(partition); + m.put("index", entry.getKey() + "//" + entry.getValue() + "\u0000" + id, ""); + writer.addMutation(m); + } + + // write the event mutation + m = new Mutation(partition); + m.put("event", id, value.toString()); + writer.addMutation(m); + } + writer.close(); + } catch (CBException e) { + e.printStackTrace(); + } catch (CBSecurityException e) { + e.printStackTrace(); + } catch (TableNotFoundException e) { + e.printStackTrace(); + } + } + + public static void writeDenProvenance(Connector connector) { + // write sample data + MultiTableBatchWriter mtbw = connector.createMultiTableBatchWriter(200000, 10000, 1); + try { + BatchWriter writer; + if (mtbw != null) { + writer = mtbw.getBatchWriter("provenance"); + } else { + writer = connector.createBatchWriter("provenance", 200000, 10000, 1); + } + Mutation m; + for (int sid = 1; sid <= 2; sid++) { + for (int time = 1; time <= 3; time++) { + for (int uuid = 1; uuid <= (6 + 2 * time); uuid++) { + m = new Mutation(new Text("sid" + sid)); + m.put("time" + time, "uuid-" + Integer.toHexString(uuid), ""); + writer.addMutation(m); + } + } + } + writer.close(); + } catch (CBException e) { + e.printStackTrace(); + } catch (CBSecurityException e) { + e.printStackTrace(); + } catch (TableNotFoundException e) { + e.printStackTrace(); + } + } + + public static void writeMinIndexes(Connector connector) { + // write sample data + MultiTableBatchWriter mtbw = connector.createMultiTableBatchWriter(200000, 10000, 1); + try { + BatchWriter writer; + if (mtbw != null) { + writer = mtbw.getBatchWriter("partition"); + } else { + writer = connector.createBatchWriter("partition", 200000, 10000, 1); + } + Mutation m; + for (int i = 1; i <= NUM_SAMPLES; i++) { + m = new Mutation(new Text("" + (i % NUM_PARTITIONS))); + + String id = (i < 10 ? "0" + i: "" + i); + + m.put("index", "z_" + id + "_rdate\u0000" + id, ""); + writer.addMutation(m); + } + writer.close(); + } catch (CBException e) { + e.printStackTrace(); + } catch (CBSecurityException e) { + e.printStackTrace(); + } catch (TableNotFoundException e) { + e.printStackTrace(); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/partition/common-query-ext/src/test/java/SampleGVData.java ---------------------------------------------------------------------- diff --git a/partition/common-query-ext/src/test/java/SampleGVData.java b/partition/common-query-ext/src/test/java/SampleGVData.java new file mode 100644 index 0000000..d8168de --- /dev/null +++ b/partition/common-query-ext/src/test/java/SampleGVData.java @@ -0,0 +1,182 @@ + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.hadoop.io.Text; + +import cloudbase.core.client.BatchWriter; +import cloudbase.core.client.CBException; +import cloudbase.core.client.CBSecurityException; +import cloudbase.core.client.Connector; +import cloudbase.core.client.Instance; +import cloudbase.core.client.MultiTableBatchWriter; +import cloudbase.core.client.TableExistsException; +import cloudbase.core.client.TableNotFoundException; +import cloudbase.core.client.mock.MockInstance; +import cloudbase.core.data.Mutation; +import cloudbase.core.security.Authorizations; + +// For use in testing the Date Filter and Frequency Filter classes +public class SampleGVData +{ + + public static int NUM_PARTITIONS = 2; + + + public static Connector initConnector() + { + Instance instance = new MockInstance(); + + try + { + Connector connector = instance.getConnector("root", "password".getBytes()); + + // set up table + connector.tableOperations().create("partition"); + + // set up root's auths + connector.securityOperations().changeUserAuthorizations("root", new Authorizations("ALPHA,BETA,GAMMA".split(","))); + + return connector; + } + catch (CBException e) + { + e.printStackTrace(); + } + catch (CBSecurityException e) + { + e.printStackTrace(); + } + catch (TableExistsException e) + { + e.printStackTrace(); + } + + return null; + } + + public static Collection<Map<String, String>> sampleData() + { + List<Map<String, String>> list = new ArrayList<Map<String, String>>(); + Map<String, String> item; + + item = new HashMap<String, String>(); + item.put("a", "a"); + item.put("b", "b"); + + //This one is like RB + item.put("date-start", "2009-01-01"); + item.put("date-end", "2011-02-24"); + item.put("date-update", "2011-02-24T00:00:00Z"); + item.put("frequency", "1250000000"); + item.put("bandwidth", "500000000"); + item.put("version", "1"); + list.add(item); + + item = new HashMap<String, String>(); + item.put("a", "a"); + item.put("b", "b"); + list.add(item); + + //This one is like GV + item = new HashMap<String, String>(); + item.put("a", "a"); + item.put("b", "b"); + item.put("date-start", "2010-01-01"); + item.put("date-update", "2010-01-23"); + item.put("frequency", "1150000000"); + item.put("bandwidth", "300000000"); + list.add(item); + + item = new HashMap<String, String>(); + item.put("a", "a"); + item.put("b", "b"); + item.put("date-start", "2009-01-01"); + item.put("date-end", "2011-02-24"); + item.put("date-update", "2008-01-23"); + list.add(item); + + item = new HashMap<String, String>(); + item.put("a", "a"); + item.put("b", "b"); + list.add(item); + + return list; + } + + + public static void writeDenSerialized(Connector connector, Collection<Map<String, String>> data) + { + // write sample data + MultiTableBatchWriter mtbw = connector.createMultiTableBatchWriter(200000, 10000, 1); + try + { + BatchWriter writer; + if (mtbw != null) + { + writer = mtbw.getBatchWriter("partition"); + } + else + { + writer = connector.createBatchWriter("partition", 200000, 10000, 1); + } + int count = 0; + Mutation m; + for (Map<String, String> object : data) + { + count++; + String id = (count < 10 ? "0" + count : "" + count); + Text partition = new Text("" + (count % NUM_PARTITIONS)); + + StringBuilder value = new StringBuilder(); + boolean first = true; + for (Entry<String, String> entry : object.entrySet()) + { + if (!first) + { + value.append("\u0000"); + } + else + { + first = false; + } + value.append(entry.getKey()); + value.append("\uFFFD"); + value.append(entry.getValue()); + + // write the general index mutation + m = new Mutation(partition); + m.put("index", entry.getValue() + "\u0000" + id, ""); + writer.addMutation(m); + + // write the specific index mutation + m = new Mutation(partition); + m.put("index", entry.getKey() + "//" + entry.getValue() + "\u0000" + id, ""); + writer.addMutation(m); + } + + // write the event mutation + m = new Mutation(partition); + m.put("event", id, value.toString()); + writer.addMutation(m); + } + writer.close(); + } + catch (CBException e) + { + e.printStackTrace(); + } + catch (CBSecurityException e) + { + e.printStackTrace(); + } + catch (TableNotFoundException e) + { + e.printStackTrace(); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/partition/common-query-ext/src/test/java/SampleJTSData.java ---------------------------------------------------------------------- diff --git a/partition/common-query-ext/src/test/java/SampleJTSData.java b/partition/common-query-ext/src/test/java/SampleJTSData.java new file mode 100644 index 0000000..41df658 --- /dev/null +++ b/partition/common-query-ext/src/test/java/SampleJTSData.java @@ -0,0 +1,171 @@ + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.hadoop.io.Text; + +import cloudbase.core.client.BatchWriter; +import cloudbase.core.client.CBException; +import cloudbase.core.client.CBSecurityException; +import cloudbase.core.client.Connector; +import cloudbase.core.client.Instance; +import cloudbase.core.client.MultiTableBatchWriter; +import cloudbase.core.client.TableExistsException; +import cloudbase.core.client.TableNotFoundException; +import cloudbase.core.client.mock.MockInstance; +import cloudbase.core.data.Mutation; +import cloudbase.core.security.Authorizations; + +// For use in testing the Date Filter and Frequency Filter classes +public class SampleJTSData +{ + + public static int NUM_PARTITIONS = 2; + + + public static Connector initConnector() + { + Instance instance = new MockInstance(); + + try + { + Connector connector = instance.getConnector("root", "password".getBytes()); + + // set up table + connector.tableOperations().create("partition"); + + // set up root's auths + connector.securityOperations().changeUserAuthorizations("root", new Authorizations("ALPHA,BETA,GAMMA".split(","))); + + return connector; + } + catch (CBException e) + { + e.printStackTrace(); + } + catch (CBSecurityException e) + { + e.printStackTrace(); + } + catch (TableExistsException e) + { + e.printStackTrace(); + } + + return null; + } + + public static Collection<Map<String, String>> sampleData() + { + List<Map<String, String>> list = new ArrayList<Map<String, String>>(); + Map<String, String> item; + + item = new HashMap<String, String>(); + item.put("geometry-contour", "SDO_GEOMETRY(2007, 8307, NULL, SDO_ELEM_INFO_ARRAY(1, 1003, 1), SDO_ORDINATE_ARRAY(91.985, -12.108, 94.657, -12.059, 98.486, -11.988, 101.385, -12.296, 102.911, -12.569, 103.93, -12.852, 105.005, -12.531, 106.37, -12.204, 108.446, -11.503, 109.585, -10.88, 110.144, -10.207, 108.609, -9.573, 106.05, -8.535, 104.145, -7.606, 102.191, -7.522, 99.522, -7.691, 97.64, -7.606, 95.482, -7.947, 94.546, -8.084, 92.465, -8.605, 90.554, -9.366, 90.197, -10.436, 89.84, -11.729, 90.554, -12.175, 91.985, -12.108))"); + item.put("beam-name", "OPTUS D1 Ku-BAND NATIONAL A & B AUSTRALIA Downlink"); + list.add(item); + //This is Australia + //Points like 22S 135E are in the beam + + //This one is like GV + item = new HashMap<String, String>(); + item.put("beam-name", "AMC 1 Ku-BAND ZONAL NORTH AMERICA Down HV"); + item.put("geometry-contour", "SDO_GEOMETRY(2007, 8307, NULL, SDO_ELEM_INFO_ARRAY(1, 1003, 1), SDO_ORDINATE_ARRAY(-70.838, 39.967, -70.506, 40.331, -70.698, 41.679, -71.179, 42.401, -71.578, 42.38, -72.994, 42.924, -74.353, 43.242, -75.715, 43.26, -77.318, 42.981, -78.684, 42.774, -80.05, 42.491, -82.005, 42.517, -83.608, 42.312, -84.977, 41.805, -86.58, 41.525, -88.127, 41.02, -89.731, 40.741, -90.905, 41.582, -92.264, 41.9, -93.861, 42.147, -95.411, 41.341, -96.257, 40.076, -97.222, 38.737, -98.011, 37.17, -98.031, 35.593, -97.691, 34.312, -96.875, 33.25, -97.307, 31.904, -97.916, 30.561, -98.702, 29.295, -99.134, 27.949, -98.14, 26.884, -97.205, 25.821, -95.842, 25.803, -94.42, 25.784, -92.876, 26.064, -91.277, 26.043, -90.085, 26.553, -88.729, 26.01, -87.38, 24.941, -86.031, 23.797, -84.616, 23.253, -83.256, 23.01, -81.887, 23.517, -80.866, 24.555, -80.254, 26.124, -79.642, 27.693, -78.444, 28.728, -77.486, 29.542, -76.463, 30.805, -76.088, 32.377, -75.656, 33.723, -76.051, 35.305, -75.442, 36.649, -74.426, 37.386, -73.228, 38.422, -72.032, 39.232, -70.838, 39.967))"); + list.add(item); + //This is North America + //Points 39°44'21.00"N 104°59'3.00"W (Denver) are in the footprint + + item = new HashMap<String, String>(); + item.put("beam-name", "testa"); + item.put("beam-footprint", "MULTIPOLYGON (((-169.286 40.431, -164.971 39.992, -155.397 38.482, -146.566 36.233, -136.975 32.539, -128.124 27.742, -121.946 24.548, -116.849 21.339, -112.156 17.479, -109.391 14.206, -107.301 11.715, -105.274 9.477, -103.443 8.229, -102.108 7.7, -99.109 7.428, -96.681 7.745, -93.894 8.843, -89.917 11.687, -85.953 15.017, -81.148 17.266, -78.145 17.986, -75.582 17.887, -68.1 17.987, -64.696 18.493, -61.445 19.38, -60.094 20.288, -59.315 21.564, -57.026 26.51, -55.089 30.962, -53.59 33.657, -52.495 34.691, -50.468 36.204, -46.146 38.672, -41.684 40.663, -37.914 42.055, -33.806 43.082, -27.523 44.149, -21.645 44.96, -16.578 45.406, -13.807 45.771, -14.929 50.108, -16.186 53.919, -17.051 56.0, -18.388 58.824, -19.861 61.567, -21.807 64.188, -23.104 65.742, -25.28 67.904, -27.699 69.823, -28.955 70.728, -32.415 72.768, -34.968 73.998, -38.468 75.309, -48.292 73.025, -56.545 71.12, -64.023 70.474, -72.753 70.357, -78.41 70.827, -80.466 71.093, -82.412 71.876, -83.02 72.944, -83.175 74.04, -82.493 74.782, -82.412 75.552, -82.697 76.778, -84.041 78.398, -86.316 81.078, -104.098 80.819, -110.861 80.482, -115.73 80.17, -120.936 79.669, -125.84 79.176, -126.696 79.02, -134.316 77.732, -139.505 76.478, -144.823 74.826, -148.231 73.417, -151.517 71.687, -153.87 70.165, -154.536 69.672, -155.868 68.678, -156.482 68.098, -158.281 66.421, -159.716 64.804, -160.996 63.126, -161.878 61.786, -163.046 59.875, -164.369 57.254, -165.563 54.479, -166.73 51.089, -167.811 47.267, -168.581 44.041, -169.286 40.431)), ((-171.333 23.244, -171.523 18.894, -170.127 18.986, -161.559 18.555, -156.977 18.134, -153.574 18.116, -151.108 18.324, -149.947 18.45, -149.018 18.957, -148.515 19.822, -148.524 20.914, -149.018 21.766, -149.947 22.272, -152.185 23.054, -155.563 23.434, -158.075 23.75, -160.272 24.034, -162.184 24.008, -163.514 23.99, -164.595 23.976, -166.52 23.687, -169.159 23.18, -171.333 23.244)))"); + list.add(item); +// this point should be in there... + // -164 40 - somewhere near hawaii + + item = new HashMap<String, String>(); + item.put("beam-name", "testb"); + item.put("beam-footprint", "POLYGON ((-140.153 34.772, -140.341 33.272, -137.024 33.026, -132.723 32.369, -130.947 31.916, -128.664 31.225, -125.293 29.612, -121.813 27.871, -118.699 25.892, -115.589 23.79, -112.593 21.875, -109.136 19.335, -106.939 16.701, -105.006 14.97, -104.195 14.407, -103.049 13.659, -100.363 12.717, -98.063 12.288, -94.299 11.612, -90.825 11.097, -87.997 11.584, -86.815 12.109, -86.163 12.893, -85.014 14.342, -83.804 15.788, -82.104 16.998, -80.413 17.269, -78.005 16.574, -76.181 16.531, -74.65 16.68, -73.552 17.392, -72.957 18.3, -72.917 19.651, -73.526 21.325, -74.913 23.018, -76.036 24.519, -76.159 26.428, -75.741 28.447, -74.257 30.072, -72.771 31.331, -70.517 34.328, -69.638 36.04, -68.624 39.467, -68.015 41.851, -67.607 43.501, -67.548 45.528, -67.586 47.308, -68.601 49.066, -69.868 50.07, -71.621 50.778, -73.285 50.888, -74.9 50.926, -76.994 50.975, -79.332 50.846, -81.066 50.887, -83.842 51.136, -86.569 51.016, -87.95 50.864, -90.831 50.563, -94 .27 50.644, -98.068 50.733, -102.937 51.032, -106.455 51.484, -109.973 51.936, -114.119 52.402, -117.363 53.031, -119.899 53.276, -123.243 53.539, -127.017 54.427, -130.519 55.431, -133.643 56.058, -134.826 56.279, -135.354 55.029, -135.792 53.864, -136.168965072136 52.8279962761917, -136.169 52.828, -136.169497186166 52.8264970826432, -136.192 52.763, -136.556548517884 51.6453176911637, -136.703232746756 51.2152965828266, -136.781220290925 50.9919311116929, -136.793 50.959, -136.80274055379 50.9259886895048, -136.992 50.295, -137.200898649547 49.5808675274021, -137.202 49.581, -137.200962495599 49.5806459535167, -137.360714473458 49.0197683891632, -137.459 48.677, -137.462166719028 48.6649126473121, -137.471 48.634, -137.515105536699 48.4619710228524, -137.74710368039 47.5528216167105, -137.793718522461 47.3758260237407, -137.854 47.152, -137.977773277882 46.6610808974241, -138.044 46.403, -138.330834102374 45.1674736036557, -138.365 45.019, -138.38180854655 44.9421315900087, -138. 449801069917 44.6389849661384, -138.485 44.484, -138.497077239724 44.4262941289417, -138.536 44.25, -138.622787032392 43.8206200438395, -138.743816168807 43.232032787661, -138.981390224617 42.0843314825185, -138.989 42.048, -138.990605533614 42.0389442888447, -138.991 42.037, -138.997785044232 41.9994454595406, -139.004 41.969, -139.035645873997 41.7890661698517, -139.061212567475 41.6462082823816, -139.428 39.584, -139.673 38.073, -139.713116752585 37.8001474769807, -139.766 37.457, -139.764942047737 37.4567768906428, -139.898 36.573, -139.897723683259 36.5729429963606, -139.986 35.994, -140.04777653037 35.5462970502163, -140.094 35.232, -140.090797568766 35.2315144621917, -140.153 34.772))"); + list.add(item); + + + + //London is in niether - 51°30'0.00"N 0° 7'0.00"W + return list; + } + + + public static void writeDenSerialized(Connector connector, Collection<Map<String, String>> data) + { + // write sample data + MultiTableBatchWriter mtbw = connector.createMultiTableBatchWriter(200000, 10000, 1); + try + { + BatchWriter writer; + if (mtbw != null) + { + writer = mtbw.getBatchWriter("partition"); + } + else + { + writer = connector.createBatchWriter("partition", 200000, 10000, 1); + } + int count = 0; + Mutation m; + for (Map<String, String> object : data) + { + count++; + String id = (count < 10 ? "0" + count : "" + count); + Text partition = new Text("" + (count % NUM_PARTITIONS)); + + StringBuilder value = new StringBuilder(); + boolean first = true; + for (Entry<String, String> entry : object.entrySet()) + { + if (!first) + { + value.append("\u0000"); + } + else + { + first = false; + } + value.append(entry.getKey()); + value.append("\uFFFD"); + value.append(entry.getValue()); + + // write the general index mutation + m = new Mutation(partition); + m.put("index", entry.getValue() + "\u0000" + id, ""); + writer.addMutation(m); + + // write the specific index mutation + m = new Mutation(partition); + m.put("index", entry.getKey() + "//" + entry.getValue() + "\u0000" + id, ""); + writer.addMutation(m); + } + + // write the event mutation + m = new Mutation(partition); + m.put("event", id, value.toString()); + writer.addMutation(m); + } + writer.close(); + } + catch (CBException e) + { + e.printStackTrace(); + } + catch (CBSecurityException e) + { + e.printStackTrace(); + } + catch (TableNotFoundException e) + { + e.printStackTrace(); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/partition/common-query/pom.xml ---------------------------------------------------------------------- diff --git a/partition/common-query/pom.xml b/partition/common-query/pom.xml new file mode 100644 index 0000000..6db84bf --- /dev/null +++ b/partition/common-query/pom.xml @@ -0,0 +1,103 @@ +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <!--<parent>--> + <!--<groupId>sitestore</groupId>--> + <!--<artifactId>sitestore</artifactId>--> + <!--<version>2.0.0-SNAPSHOT</version>--> + <!--</parent>--> + + <parent> + <groupId>mvm.rya</groupId> + <artifactId>parent</artifactId> + <version>2.0.0-SNAPSHOT</version> + </parent> + + <groupId>sitestore.common</groupId> + <artifactId>common-query</artifactId> + <name>common-query (${project.version})</name> + <version>2.0.0-SNAPSHOT</version> + <description>A set of filters and iterators for cloudbase queries</description> + + <properties> + <skipTests>true</skipTests> + </properties> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-source-plugin</artifactId> + <version>2.1.2</version> + <executions> + <execution> + <id>attach-sources</id> + <phase>install</phase> + <goals> + <goal>jar</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <version>2.7.2</version> + <configuration> + <skipTests>${skipTests}</skipTests> + </configuration> + </plugin> + </plugins> + </build> + <!--<scm>--> + <!--<connection>${scmLocation}/tto/ss/common/trunk/common-query</connection>--> + <!--</scm>--> + <dependencies> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + </dependency> + <dependency> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + <version>1.2.14</version> + </dependency> + <dependency> + <groupId>cloudbase</groupId> + <artifactId>cloudbase-core</artifactId> + </dependency> + <dependency> + <groupId>commons-logging</groupId> + <artifactId>commons-logging</artifactId> + <version>1.0.4</version> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + </dependency> + <dependency> + <groupId>cloudbase</groupId> + <artifactId>cloudbase-start</artifactId> + </dependency> + <dependency> + <groupId>commons-codec</groupId> + <artifactId>commons-codec</artifactId> + <version>1.3</version> + </dependency> + <dependency> + <groupId>org.apache.thrift</groupId> + <artifactId>thrift</artifactId> + </dependency> + <dependency> + <groupId>com.vividsolutions</groupId> + <artifactId>jts</artifactId> + <version>1.11</version> + </dependency> + <dependency> + <groupId>xerces</groupId> + <artifactId>xercesImpl</artifactId> + <version>2.8.1</version> + </dependency> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/partition/common-query/src/main/java/ss/cloudbase/core/iterators/CellLevelFilteringIterator.java ---------------------------------------------------------------------- diff --git a/partition/common-query/src/main/java/ss/cloudbase/core/iterators/CellLevelFilteringIterator.java b/partition/common-query/src/main/java/ss/cloudbase/core/iterators/CellLevelFilteringIterator.java new file mode 100644 index 0000000..e0126fa --- /dev/null +++ b/partition/common-query/src/main/java/ss/cloudbase/core/iterators/CellLevelFilteringIterator.java @@ -0,0 +1,163 @@ +package ss.cloudbase.core.iterators; + +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import org.apache.hadoop.io.Text; + +import ss.cloudbase.core.iterators.filter.ogc.OGCFilter; +import cloudbase.core.data.ByteSequence; +import cloudbase.core.data.Key; +import cloudbase.core.data.Range; +import cloudbase.core.data.Value; +import cloudbase.core.iterators.IteratorEnvironment; +import cloudbase.core.iterators.SortedKeyValueIterator; +import cloudbase.core.iterators.WrappingIterator; + +public class CellLevelFilteringIterator extends WrappingIterator { + private static final Collection<ByteSequence> EMPTY_SET = Collections.emptySet(); + + /** The OGC Filter string **/ + public static final String OPTION_FILTER = "filter"; + + /** The character or characters that defines the end of the field in the column qualifier. Defaults to '@' **/ + public static final String OPTION_FIELD_END = "fieldEnd"; + + protected SortedKeyValueIterator<Key, Value> checkSource; + + protected Map<String, Boolean> cache = new HashMap<String, Boolean>(); + + protected OGCFilter filter; + + protected String fieldEnd = "@"; + + public CellLevelFilteringIterator() {} + + public CellLevelFilteringIterator(CellLevelFilteringIterator other, IteratorEnvironment env) { + setSource(other.getSource().deepCopy(env)); + checkSource = other.checkSource.deepCopy(env); + cache = other.cache; + fieldEnd = other.fieldEnd; + } + + @Override + public CellLevelFilteringIterator deepCopy(IteratorEnvironment env) { + return new CellLevelFilteringIterator(this, env); + } + + @Override + public void init(SortedKeyValueIterator<Key, Value> source, Map<String, String> options, IteratorEnvironment env) throws IOException { + super.init(source, options, env); + if (source instanceof GMDenIntersectingIterator) { + checkSource = ((GMDenIntersectingIterator) source).docSource.deepCopy(env); + } else if (source instanceof SortedRangeIterator) { + checkSource = ((SortedRangeIterator) source).docSource.deepCopy(env); + } else { + checkSource = source.deepCopy(env); + } + filter = new OGCFilter(); + filter.init(options); + + if (options.containsKey(OPTION_FIELD_END)) { + fieldEnd = options.get(OPTION_FIELD_END); + } + } + + @Override + public void next() throws IOException { + getSource().next(); + findTop(); + } + + protected String getDocId(Key key) { + String colq = key.getColumnQualifier().toString(); + int i = colq.indexOf("\u0000"); + if (i == -1) { + i = colq.length(); + } + return colq.substring(0, i); + } + + protected Key getRecordStartKey(Key key, String docId) { + return new Key(key.getRow(), key.getColumnFamily(), new Text(docId + "\u0000")); + } + + protected Key getRecordEndKey(Key key, String docId) { + return new Key(key.getRow(), key.getColumnFamily(), new Text(docId + "\u0000\uFFFD")); + } + + protected String getField(Key key, Value value) { + String colq = key.getColumnQualifier().toString(); + int i = colq.indexOf("\u0000"); + if (i == -1) { + return null; + } + + int j = colq.indexOf(fieldEnd, i + 1); + if (j == -1) { + j = colq.length(); + } + + return colq.substring(i + 1, j); + } + + protected String getValue(Key key, Value value) { + return value.toString(); + } + + protected void findTop() throws IOException { + boolean goodKey; + String docId; + Map<String, String> record = new HashMap<String, String>(); + + while (getSource().hasTop()) { + docId = getDocId(getSource().getTopKey()); + + // if the document is in the cache, then we have already scanned it + if (cache.containsKey(docId)) { + goodKey = cache.get(docId); + } else { + // we need to scan the whole record into a map and evaluate the filter + + // seek the check source to the beginning of the record + Range range = new Range( + getRecordStartKey(getSource().getTopKey(), docId), + true, + getRecordEndKey(getSource().getTopKey(), docId), + true + ); + + checkSource.seek(range, EMPTY_SET, false); + + // read in the record to the map + record.clear(); + while (checkSource.hasTop()) { + String field = getField(checkSource.getTopKey(), checkSource.getTopValue()); + if (field != null) { + record.put(field, getValue(checkSource.getTopKey(), checkSource.getTopValue())); + } + checkSource.next(); + } + + // evaluate the filter + goodKey = filter.accept(record); + + // cache the result so that we don't do this for every cell + cache.put(docId, goodKey); + } + + if (goodKey==true) + return; + getSource().next(); + } + } + + @Override + public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException { + getSource().seek(range, columnFamilies, inclusive); + findTop(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/partition/common-query/src/main/java/ss/cloudbase/core/iterators/CellLevelRecordIterator.java ---------------------------------------------------------------------- diff --git a/partition/common-query/src/main/java/ss/cloudbase/core/iterators/CellLevelRecordIterator.java b/partition/common-query/src/main/java/ss/cloudbase/core/iterators/CellLevelRecordIterator.java new file mode 100644 index 0000000..1f59882 --- /dev/null +++ b/partition/common-query/src/main/java/ss/cloudbase/core/iterators/CellLevelRecordIterator.java @@ -0,0 +1,144 @@ +package ss.cloudbase.core.iterators; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import org.apache.hadoop.io.Text; + +import ss.cloudbase.core.iterators.filter.CBConverter; + +import cloudbase.core.data.Key; +import cloudbase.core.data.Range; +import cloudbase.core.data.Value; +import cloudbase.core.iterators.IteratorEnvironment; +import cloudbase.core.iterators.SkippingIterator; +import cloudbase.core.iterators.SortedKeyValueIterator; + +public class CellLevelRecordIterator extends SkippingIterator { + public static final String OPTION_FIELD_END = "fieldEnd"; + public static final String OPTION_MULTIPLE_DELIMITER = "multipleDelimiter"; + + protected String multipleDelimiter = ","; + + protected Key topKey; + protected Value topValue; + protected String fieldEnd = "@"; + protected String docId = null; + protected CBConverter converter = new CBConverter(); + + @Override + public SortedKeyValueIterator<Key, Value> deepCopy(IteratorEnvironment env) { + CellLevelRecordIterator itr = new CellLevelRecordIterator(); + itr.setSource(this.getSource().deepCopy(env)); + itr.fieldEnd = this.fieldEnd; + return itr; + } + + @Override + public void init(SortedKeyValueIterator<Key, Value> source, Map<String, String> options, IteratorEnvironment env) throws IOException { + super.init(source, options, env); + converter.init(options); + if (options.containsKey(OPTION_FIELD_END)) { + fieldEnd = options.get(OPTION_FIELD_END); + } + + if (options.containsKey(OPTION_MULTIPLE_DELIMITER)) { + multipleDelimiter = options.get(OPTION_MULTIPLE_DELIMITER); + } + } + + @Override + public void next() throws IOException { + consume(); + } + + @Override + public boolean hasTop() { + return getSource().hasTop() || topKey != null || topValue != null; + } + + @Override + public Key getTopKey() { + return topKey; + } + + @Override + public Value getTopValue() { + return topValue; + } + + protected String getDocId(Key key) { + String colq = key.getColumnQualifier().toString(); + int i = colq.indexOf("\u0000"); + if (i == -1) { + i = colq.length(); + } + return colq.substring(0, i); + } + + protected Key buildTopKey(Key key, String docId) { + return new Key(key.getRow(), key.getColumnFamily(), new Text(docId), key.getColumnVisibility(), key.getTimestamp()); + } + + protected String getField(Key key, Value value) { + String colq = key.getColumnQualifier().toString(); + int i = colq.indexOf("\u0000"); + if (i == -1) { + return null; + } + + int j = colq.indexOf(fieldEnd, i + 1); + if (j == -1) { + j = colq.length(); + } + + return colq.substring(i + 1, j); + } + + protected String getValue(Key key, Value value) { + return value.toString(); + } + + protected Key getRecordStartKey(Key key, String docId) { + return new Key(key.getRow(), key.getColumnFamily(), new Text(docId)); + } + + protected Key getRecordEndKey(Key key, String docId) { + return new Key(key.getRow(), key.getColumnFamily(), new Text(docId + "\u0000\uFFFD")); + } + + @Override + protected void consume() throws IOException { + // build the top key + if (getSource().hasTop()) { + docId = getDocId(getSource().getTopKey()); + topKey = buildTopKey(getSource().getTopKey(), docId); + + Range range = new Range( + getRecordStartKey(getSource().getTopKey(), docId), + true, + getRecordEndKey(getSource().getTopKey(), docId), + true + ); + + Map<String, String> record = new HashMap<String, String>(); + while (getSource().hasTop() && range.contains(getSource().getTopKey())) { + String field = getField(getSource().getTopKey(), getSource().getTopValue()); + if (field != null) { + if (record.get(field) == null) { + record.put(field, getValue(getSource().getTopKey(), getSource().getTopValue())); + } else { + record.put(field, record.get(field) + multipleDelimiter + getValue(getSource().getTopKey(), getSource().getTopValue())); + } + } + getSource().next(); + } + + topValue = converter.toValue(record); + } else { + topKey = null; + topValue = null; + } + } +}
