http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/partition/partition.rdf/src/test/java/mvm/mmrts/rdf/partition/TstBed.java ---------------------------------------------------------------------- diff --git a/partition/partition.rdf/src/test/java/mvm/mmrts/rdf/partition/TstBed.java b/partition/partition.rdf/src/test/java/mvm/mmrts/rdf/partition/TstBed.java new file mode 100644 index 0000000..084cd2e --- /dev/null +++ b/partition/partition.rdf/src/test/java/mvm/mmrts/rdf/partition/TstBed.java @@ -0,0 +1,90 @@ +package mvm.mmrts.rdf.partition; + +import cloudbase.core.client.BatchScanner; +import cloudbase.core.client.Connector; +import cloudbase.core.client.ZooKeeperInstance; +import cloudbase.core.data.Key; +import cloudbase.core.data.Range; +import cloudbase.core.data.Value; +import com.google.common.io.ByteStreams; +import mvm.mmrts.rdf.partition.utils.RdfIO; +import ss.cloudbase.core.iterators.CellLevelRecordIterator; +import ss.cloudbase.core.iterators.SortedRangeIterator; +import ss.cloudbase.core.iterators.filter.CBConverter; + +import java.util.Collections; +import java.util.Iterator; +import java.util.Map; + +import static mvm.mmrts.rdf.partition.PartitionConstants.*; + +/** + * Class TstBed + * Date: Aug 2, 2011 + * Time: 9:22:11 AM + */ +public class TstBed { + public static void main(String[] args) { + try { + + String predicate = "http://here/2010/tracked-data-provenance/ns#createdItem";; + + Connector connector = new ZooKeeperInstance("stratus", "stratus13:2181").getConnector("root", "password"); + BatchScanner bs = connector.createBatchScanner("partitionRdf", ALL_AUTHORIZATIONS, 3); + + bs.setScanIterators(21, CellLevelRecordIterator.class.getName(), "ci"); + + bs.setScanIterators(20, SortedRangeIterator.class.getName(), "ri"); + bs.setScanIteratorOption("ri", SortedRangeIterator.OPTION_DOC_COLF, DOC.toString()); + bs.setScanIteratorOption("ri", SortedRangeIterator.OPTION_COLF, INDEX.toString()); + bs.setScanIteratorOption("ri", SortedRangeIterator.OPTION_START_INCLUSIVE, "" + true); + bs.setScanIteratorOption("ri", SortedRangeIterator.OPTION_END_INCLUSIVE, "" + true); + bs.setScanIteratorOption("ri", SortedRangeIterator.OPTION_MULTI_DOC, "" + true); + + bs.setScanIteratorOption("ri", SortedRangeIterator.OPTION_LOWER_BOUND, + "\07http://here/2010/tracked-data-provenance/ns#reportedAt\u0001\u000B2011-08-26T18:01:51.000Z"; + ); + bs.setScanIteratorOption("ri", SortedRangeIterator.OPTION_UPPER_BOUND, + "\07http://here/2010/tracked-data-provenance/ns#reportedAt\u0001\u000B2011-08-26T18:01:51.400Z"; + ); + + Range range = new Range( + new Key("2011-08\0"), + new Key("2011-08\uFFFD") + ); + +// scanner.setRange(range); + bs.setRanges(Collections.singleton(range)); +// bs.fetchColumnFamily(INDEX); +// bs.setColumnFamilyRegex(INDEX.toString()); +// bs.setColumnQualifierRegex(URI_MARKER_STR + predicate + INDEX_DELIM_STR + "(.*)"); + + int count = 0; + Iterator<Map.Entry<Key, Value>> iter = bs.iterator(); + CBConverter converter = new CBConverter(); + while (iter.hasNext()) { + count++; +// iter.next(); + Map.Entry<Key, Value> entry = iter.next(); + Value value = entry.getValue(); +// System.out.println(entry.getKey().getColumnQualifier() + "----" + value); + org.openrdf.model.Value subj = RdfIO.readValue(ByteStreams.newDataInput(entry.getKey().getColumnQualifier().getBytes()), VALUE_FACTORY, FAMILY_DELIM); + Map<String, String> map = converter.toMap(entry.getKey(), value); + for (Map.Entry<String, String> e : map.entrySet()) { + String predObj = e.getKey(); + String[] split = predObj.split("\0"); + byte[] look = split[0].getBytes(); + System.out.println(subj + + " : " + VALUE_FACTORY.createURI(split[0]) + " : " + + RdfIO.readValue(ByteStreams.newDataInput(split[1].getBytes()), VALUE_FACTORY, FAMILY_DELIM)); + } + } + System.out.println(count); + + bs.close(); + + } catch (Exception e) { + e.printStackTrace(); + } + } +}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/partition/partition.rdf/src/test/java/mvm/mmrts/rdf/partition/TstBedGMDen.java ---------------------------------------------------------------------- diff --git a/partition/partition.rdf/src/test/java/mvm/mmrts/rdf/partition/TstBedGMDen.java b/partition/partition.rdf/src/test/java/mvm/mmrts/rdf/partition/TstBedGMDen.java new file mode 100644 index 0000000..44309e2 --- /dev/null +++ b/partition/partition.rdf/src/test/java/mvm/mmrts/rdf/partition/TstBedGMDen.java @@ -0,0 +1,94 @@ +package mvm.mmrts.rdf.partition; + +import cloudbase.core.client.BatchScanner; +import cloudbase.core.client.Connector; +import cloudbase.core.client.ZooKeeperInstance; +import cloudbase.core.data.Key; +import cloudbase.core.data.Range; +import cloudbase.core.data.Value; +import com.google.common.io.ByteStreams; +import mvm.mmrts.rdf.partition.utils.RdfIO; +import org.apache.hadoop.io.Text; +import org.openrdf.query.algebra.Var; +import ss.cloudbase.core.iterators.CellLevelRecordIterator; +import ss.cloudbase.core.iterators.GMDenIntersectingIterator; +import ss.cloudbase.core.iterators.SortedRangeIterator; +import ss.cloudbase.core.iterators.filter.CBConverter; + +import java.util.Collections; +import java.util.Iterator; +import java.util.Map; + +import static mvm.mmrts.rdf.partition.PartitionConstants.*; + +/** + * Class TstBed + * Date: Aug 2, 2011 + * Time: 9:22:11 AM + */ +public class TstBedGMDen { + public static void main(String[] args) { + try { + + Connector connector = new ZooKeeperInstance("stratus", "stratus13:2181").getConnector("root", "password"); + BatchScanner bs = connector.createBatchScanner("rdfPartition", ALL_AUTHORIZATIONS, 3); + + String[] predicates = {"urn:lubm:test#takesCourse", + "urn:lubm:test#name", + "urn:lubm:test#specific", + "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"}; + + Text[] queries = new Text[predicates.length]; + for (int i = 0; i < predicates.length; i++) { + String predicate = predicates[i]; + queries[i] = new Text(GMDenIntersectingIterator.getRangeTerm(INDEX.toString(), + URI_MARKER_STR + predicate + INDEX_DELIM_STR + "\0" + , true, + URI_MARKER_STR + predicate + INDEX_DELIM_STR + "\uFFFD", + true + )); + System.out.println(queries[i]); + } + + bs.setScanIterators(21, CellLevelRecordIterator.class.getName(), "ci"); + + bs.setScanIterators(20, GMDenIntersectingIterator.class.getName(), "ii"); + bs.setScanIteratorOption("ii", GMDenIntersectingIterator.docFamilyOptionName, DOC.toString()); + bs.setScanIteratorOption("ii", GMDenIntersectingIterator.indexFamilyOptionName, INDEX.toString()); + bs.setScanIteratorOption("ii", GMDenIntersectingIterator.columnFamiliesOptionName, GMDenIntersectingIterator.encodeColumns(queries)); + bs.setScanIteratorOption("ii", GMDenIntersectingIterator.OPTION_MULTI_DOC, "" + true); + + Range range = new Range( + new Key("2011-11\0"), + new Key("2011-11\uFFFD") + ); + + bs.setRanges(Collections.singleton(range)); + + int count = 0; + Iterator<Map.Entry<Key, Value>> iter = bs.iterator(); + CBConverter converter = new CBConverter(); + while (iter.hasNext()) { + count++; + Map.Entry<Key, Value> entry = iter.next(); + Value value = entry.getValue(); + org.openrdf.model.Value subj = RdfIO.readValue(ByteStreams.newDataInput(entry.getKey().getColumnQualifier().getBytes()), VALUE_FACTORY, FAMILY_DELIM); + Map<String, String> map = converter.toMap(entry.getKey(), value); + for (Map.Entry<String, String> e : map.entrySet()) { + String predObj = e.getKey(); + String[] split = predObj.split("\0"); + byte[] look = split[0].getBytes(); + System.out.println(subj + + " : " + VALUE_FACTORY.createURI(split[0]) + " : " + + RdfIO.readValue(ByteStreams.newDataInput(split[1].getBytes()), VALUE_FACTORY, FAMILY_DELIM)); + } + } + System.out.println(count); + + bs.close(); + + } catch (Exception e) { + e.printStackTrace(); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/partition/partition.rdf/src/test/java/mvm/mmrts/rdf/partition/TstDocumentReader.java ---------------------------------------------------------------------- diff --git a/partition/partition.rdf/src/test/java/mvm/mmrts/rdf/partition/TstDocumentReader.java b/partition/partition.rdf/src/test/java/mvm/mmrts/rdf/partition/TstDocumentReader.java new file mode 100644 index 0000000..7723431 --- /dev/null +++ b/partition/partition.rdf/src/test/java/mvm/mmrts/rdf/partition/TstDocumentReader.java @@ -0,0 +1,217 @@ +//package mvm.mmrts.rdf.partition; +// +//import cloudbase.core.CBConstants; +//import cloudbase.core.client.Connector; +//import cloudbase.core.client.ZooKeeperInstance; +//import cloudbase.core.client.mock.MockInstance; +//import cloudbase.core.data.Key; +//import cloudbase.core.data.Range; +//import cloudbase.core.data.Value; +//import cloudbase.core.util.TextUtil; +//import mvm.mmrts.cloudbase.utils.client.DocumentBatchScanner; +//import mvm.mmrts.cloudbase.utils.client.DocumentConnectorImpl; +//import mvm.mmrts.rdf.partition.shard.DateHashModShardValueGenerator; +//import org.apache.hadoop.io.Text; +//import org.apache.log4j.Level; +//import org.apache.log4j.Logger; +//import org.openrdf.model.ValueFactory; +//import org.openrdf.model.impl.StatementImpl; +//import org.openrdf.model.impl.ValueFactoryImpl; +//import org.openrdf.repository.RepositoryException; +//import org.openrdf.repository.sail.SailRepository; +//import org.openrdf.repository.sail.SailRepositoryConnection; +//import ss.cloudbase.core.iterators.SortedRangeIterator; +// +//import javax.xml.datatype.DatatypeConfigurationException; +//import javax.xml.datatype.DatatypeFactory; +//import java.util.Collections; +//import java.util.Iterator; +//import java.util.List; +//import java.util.Map; +// +//import static mvm.mmrts.rdf.partition.PartitionConstants.*; +// +///** +// * Class TstDocumentReader +// * Date: Sep 8, 2011 +// * Time: 9:11:27 AM +// */ +//public class TstDocumentReader { +// public static final String NAMESPACE = "http://here/2010/tracked-data-provenance/ns#";//44 len +// public static final String RDF_NS = "http://www.w3.org/1999/02/22-rdf-syntax-ns#";; +// public static final String HBNAMESPACE = "http://here/2010/tracked-data-provenance/heartbeat/ns#";; +// public static final String HB_TIMESTAMP = HBNAMESPACE + "timestamp"; +// +// private static SailRepository repository; +// private static SailRepositoryConnection connection; +// +// private static ValueFactory vf = ValueFactoryImpl.getInstance(); +// +// private static final String TABLE = "partitionRdf"; +// private static final long START = 1309532965000l; +// private static final long END = 1310566686000l; +// private static String objectUuid = "objectuuid1"; +// +// public static void main(String[] args) { +// try { +//// Logger.getRootLogger().setLevel(Level.TRACE); +// DocumentConnectorImpl connector = new DocumentConnectorImpl(new ZooKeeperInstance("stratus", "stratus13:2181"), "root", "password".getBytes()); +//// DocumentConnectorImpl connector = new DocumentConnectorImpl(new MockInstance(), "", "".getBytes()); +// +//// PartitionSail sail = new PartitionSail(connector, TABLE, new DateHashModShardValueGenerator() { +//// @Override +//// public String generateShardValue(Object obj) { +//// return this.generateShardValue(START + 1000, obj); +//// } +//// }); +//// +//// repository = new SailRepository(sail); +//// repository.initialize(); +//// connection = repository.getConnection(); +//// +//// loadData(); +// +// DocumentBatchScanner bs = connector.createDocumentBatchScanner(TABLE, CBConstants.NO_AUTHS, 2); +// +// bs.setScanIterators(20, SortedRangeIterator.class.getName(), "ri"); +// bs.setScanIteratorOption("ri", SortedRangeIterator.OPTION_DOC_COLF, DOC.toString()); +// bs.setScanIteratorOption("ri", SortedRangeIterator.OPTION_COLF, INDEX.toString()); +// bs.setScanIteratorOption("ri", SortedRangeIterator.OPTION_START_INCLUSIVE, "" + true); +// bs.setScanIteratorOption("ri", SortedRangeIterator.OPTION_END_INCLUSIVE, "" + true); +// bs.setScanIteratorOption("ri", SortedRangeIterator.OPTION_MULTI_DOC, "" + true); +// +// bs.setScanIteratorOption("ri", SortedRangeIterator.OPTION_LOWER_BOUND, +// "\07http://here/2010/tracked-data-provenance/ns#reportedAt\u0001\u000B2011-08-26T18:01:51Z"; +// ); +// bs.setScanIteratorOption("ri", SortedRangeIterator.OPTION_UPPER_BOUND, +// "\07http://here/2010/tracked-data-provenance/ns#reportedAt\u0001\u000B2011-08-26T18:02:00Z"; +// ); +// +// Range range = new Range( +// new Key("2011-08\0"), +// new Key("2011-08\uFFFD") +// ); +// +// bs.setRanges(Collections.singleton(range)); +// +// int count = 0; +// int innerCount = 0; +// Iterator<List<? extends Map.Entry<Key, Value>>> iter = bs.documentIterator(); +// while (iter.hasNext()) { +// count++; +// List<? extends Map.Entry<Key, Value>> entries = iter.next(); +// for (Map.Entry<Key, Value> entry : entries) { +// System.out.print(entry.getKey().getColumnQualifier()); +// System.out.println(" "); +// innerCount++; +// } +// System.out.println(); +// } +// System.out.println(count); +// System.out.println(innerCount); +// +// bs.close(); +//// connection.close(); +//// repository.shutDown(); +// +// } catch (Exception e) { +// e.printStackTrace(); +// } +// } +// +// private static void loadData() throws RepositoryException, DatatypeConfigurationException { +// connection.add(new StatementImpl(vf.createURI(NAMESPACE, objectUuid), vf.createURI(NAMESPACE, "name"), vf.createLiteral("objUuid"))); +// //created +// String uuid = "uuid1"; +// connection.add(new StatementImpl(vf.createURI(NAMESPACE, uuid), vf.createURI(RDF_NS, "type"), vf.createURI(NAMESPACE, "Created"))); +// connection.add(new StatementImpl(vf.createURI(NAMESPACE, uuid), vf.createURI(NAMESPACE, "createdItem"), vf.createURI(NAMESPACE, objectUuid))); +// connection.add(new StatementImpl(vf.createURI(NAMESPACE, uuid), vf.createURI(NAMESPACE, "performedBy"), vf.createURI("urn:system:A"))); +// connection.add(new StatementImpl(vf.createURI(NAMESPACE, uuid), vf.createURI(NAMESPACE, "stringLit"), vf.createLiteral("stringLit1"))); +// connection.add(new StatementImpl(vf.createURI(NAMESPACE, uuid), vf.createURI(NAMESPACE, "stringLit"), vf.createLiteral("stringLit2"))); +// connection.add(new StatementImpl(vf.createURI(NAMESPACE, uuid), vf.createURI(NAMESPACE, "stringLit"), vf.createLiteral("stringLit3"))); +// connection.add(new StatementImpl(vf.createURI(NAMESPACE, uuid), vf.createURI(NAMESPACE, "stringLit"), vf.createLiteral("stringLit4"))); +// connection.add(new StatementImpl(vf.createURI(NAMESPACE, uuid), vf.createURI(NAMESPACE, "strLit1"), vf.createLiteral("strLit1"))); +// connection.add(new StatementImpl(vf.createURI(NAMESPACE, uuid), vf.createURI(NAMESPACE, "strLit1"), vf.createLiteral("strLit2"))); +// connection.add(new StatementImpl(vf.createURI(NAMESPACE, uuid), vf.createURI(NAMESPACE, "strLit1"), vf.createLiteral("strLit3"))); +// connection.add(new StatementImpl(vf.createURI(NAMESPACE, uuid), vf.createURI(NAMESPACE, "performedAt"), vf.createLiteral(DatatypeFactory.newInstance().newXMLGregorianCalendar(2011, 7, 12, 6, 0, 0, 0, 0)))); +// connection.add(new StatementImpl(vf.createURI(NAMESPACE, uuid), vf.createURI(NAMESPACE, "reportedAt"), vf.createLiteral(DatatypeFactory.newInstance().newXMLGregorianCalendar(2011, 7, 12, 6, 1, 0, 0, 0)))); +// //clicked +// uuid = "uuid2"; +// connection.add(new StatementImpl(vf.createURI(NAMESPACE, uuid), vf.createURI(RDF_NS, "type"), vf.createURI(NAMESPACE, "Clicked"))); +// connection.add(new StatementImpl(vf.createURI(NAMESPACE, uuid), vf.createURI(NAMESPACE, "clickedItem"), vf.createURI(NAMESPACE, objectUuid))); +// connection.add(new StatementImpl(vf.createURI(NAMESPACE, uuid), vf.createURI(NAMESPACE, "performedBy"), vf.createURI("urn:system:B"))); +// connection.add(new StatementImpl(vf.createURI(NAMESPACE, uuid), vf.createURI(NAMESPACE, "performedAt"), vf.createLiteral(DatatypeFactory.newInstance().newXMLGregorianCalendar(2011, 7, 12, 6, 2, 0, 0, 0)))); +// connection.add(new StatementImpl(vf.createURI(NAMESPACE, uuid), vf.createURI(NAMESPACE, "reportedAt"), vf.createLiteral(DatatypeFactory.newInstance().newXMLGregorianCalendar(2011, 7, 12, 6, 3, 0, 0, 0)))); +// //deleted +// uuid = "uuid3"; +// connection.add(new StatementImpl(vf.createURI(NAMESPACE, uuid), vf.createURI(RDF_NS, "type"), vf.createURI(NAMESPACE, "Deleted"))); +// connection.add(new StatementImpl(vf.createURI(NAMESPACE, uuid), vf.createURI(NAMESPACE, "deletedItem"), vf.createURI(NAMESPACE, objectUuid))); +// connection.add(new StatementImpl(vf.createURI(NAMESPACE, uuid), vf.createURI(NAMESPACE, "performedBy"), vf.createURI("urn:system:C"))); +// connection.add(new StatementImpl(vf.createURI(NAMESPACE, uuid), vf.createURI(NAMESPACE, "performedAt"), vf.createLiteral(DatatypeFactory.newInstance().newXMLGregorianCalendar(2011, 7, 12, 6, 4, 0, 0, 0)))); +// connection.add(new StatementImpl(vf.createURI(NAMESPACE, uuid), vf.createURI(NAMESPACE, "reportedAt"), vf.createLiteral(DatatypeFactory.newInstance().newXMLGregorianCalendar(2011, 7, 12, 6, 5, 0, 0, 0)))); +// //dropped +// uuid = "uuid4"; +// connection.add(new StatementImpl(vf.createURI(NAMESPACE, uuid), vf.createURI(RDF_NS, "type"), vf.createURI(NAMESPACE, "Dropped"))); +// connection.add(new StatementImpl(vf.createURI(NAMESPACE, uuid), vf.createURI(NAMESPACE, "droppedItem"), vf.createURI(NAMESPACE, objectUuid))); +// connection.add(new StatementImpl(vf.createURI(NAMESPACE, uuid), vf.createURI(NAMESPACE, "performedBy"), vf.createURI("urn:system:D"))); +// connection.add(new StatementImpl(vf.createURI(NAMESPACE, uuid), vf.createURI(NAMESPACE, "performedAt"), vf.createLiteral(DatatypeFactory.newInstance().newXMLGregorianCalendar(2011, 7, 12, 6, 6, 0, 0, 0)))); +// connection.add(new StatementImpl(vf.createURI(NAMESPACE, uuid), vf.createURI(NAMESPACE, "reportedAt"), vf.createLiteral(DatatypeFactory.newInstance().newXMLGregorianCalendar(2011, 7, 12, 6, 7, 0, 0, 0)))); +// //received +// uuid = "uuid5"; +// connection.add(new StatementImpl(vf.createURI(NAMESPACE, uuid), vf.createURI(RDF_NS, "type"), vf.createURI(NAMESPACE, "Received"))); +// connection.add(new StatementImpl(vf.createURI(NAMESPACE, uuid), vf.createURI(NAMESPACE, "receivedItem"), vf.createURI(NAMESPACE, objectUuid))); +// connection.add(new StatementImpl(vf.createURI(NAMESPACE, uuid), vf.createURI(NAMESPACE, "performedBy"), vf.createURI("urn:system:E"))); +// connection.add(new StatementImpl(vf.createURI(NAMESPACE, uuid), vf.createURI(NAMESPACE, "performedAt"), vf.createLiteral(DatatypeFactory.newInstance().newXMLGregorianCalendar(2011, 7, 12, 6, 8, 0, 0, 0)))); +// connection.add(new StatementImpl(vf.createURI(NAMESPACE, uuid), vf.createURI(NAMESPACE, "reportedAt"), vf.createLiteral(DatatypeFactory.newInstance().newXMLGregorianCalendar(2011, 7, 12, 6, 9, 0, 0, 0)))); +// //sent +// uuid = "uuid6"; +// connection.add(new StatementImpl(vf.createURI(NAMESPACE, uuid), vf.createURI(RDF_NS, "type"), vf.createURI(NAMESPACE, "Sent"))); +// connection.add(new StatementImpl(vf.createURI(NAMESPACE, uuid), vf.createURI(NAMESPACE, "sentItem"), vf.createURI(NAMESPACE, objectUuid))); +// connection.add(new StatementImpl(vf.createURI(NAMESPACE, uuid), vf.createURI(NAMESPACE, "performedBy"), vf.createURI("urn:system:F"))); +// connection.add(new StatementImpl(vf.createURI(NAMESPACE, uuid), vf.createURI(NAMESPACE, "performedAt"), vf.createLiteral(DatatypeFactory.newInstance().newXMLGregorianCalendar(2011, 7, 12, 6, 10, 0, 0, 0)))); +// connection.add(new StatementImpl(vf.createURI(NAMESPACE, uuid), vf.createURI(NAMESPACE, "reportedAt"), vf.createLiteral(DatatypeFactory.newInstance().newXMLGregorianCalendar(2011, 7, 12, 6, 11, 0, 0, 0)))); +// //stored +// uuid = "uuid7"; +// connection.add(new StatementImpl(vf.createURI(NAMESPACE, uuid), vf.createURI(RDF_NS, "type"), vf.createURI(NAMESPACE, "Stored"))); +// connection.add(new StatementImpl(vf.createURI(NAMESPACE, uuid), vf.createURI(NAMESPACE, "storedItem"), vf.createURI(NAMESPACE, objectUuid))); +// connection.add(new StatementImpl(vf.createURI(NAMESPACE, uuid), vf.createURI(NAMESPACE, "performedBy"), vf.createURI("urn:system:G"))); +// connection.add(new StatementImpl(vf.createURI(NAMESPACE, uuid), vf.createURI(NAMESPACE, "performedAt"), vf.createLiteral(DatatypeFactory.newInstance().newXMLGregorianCalendar(2011, 7, 12, 6, 12, 0, 0, 0)))); +// connection.add(new StatementImpl(vf.createURI(NAMESPACE, uuid), vf.createURI(NAMESPACE, "reportedAt"), vf.createLiteral(DatatypeFactory.newInstance().newXMLGregorianCalendar(2011, 7, 12, 6, 13, 0, 0, 0)))); +// +// //heartbeats +// String hbuuid = "hbuuid1"; +// connection.add(new StatementImpl(vf.createURI(HBNAMESPACE, hbuuid), vf.createURI(RDF_NS, "type"), vf.createURI(HBNAMESPACE, "HeartbeatMeasurement"))); +// connection.add(new StatementImpl(vf.createURI(HBNAMESPACE, hbuuid), vf.createURI(HB_TIMESTAMP), vf.createLiteral((START + 1) + ""))); +// connection.add(new StatementImpl(vf.createURI(HBNAMESPACE, hbuuid), vf.createURI(HBNAMESPACE, "count"), vf.createLiteral(1 + ""))); +// connection.add(new StatementImpl(vf.createURI(HBNAMESPACE, hbuuid), vf.createURI(HBNAMESPACE, "systemName"), vf.createURI("urn:system:A"))); +// connection.add(new StatementImpl(vf.createURI("urn:system:A"), vf.createURI(HBNAMESPACE, "heartbeat"), vf.createURI(HBNAMESPACE, hbuuid))); +// +// hbuuid = "hbuuid2"; +// connection.add(new StatementImpl(vf.createURI(HBNAMESPACE, hbuuid), vf.createURI(RDF_NS, "type"), vf.createURI(HBNAMESPACE, "HeartbeatMeasurement"))); +// connection.add(new StatementImpl(vf.createURI(HBNAMESPACE, hbuuid), vf.createURI(HB_TIMESTAMP), vf.createLiteral((START + 2) + ""))); +// connection.add(new StatementImpl(vf.createURI(HBNAMESPACE, hbuuid), vf.createURI(HBNAMESPACE, "count"), vf.createLiteral(2 + ""))); +// connection.add(new StatementImpl(vf.createURI(HBNAMESPACE, hbuuid), vf.createURI(HBNAMESPACE, "systemName"), vf.createURI("urn:system:B"))); +// connection.add(new StatementImpl(vf.createURI("urn:system:B"), vf.createURI(HBNAMESPACE, "heartbeat"), vf.createURI(HBNAMESPACE, hbuuid))); +// +// hbuuid = "hbuuid3"; +// connection.add(new StatementImpl(vf.createURI(HBNAMESPACE, hbuuid), vf.createURI(RDF_NS, "type"), vf.createURI(HBNAMESPACE, "HeartbeatMeasurement"))); +// connection.add(new StatementImpl(vf.createURI(HBNAMESPACE, hbuuid), vf.createURI(HB_TIMESTAMP), vf.createLiteral((START + 3) + ""))); +// connection.add(new StatementImpl(vf.createURI(HBNAMESPACE, hbuuid), vf.createURI(HBNAMESPACE, "count"), vf.createLiteral(3 + ""))); +// connection.add(new StatementImpl(vf.createURI(HBNAMESPACE, hbuuid), vf.createURI(HBNAMESPACE, "systemName"), vf.createURI("urn:system:C"))); +// connection.add(new StatementImpl(vf.createURI("urn:system:C"), vf.createURI(HBNAMESPACE, "heartbeat"), vf.createURI(HBNAMESPACE, hbuuid))); +// +// connection.add(new StatementImpl(vf.createURI("urn:subj1"), vf.createURI("urn:pred"), vf.createLiteral("obj1"))); +// connection.add(new StatementImpl(vf.createURI("urn:subj1"), vf.createURI("urn:pred"), vf.createLiteral("obj2"))); +// connection.add(new StatementImpl(vf.createURI("urn:subj1"), vf.createURI("urn:pred"), vf.createLiteral("obj3"))); +// connection.add(new StatementImpl(vf.createURI("urn:subj1"), vf.createURI("urn:pred"), vf.createLiteral("obj4"))); +// connection.add(new StatementImpl(vf.createURI("urn:subj2"), vf.createURI("urn:pred"), vf.createLiteral("obj1"))); +// connection.add(new StatementImpl(vf.createURI("urn:subj2"), vf.createURI("urn:pred"), vf.createLiteral("obj2"))); +// connection.add(new StatementImpl(vf.createURI("urn:subj2"), vf.createURI("urn:pred"), vf.createLiteral("obj3"))); +// connection.add(new StatementImpl(vf.createURI("urn:subj2"), vf.createURI("urn:pred"), vf.createLiteral("obj4"))); +// connection.add(new StatementImpl(vf.createURI("urn:subj3"), vf.createURI("urn:pred"), vf.createLiteral("obj1"))); +// connection.add(new StatementImpl(vf.createURI("urn:subj3"), vf.createURI("urn:pred"), vf.createLiteral("obj4"))); +// connection.commit(); +// } +// +// +//} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/partition/partition.rdf/src/test/java/mvm/mmrts/rdf/partition/TstScanner.java ---------------------------------------------------------------------- diff --git a/partition/partition.rdf/src/test/java/mvm/mmrts/rdf/partition/TstScanner.java b/partition/partition.rdf/src/test/java/mvm/mmrts/rdf/partition/TstScanner.java new file mode 100644 index 0000000..6650cf1 --- /dev/null +++ b/partition/partition.rdf/src/test/java/mvm/mmrts/rdf/partition/TstScanner.java @@ -0,0 +1,59 @@ +package mvm.mmrts.rdf.partition; + +import cloudbase.core.client.BatchScanner; +import cloudbase.core.client.Connector; +import cloudbase.core.client.ZooKeeperInstance; +import cloudbase.core.data.Key; +import cloudbase.core.data.Range; +import cloudbase.core.data.Value; +import org.apache.hadoop.io.Text; +import ss.cloudbase.core.iterators.SortedRangeIterator; + +import java.util.Collections; +import java.util.Iterator; +import java.util.Map; + +import static mvm.mmrts.rdf.partition.PartitionConstants.*; + +/** + * Class TstBed + * Date: Aug 2, 2011 + * Time: 9:22:11 AM + */ +public class TstScanner { + public static void main(String[] args) { + try { + Connector connector = new ZooKeeperInstance("stratus", "stratus13:2181").getConnector("root", "password"); + BatchScanner bs = connector.createBatchScanner("partitionRdf", ALL_AUTHORIZATIONS, 3); + + Text shard = new Text("2011-08-40"); + String uri = "urn:tde:c5e2f4d8-a5a6-48d8-ba55-1acea969c38d"; + bs.setRanges(Collections.singleton( + new Range( + new Key( + shard, DOC, + new Text(URI_MARKER_STR + uri + FAMILY_DELIM_STR + "\0") + ), + new Key( + shard, DOC, + new Text(URI_MARKER_STR + uri + FAMILY_DELIM_STR + "\uFFFD") + ) + ) + )); + + int count = 0; + Iterator<Map.Entry<Key, Value>> iter = bs.iterator(); + while (iter.hasNext()) { + count++; +// iter.next(); + System.out.println(iter.next().getKey().getColumnQualifier()); + } + System.out.println(count); + + bs.close(); + + } catch (Exception e) { + e.printStackTrace(); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/partition/partition.rdf/src/test/java/mvm/mmrts/rdf/partition/shard/DateHashModShardValueGeneratorTest.java ---------------------------------------------------------------------- diff --git a/partition/partition.rdf/src/test/java/mvm/mmrts/rdf/partition/shard/DateHashModShardValueGeneratorTest.java b/partition/partition.rdf/src/test/java/mvm/mmrts/rdf/partition/shard/DateHashModShardValueGeneratorTest.java new file mode 100644 index 0000000..fe27ec7 --- /dev/null +++ b/partition/partition.rdf/src/test/java/mvm/mmrts/rdf/partition/shard/DateHashModShardValueGeneratorTest.java @@ -0,0 +1,30 @@ +package mvm.mmrts.rdf.partition.shard; + +import junit.framework.TestCase; + +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Date; + +/** + * Class DateHashModShardValueGeneratorTest + * Date: Jul 6, 2011 + * Time: 6:35:32 PM + */ +public class DateHashModShardValueGeneratorTest extends TestCase { + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMdd"); + Calendar cal = Calendar.getInstance(); + + public void testGenerateShardValue() throws Exception { + + DateHashModShardValueGenerator gen = new DateHashModShardValueGenerator(); + gen.setBaseMod(100); + assertEquals(gen.generateShardValue("subject"), dateFormat.format(cal.getTime()) + "_68"); + } + + public void testGenerateShardValueNullObject() throws Exception { + DateHashModShardValueGenerator gen = new DateHashModShardValueGenerator(); + gen.setBaseMod(100); + assertEquals(gen.generateShardValue(null), dateFormat.format(cal.getTime())); + } +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/partition/partition.rdf/src/test/java/mvm/mmrts/rdf/partition/utils/RdfIOTest.java ---------------------------------------------------------------------- diff --git a/partition/partition.rdf/src/test/java/mvm/mmrts/rdf/partition/utils/RdfIOTest.java b/partition/partition.rdf/src/test/java/mvm/mmrts/rdf/partition/utils/RdfIOTest.java new file mode 100644 index 0000000..d251485 --- /dev/null +++ b/partition/partition.rdf/src/test/java/mvm/mmrts/rdf/partition/utils/RdfIOTest.java @@ -0,0 +1,82 @@ +package mvm.mmrts.rdf.partition.utils; + +import com.google.common.io.ByteStreams; +import com.google.common.primitives.Bytes; +import junit.framework.TestCase; +import org.openrdf.model.Statement; +import org.openrdf.model.ValueFactory; +import org.openrdf.model.impl.StatementImpl; +import org.openrdf.model.impl.ValueFactoryImpl; + +import static mvm.mmrts.rdf.partition.utils.RdfIO.*; + +/** + * Class RdfIOTest + * Date: Jul 6, 2011 + * Time: 12:59:25 PM + */ +public class RdfIOTest extends TestCase { + + ValueFactory vf = ValueFactoryImpl.getInstance(); + + public void testWriteStatementEvent() throws Exception { + StatementImpl stmt = new StatementImpl(vf.createURI("urn:namespace#subj"), vf.createURI("urn:namespace#pred"), vf.createLiteral("object")); + byte[] bytes = writeStatement(stmt, true); + } + + public void testWriteStatementIndex() throws Exception { + StatementImpl stmt = new StatementImpl(vf.createURI("urn:namespace#subj"), vf.createURI("urn:namespace#pred"), vf.createLiteral("object")); + byte[] bytes = writeStatement(stmt, false); + } + + public void testExtraInfoInStmtBytes() throws Exception { + StatementImpl stmt = new StatementImpl(vf.createURI("urn:namespace#subj"), vf.createURI("urn:namespace#pred"), vf.createLiteral("object")); + byte[] bytes = writeStatement(stmt, true); + bytes = Bytes.concat(bytes, "extrainformation".getBytes()); + Statement readStmt = readStatement(ByteStreams.newDataInput(bytes), ValueFactoryImpl.getInstance()); + System.out.println(readStmt); + } + + public void testReadStatement() throws Exception { + StatementImpl stmt = new StatementImpl(vf.createURI("urn:namespace#subj"), vf.createURI("urn:namespace#pred"), vf.createLiteral("object")); + byte[] bytes = writeStatement(stmt, true); + + Statement readStmt = readStatement(ByteStreams.newDataInput(bytes), vf); + assertEquals(readStmt, stmt); + + //testing blank node + stmt = new StatementImpl(vf.createBNode("a12345"), vf.createURI("urn:namespace#pred"), vf.createLiteral("object")); + bytes = writeStatement(stmt, true); + + readStmt = readStatement(ByteStreams.newDataInput(bytes), vf); + assertEquals(readStmt, stmt); + + //testing boolean literal datatype + stmt = new StatementImpl(vf.createURI("urn:namespace#subj"), vf.createURI("urn:namespace#pred"), vf.createLiteral(true)); + bytes = writeStatement(stmt, true); + + readStmt = readStatement(ByteStreams.newDataInput(bytes), vf); + assertEquals(readStmt, stmt); + + //testing boolean literal datatype + stmt = new StatementImpl(vf.createURI("urn:namespace#subj"), vf.createURI("urn:namespace#pred"), vf.createLiteral("label", "language")); + bytes = writeStatement(stmt, true); + + readStmt = readStatement(ByteStreams.newDataInput(bytes), vf); + assertEquals(readStmt, stmt); + } + + public void testReadIndexStatement() throws Exception { + StatementImpl stmt = new StatementImpl(vf.createURI("urn:namespace#subj"), vf.createURI("urn:namespace#pred"), vf.createLiteral("object")); + byte[] bytes = writeStatement(stmt, false); + + Statement readStmt = readStatement(ByteStreams.newDataInput(bytes), vf, false); + assertEquals(readStmt, stmt); + + bytes = writeStatement(stmt, true); + + readStmt = readStatement(ByteStreams.newDataInput(bytes), vf, true); + assertEquals(readStmt, stmt); + } + +}
