http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/extras/rya.prospector/src/test/java/mvm/rya/joinselect/mr/JoinSelectStatisticsSumTest.java ---------------------------------------------------------------------- diff --git a/extras/rya.prospector/src/test/java/mvm/rya/joinselect/mr/JoinSelectStatisticsSumTest.java b/extras/rya.prospector/src/test/java/mvm/rya/joinselect/mr/JoinSelectStatisticsSumTest.java new file mode 100644 index 0000000..525cd94 --- /dev/null +++ b/extras/rya.prospector/src/test/java/mvm/rya/joinselect/mr/JoinSelectStatisticsSumTest.java @@ -0,0 +1,59 @@ +package mvm.rya.joinselect.mr; + +/* + * #%L + * mvm.rya.rya.prospector + * %% + * Copyright (C) 2014 - 2015 Rya + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import java.io.IOException; + +import mvm.rya.joinselect.mr.JoinSelectStatisticsSum; +import mvm.rya.joinselect.mr.utils.CardList; +import mvm.rya.joinselect.mr.utils.TripleEntry; + +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mrunit.mapreduce.MapDriver; +import org.junit.Test; + +public class JoinSelectStatisticsSumTest { + + @Test + public void testFullTripleEntry() throws InterruptedException, IOException { + + TripleEntry te1 = new TripleEntry(new Text("urn:gem:etype#1234"), new Text("urn:gem#pred"), new Text("subject"), new Text("predicate"), new Text("object")); + CardList cl = new CardList(34, 52, 63, 0, 0, 0); + TripleEntry te2 = new TripleEntry(new Text("urn:gem:etype#1234"), new Text(""), new Text("subject"), new Text(""), new Text("object")); + TripleEntry te3 = new TripleEntry(new Text("urn:gem#pred"), new Text(""), new Text("predicate"), new Text(""), new Text("object")); + + new MapDriver<TripleEntry,CardList,TripleEntry,CardList>().withMapper(new JoinSelectStatisticsSum.CardinalityIdentityMapper()).withInput(te1, cl) + .withOutput(te2, cl).withOutput(te3, cl).withOutput(te1, cl).runTest(); + + } + + @Test + public void testPartialTripleEntry() throws InterruptedException, IOException { + + TripleEntry te1 = new TripleEntry(new Text("urn:gem:etype#1234"), new Text(""), new Text("subject"), new Text(""), new Text("object")); + CardList cl = new CardList(34, 52, 63, 0, 0, 0); + + new MapDriver<TripleEntry,CardList,TripleEntry,CardList>().withMapper(new JoinSelectStatisticsSum.CardinalityIdentityMapper()).withInput(te1, cl) + .withOutput(te1, cl).runTest(); + + } + +}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/extras/rya.prospector/src/test/java/mvm/rya/joinselect/mr/JoinSelectStatisticsTest.java ---------------------------------------------------------------------- diff --git a/extras/rya.prospector/src/test/java/mvm/rya/joinselect/mr/JoinSelectStatisticsTest.java b/extras/rya.prospector/src/test/java/mvm/rya/joinselect/mr/JoinSelectStatisticsTest.java new file mode 100644 index 0000000..c9a04a9 --- /dev/null +++ b/extras/rya.prospector/src/test/java/mvm/rya/joinselect/mr/JoinSelectStatisticsTest.java @@ -0,0 +1,870 @@ +package mvm.rya.joinselect.mr; + +/* + * #%L + * mvm.rya.rya.prospector + * %% + * Copyright (C) 2014 - 2015 Rya + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import static mvm.rya.joinselect.mr.utils.JoinSelectConstants.INPUTPATH; +import static mvm.rya.joinselect.mr.utils.JoinSelectConstants.INSTANCE; +import static mvm.rya.joinselect.mr.utils.JoinSelectConstants.OUTPUTPATH; +import static mvm.rya.joinselect.mr.utils.JoinSelectConstants.PASSWORD; +import static mvm.rya.joinselect.mr.utils.JoinSelectConstants.PROSPECTS_OUTPUTPATH; +import static mvm.rya.joinselect.mr.utils.JoinSelectConstants.PROSPECTS_TABLE; +import static mvm.rya.joinselect.mr.utils.JoinSelectConstants.SELECTIVITY_TABLE; +import static mvm.rya.joinselect.mr.utils.JoinSelectConstants.SPO_OUTPUTPATH; +import static mvm.rya.joinselect.mr.utils.JoinSelectConstants.SPO_TABLE; +import static mvm.rya.joinselect.mr.utils.JoinSelectConstants.USERNAME; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import mvm.rya.accumulo.AccumuloRdfConfiguration; +import mvm.rya.api.RdfCloudTripleStoreConstants.TABLE_LAYOUT; +import mvm.rya.api.domain.RyaStatement; +import mvm.rya.api.domain.RyaType; +import mvm.rya.api.domain.RyaURI; +import mvm.rya.api.resolver.RyaTripleContext; +import mvm.rya.api.resolver.triple.TripleRow; +import mvm.rya.joinselect.mr.JoinSelectAggregate.JoinReducer; +import mvm.rya.joinselect.mr.JoinSelectAggregate.JoinSelectAggregateMapper; +import mvm.rya.joinselect.mr.JoinSelectAggregate.JoinSelectGroupComparator; +import mvm.rya.joinselect.mr.JoinSelectAggregate.JoinSelectPartitioner; +import mvm.rya.joinselect.mr.JoinSelectAggregate.JoinSelectSortComparator; +import mvm.rya.joinselect.mr.JoinSelectProspectOutput.CardinalityMapper; +import mvm.rya.joinselect.mr.JoinSelectSpoTableOutput.JoinSelectMapper; +import mvm.rya.joinselect.mr.JoinSelectStatisticsSum.CardinalityIdentityCombiner; +import mvm.rya.joinselect.mr.JoinSelectStatisticsSum.CardinalityIdentityMapper; +import mvm.rya.joinselect.mr.JoinSelectStatisticsSum.CardinalityIdentityReducer; +import mvm.rya.joinselect.mr.utils.CardList; +import mvm.rya.joinselect.mr.utils.CompositeType; +import mvm.rya.joinselect.mr.utils.JoinSelectStatsUtil; +import mvm.rya.joinselect.mr.utils.TripleCard; +import mvm.rya.joinselect.mr.utils.TripleEntry; + +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.BatchWriter; +import org.apache.accumulo.core.client.BatchWriterConfig; +import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.client.Scanner; +import org.apache.accumulo.core.client.TableExistsException; +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat; +import org.apache.accumulo.core.client.mapreduce.AccumuloOutputFormat; +import org.apache.accumulo.core.client.mock.MockInstance; +import org.apache.accumulo.core.client.security.tokens.PasswordToken; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Mutation; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.security.Authorizations; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.lib.input.MultipleInputs; +import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat; +import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class JoinSelectStatisticsTest { + + private static final String PREFIX = JoinSelectStatisticsTest.class.getSimpleName(); + + private static final String DELIM = "\u0000"; + private static final String uri = "uri:"; + private List<String> cardList = Arrays.asList("subject", "predicate", "object"); + private List<String> aggCardList = Arrays.asList("subjectobject", "subjectpredicate", "subjectsubject", "predicateobject", "predicatepredicate", "predicatesubject"); + private static File SPOOUT; + private static File PROSPECTSOUT; + private static File tempDir; + private Connector c; + private RyaTripleContext ryaContext; + private static final String INSTANCE_NAME = "mapreduce_instance"; + + private static class JoinSelectTester1 extends Configured implements Tool { + + + + @Override + public int run(String[] args) throws Exception { + + Configuration conf = getConf(); + + String inTable = conf.get(SPO_TABLE); + String outPath = conf.get(SPO_OUTPUTPATH); + + + assert inTable != null && outPath != null; + + Job job = new Job(conf, this.getClass().getSimpleName() + "_" + System.currentTimeMillis()); + job.setJarByClass(this.getClass()); + job.setUserClassesTakesPrecedence(true); + + initTabToSeqFileJob(job, inTable, outPath); + job.setMapperClass(JoinSelectMapper.class); + job.setNumReduceTasks(0); + job.waitForCompletion(true); + + return job.isSuccessful() ? 0 : 1; + } + } + + private static class JoinSelectTester2 extends Configured implements Tool { + + + + @Override + public int run(String[] args) throws Exception { + + Configuration conf = getConf(); + + String inTable = conf.get(PROSPECTS_TABLE); + System.out.println("Table is " + inTable); + String outPath = conf.get(PROSPECTS_OUTPUTPATH); + + + assert inTable != null && outPath != null; + + Job job = new Job(conf, this.getClass().getSimpleName() + "_" + System.currentTimeMillis()); + job.setJarByClass(this.getClass()); + job.setUserClassesTakesPrecedence(true); + + initTabToSeqFileJob(job, inTable, outPath); + job.setMapperClass(CardinalityMapper.class); + job.setNumReduceTasks(0); + job.waitForCompletion(true); + + return job.isSuccessful() ? 0 : 1; + } + } + + + private static class JoinSelectTester4 extends Configured implements Tool { + + + + @Override + public int run(String[] args) throws Exception { + + Configuration conf = getConf(); + String outpath = conf.get(OUTPUTPATH); + + Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis()); + job.setJarByClass(this.getClass()); + job.setUserClassesTakesPrecedence(true); + + MultipleInputs.addInputPath(job, new Path(PROSPECTSOUT.getAbsolutePath()), + SequenceFileInputFormat.class, JoinSelectAggregateMapper.class); + MultipleInputs.addInputPath(job,new Path(SPOOUT.getAbsolutePath()) , + SequenceFileInputFormat.class, JoinSelectAggregateMapper.class); + job.setMapOutputKeyClass(CompositeType.class); + job.setMapOutputValueClass(TripleCard.class); + + tempDir = new File(File.createTempFile(outpath, "txt").getParentFile(), System.currentTimeMillis() + ""); + SequenceFileOutputFormat.setOutputPath(job, new Path(tempDir.getAbsolutePath())); + job.setOutputFormatClass(SequenceFileOutputFormat.class); + job.setOutputKeyClass(TripleEntry.class); + job.setOutputValueClass(CardList.class); + + + job.setSortComparatorClass(JoinSelectSortComparator.class); + job.setGroupingComparatorClass(JoinSelectGroupComparator.class); + job.setPartitionerClass(JoinSelectPartitioner.class); + job.setReducerClass(JoinReducer.class); + job.setNumReduceTasks(32); + job.waitForCompletion(true); + + return job.isSuccessful() ? 0 : 1; + } + } + + + + private static class JoinSelectTester3 extends Configured implements Tool { + + + + @Override + public int run(String[] args) throws Exception { + + Configuration conf = getConfig(); + + String outTable = conf.get(SELECTIVITY_TABLE); + String inPath = conf.get(INPUTPATH); + + + assert outTable != null && inPath != null; + + Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis()); + job.setJarByClass(this.getClass()); + initSumMRJob(job, inPath, outTable); + + job.setMapperClass(CardinalityIdentityMapper.class); + job.setCombinerClass(CardinalityIdentityCombiner.class); + job.setReducerClass(CardinalityIdentityReducer.class); + job.setNumReduceTasks(32); + job.waitForCompletion(true); + + return job.isSuccessful() ? 0 : 1; + + } + + + + } + + + + + + + + public class JoinSelectTestDriver extends Configured implements Tool { + + Configuration conf = getConfig(); + + @Override + public int run(String[] args) throws Exception { + + int res0 = ToolRunner.run(conf, new JoinSelectTester1(), args); + int res1 = 1; + int res2 = 1; + int res3 = 1; + + + + if(res0 == 0) { + res1 = ToolRunner.run(conf, new JoinSelectTester2(), args); + } + if(res1 == 0) { + res2 = ToolRunner.run(conf, new JoinSelectTester4(), args); + } + if(res2 == 0) { + res3 = ToolRunner.run(conf, new JoinSelectTester3(), args); + } + + return res3; + } + + } + + + + + private static Configuration getConfig() { + + Configuration conf = new Configuration(); + conf.set("fs.default.name", "file:///"); + conf.set("mapreduce.framework.name", "local"); + conf.set("spo.table", "rya_spo"); + conf.set("prospects.table", "rya_prospects"); + conf.set("selectivity.table", "rya_selectivity"); + conf.set("auths", ""); + conf.set("instance",INSTANCE_NAME); + conf.set("username","root"); + conf.set("password", ""); + conf.set("inputpath","temp"); + conf.set("outputpath","temp"); + conf.set("prospects.outputpath","prospects"); + conf.set("spo.outputpath", "spo"); + + + return conf; + + } + + + + + + + + + + + public static void initTabToSeqFileJob(Job job, String intable, String outpath) throws AccumuloSecurityException, IOException { + + Configuration conf = job.getConfiguration(); + + String username = conf.get(USERNAME); + System.out.println("Username is " + username); + String password = conf.get(PASSWORD); + String instance = conf.get(INSTANCE); + System.out.println("Instance is " + instance); + + + AccumuloInputFormat.setMockInstance(job, instance); + AccumuloInputFormat.setConnectorInfo(job, username, new PasswordToken(password)); + AccumuloInputFormat.setInputTableName(job, intable); + + job.setInputFormatClass(AccumuloInputFormat.class); + job.setMapOutputKeyClass(CompositeType.class); + job.setMapOutputValueClass(TripleCard.class); + + System.out.println("Outpath is " + outpath); + + // OUTPUT + if(outpath.equals("spo")) { + SPOOUT = new File(File.createTempFile(outpath, "txt").getParentFile(), System.currentTimeMillis() + "spo"); + SequenceFileOutputFormat.setOutputPath(job, new Path(SPOOUT.getAbsolutePath())); + } else { + PROSPECTSOUT = new File(File.createTempFile(outpath, "txt").getParentFile(), System.currentTimeMillis() + "prospects"); + SequenceFileOutputFormat.setOutputPath(job, new Path(PROSPECTSOUT.getAbsolutePath())); + } + job.setOutputFormatClass(SequenceFileOutputFormat.class); + job.setOutputKeyClass(CompositeType.class); + job.setOutputValueClass(TripleCard.class); + + } + + public static void initSumMRJob(Job job, String inputPath, String outtable) throws AccumuloSecurityException, IOException { + + Configuration conf = job.getConfiguration(); + + String username = conf.get(USERNAME); + String password = conf.get(PASSWORD); + String instance = conf.get(INSTANCE); + + + + AccumuloOutputFormat.setConnectorInfo(job, username, new PasswordToken(password)); + AccumuloOutputFormat.setMockInstance(job, instance); + AccumuloOutputFormat.setDefaultTableName(job, outtable); + + + SequenceFileInputFormat.addInputPath(job, new Path(tempDir.getAbsolutePath())); + job.setInputFormatClass(SequenceFileInputFormat.class); + job.setMapOutputKeyClass(TripleEntry.class); + job.setMapOutputValueClass(CardList.class); + + + job.setOutputFormatClass(AccumuloOutputFormat.class); + job.setOutputKeyClass(Text.class); + job.setOutputValueClass(Mutation.class); + + + } + + + + + + @Before + public void init() throws AccumuloException, AccumuloSecurityException, TableExistsException, TableNotFoundException { + + MockInstance mockInstance = new MockInstance(INSTANCE_NAME); + c = mockInstance.getConnector("root", new PasswordToken("")); + + if (c.tableOperations().exists("rya_prospects")) { + c.tableOperations().delete("rya_prospects"); + } + if (c.tableOperations().exists("rya_selectivity")) { + c.tableOperations().delete("rya_selectivity"); + } + if (c.tableOperations().exists("rya_spo")) { + c.tableOperations().delete("rya_spo"); + } + + + c.tableOperations().create("rya_spo"); + c.tableOperations().create("rya_prospects"); + c.tableOperations().create("rya_selectivity"); + ryaContext = RyaTripleContext.getInstance(new AccumuloRdfConfiguration(getConfig())); + } + + + + + + + + + + @Test + public void testMap1() throws Exception { + init(); + + System.out.println("*****************************Test1**************************** "); + + BatchWriter bw_table1 = c.createBatchWriter("rya_spo", new BatchWriterConfig()); + for (int i = 1; i < 3; i++) { + + RyaStatement rs = new RyaStatement(new RyaURI(uri + i), new RyaURI(uri + 5), new RyaType(uri + (i + 2))); + Map<TABLE_LAYOUT, TripleRow> tripleRowMap = ryaContext.serializeTriple(rs); + TripleRow tripleRow = tripleRowMap.get(TABLE_LAYOUT.SPO); + Mutation m = JoinSelectStatsUtil.createMutation(tripleRow); + bw_table1.addMutation(m); + + + } + bw_table1.close(); + + BatchWriter bw_table2 = c.createBatchWriter("rya_prospects", new BatchWriterConfig()); + for (int i = 1; i < 6; i++) { + + int j = 1; + + for (String s : cardList) { + Mutation m = new Mutation(new Text(s + DELIM + uri + i + DELIM + i)); + m.put(new Text(), new Text(), new Value(new IntWritable(i + j).toString().getBytes())); + bw_table2.addMutation(m); + j++; + } + + } + bw_table2.close(); + + + + + + Assert.assertEquals(0, ToolRunner.run(new JoinSelectTestDriver(), new String[]{""})); + Scanner scan = c.createScanner("rya_selectivity", new Authorizations()); + scan.setRange(new Range()); + + for (Map.Entry<Key, Value> entry : scan) { + System.out.println("Key row string is " + entry.getKey().getRow().toString()); + System.out.println("Join type is " + entry.getKey().getColumnFamily().toString()); + System.out.println("Value is " + entry.getKey().getColumnQualifier().toString()); + } + + Scanner scan1 = c.createScanner("rya_selectivity" , new Authorizations()); + scan1.setRange(Range.prefix("predicate" +DELIM + uri + 5)); + int i = 5; + + for (Map.Entry<Key, Value> entry : scan1) { + + int val1 = 5 + 2*i; + int val2 = 5 + 2*(i-1); + int val = Integer.parseInt(entry.getKey().getColumnQualifier().toString()); + + if(i < 3) { + Assert.assertTrue( val == val1); + } + if(i >= 3 && i < 6) { + Assert.assertTrue(val == val2); + } + i--; + } + Assert.assertTrue(i == -1); + + + + Scanner scan2 = c.createScanner("rya_selectivity" , new Authorizations()); + scan2.setRange(Range.prefix("object" +DELIM + uri + 3)); + int j = 5; + + for (Map.Entry<Key, Value> entry : scan2) { + + int val1 = 5 + (j-2); + int val2 = 2+j; + int val = Integer.parseInt(entry.getKey().getColumnQualifier().toString()); + + if(j < 3) { + Assert.assertTrue( val == val2); + } + if(j >= 3 && j < 6) { + Assert.assertTrue(val == val1); + } + j--; + } + Assert.assertTrue(j == -1); + + + + + Scanner scan3 = c.createScanner("rya_selectivity", new Authorizations()); + scan3.setRange(Range.prefix("objectsubject" + DELIM + uri + 3 +DELIM +uri +1 )); + int k = 8; + + for (Map.Entry<Key, Value> entry : scan3) { + + int val = Integer.parseInt(entry.getKey().getColumnQualifier().toString()); + + Assert.assertTrue(val == k); + k--; + } + Assert.assertTrue(k == 5); + + + + + + + + } + + + + + + @Test + public void testMap2() throws Exception { + + System.out.println("*********************Test2******************* "); + + init(); + + BatchWriter bw_table1 = c.createBatchWriter("rya_spo", new BatchWriterConfig()); + for (int i = 1; i < 4; i++) { + + RyaStatement rs = new RyaStatement(new RyaURI(uri + 1), new RyaURI(uri + 2), new RyaType(uri + i)); + Map<TABLE_LAYOUT, TripleRow> tripleRowMap = ryaContext.serializeTriple(rs); + TripleRow tripleRow = tripleRowMap.get(TABLE_LAYOUT.SPO); + Mutation m = JoinSelectStatsUtil.createMutation(tripleRow); + bw_table1.addMutation(m); + + } + bw_table1.close(); + + BatchWriter bw_table2 = c.createBatchWriter("rya_prospects", new BatchWriterConfig()); + for (int i = 1; i < 4; i++) { + + for (String s : cardList) { + Mutation m = new Mutation(new Text(s + DELIM + uri + i + DELIM + i)); + m.put(new Text(), new Text(), new Value(new IntWritable(i + 2).toString().getBytes())); + bw_table2.addMutation(m); + } + + } + bw_table2.close(); + + Assert.assertEquals(0, ToolRunner.run(new JoinSelectTestDriver(), new String[]{""})); + Scanner scan1 = c.createScanner("rya_selectivity" , new Authorizations()); + scan1.setRange(Range.prefix("subject" +DELIM + uri + 1)); + int i = 0; + + for (Map.Entry<Key, Value> entry : scan1) { + + Assert.assertTrue(entry.getKey().getColumnQualifier().toString().equals("12")); + i++; + } + Assert.assertTrue(i == 6); + + Scanner scan2 = c.createScanner("rya_selectivity" , new Authorizations()); + scan2.setRange(Range.prefix("predicate" +DELIM + uri + 2)); + int j = 0; + + for (Map.Entry<Key, Value> entry : scan2) { + + if(j < 3) { + Assert.assertTrue(entry.getKey().getColumnQualifier().toString().equals("12")); + } + if(j > 3 && j < 6) { + Assert.assertTrue(entry.getKey().getColumnQualifier().toString().equals("9")); + } + j++; + } + Assert.assertTrue(j == 6); + + Scanner scan3 = c.createScanner("rya_selectivity" , new Authorizations()); + scan3.setRange(Range.prefix("predicateobject" +DELIM + uri + 2 +DELIM + uri + 2)); + int k = 0; + + for (Map.Entry<Key, Value> entry : scan3) { + Assert.assertTrue(entry.getKey().getColumnQualifier().toString().equals("3")); + k++; + } + Assert.assertTrue(k == 3); + + + } + + + + + @Test + public void testMap3() throws Exception { + init(); + + System.out.println("*************************Test3**************************** "); + + BatchWriter bw_table1 = c.createBatchWriter("rya_spo", new BatchWriterConfig()); + for (int i = 1; i < 3; i++) { + for (int j = 1; j < 3; j++) { + for (int k = 1; k < 3; k++) { + + RyaStatement rs = new RyaStatement(new RyaURI(uri + i), new RyaURI(uri + (j)), new RyaType(uri + k)); + Map<TABLE_LAYOUT, TripleRow> tripleRowMap = ryaContext.serializeTriple(rs); + TripleRow tripleRow = tripleRowMap.get(TABLE_LAYOUT.SPO); + Mutation m = JoinSelectStatsUtil.createMutation(tripleRow); + bw_table1.addMutation(m); + + } + } + + } + bw_table1.close(); + + BatchWriter bw_table2 = c.createBatchWriter("rya_prospects", new BatchWriterConfig()); + for (int i = 1; i < 3; i++) { + + int k = 1; + for (String s : cardList) { + Mutation m = new Mutation(new Text(s + DELIM + uri + i + DELIM + i)); + m.put(new Text(), new Text(), new Value(new IntWritable(i + k).toString().getBytes())); + bw_table2.addMutation(m); + k++; + } + + for (int j = 1; j < 3; j++) { + k = 1; + for (String s : aggCardList) { + Mutation m = new Mutation(new Text(s + DELIM + uri + i + DELIM + uri + j + DELIM + i)); + m.put(new Text(), new Text(), new Value(new IntWritable(i + k +j).toString().getBytes())); + bw_table2.addMutation(m); + k++; + } + } + + } + bw_table2.close(); + + + + + + Assert.assertEquals(0, ToolRunner.run(new JoinSelectTestDriver(), new String[]{""})); + Scanner scan = c.createScanner("rya_selectivity", new Authorizations()); + scan.setRange(new Range()); + + for (Map.Entry<Key, Value> entry : scan) { + System.out.println("Key row string is " + entry.getKey().getRow().toString()); + System.out.println("Join type is " + entry.getKey().getColumnFamily().toString()); + System.out.println("Value is " + entry.getKey().getColumnQualifier().toString()); + } + + + + Scanner scan1 = c.createScanner("rya_selectivity" , new Authorizations()); + scan1.setRange(Range.prefix("subject" +DELIM + uri + 1)); + int i = 0; + + for (Map.Entry<Key, Value> entry : scan1) { + + Key key = entry.getKey(); + String s = key.getColumnFamily().toString(); + int val = Integer.parseInt(key.getColumnQualifier().toString()); + + if(s.equals("predicatepredicate")) { + Assert.assertTrue(val == 14); + } + if(s.equals("objectobject")) { + Assert.assertTrue(val == 18); + } + if(s.equals("predicateobjectpredicateobject")) { + Assert.assertTrue(val == 28); + } + if(s.equals("predicateobjectsubjectpredicate")) { + Assert.assertTrue(val == 20); + } + if(s.equals("predicateobjectobjectsubject")) { + Assert.assertTrue(val == 16); + } + + i++; + } + Assert.assertTrue(i == 12); + + + + + + + + } + + + + + + @Test + public void testMap4() throws Exception { + init(); + + System.out.println("*************************Test4**************************** "); + System.out.println("*************************Test4**************************** "); + + BatchWriter bw_table1 = c.createBatchWriter("rya_spo", new BatchWriterConfig()); + for (int i = 1; i < 3; i++) { + for (int j = 1; j < 3; j++) { + for (int k = 1; k < 3; k++) { + + if(j == 1 && k ==2) { + break; + } + + RyaStatement rs = new RyaStatement(new RyaURI(uri + i), new RyaURI(uri + (j)), new RyaType(uri + k)); + Map<TABLE_LAYOUT, TripleRow> tripleRowMap = ryaContext.serializeTriple(rs); + TripleRow tripleRow = tripleRowMap.get(TABLE_LAYOUT.SPO); + Mutation m = JoinSelectStatsUtil.createMutation(tripleRow); + bw_table1.addMutation(m); + + } + } + + } + bw_table1.close(); + + BatchWriter bw_table2 = c.createBatchWriter("rya_prospects", new BatchWriterConfig()); + for (int i = 1; i < 3; i++) { + + int k = 1; + for (String s : cardList) { + Mutation m = new Mutation(new Text(s + DELIM + uri + i + DELIM + i)); + m.put(new Text(), new Text(), new Value(new IntWritable(i + k).toString().getBytes())); + bw_table2.addMutation(m); + k++; + } + + for (int j = 1; j < 3; j++) { + k = 1; + for (String s : aggCardList) { + Mutation m = new Mutation(new Text(s + DELIM + uri + i + DELIM + uri + j + DELIM + i)); + m.put(new Text(), new Text(), new Value(new IntWritable(i + k + 2*j).toString().getBytes())); + bw_table2.addMutation(m); + k++; + } + } + + } + bw_table2.close(); + + + + + + Assert.assertEquals(0, ToolRunner.run(new JoinSelectTestDriver(), new String[]{""})); + Scanner scan = c.createScanner("rya_selectivity", new Authorizations()); + scan.setRange(new Range()); + + for (Map.Entry<Key, Value> entry : scan) { + System.out.println("Key row string is " + entry.getKey().getRow().toString()); + System.out.println("Join type is " + entry.getKey().getColumnFamily().toString()); + System.out.println("Value is " + entry.getKey().getColumnQualifier().toString()); + } + + + + Scanner scan1 = c.createScanner("rya_selectivity" , new Authorizations()); + scan1.setRange(Range.prefix("subject" +DELIM + uri + 1)); + int i = 0; + + for (Map.Entry<Key, Value> entry : scan1) { + + Key key = entry.getKey(); + String s = key.getColumnFamily().toString(); + int val = Integer.parseInt(key.getColumnQualifier().toString()); + + if(s.equals("predicatepredicate")) { + Assert.assertTrue(val == 11); + } + if(s.equals("objectobject")) { + Assert.assertTrue(val == 13); + } + if(s.equals("predicateobjectobjectpredicate")) { + Assert.assertTrue(val == 26); + } + if(s.equals("predicateobjectpredicateobject")) { + Assert.assertTrue(val == 25); + } + if(s.equals("predicateobjectsubjectpredicate")) { + Assert.assertTrue(val == 19); + } + if(s.equals("predicateobjectpredicatesubject")) { + Assert.assertTrue(val == 20); + } + + i++; + } + Assert.assertTrue(i == 12); + + + + Scanner scan2 = c.createScanner("rya_selectivity" , new Authorizations()); + scan2.setRange(Range.prefix("predicate" +DELIM + uri + 1)); + int j = 0; + + for (Map.Entry<Key, Value> entry : scan2) { + + Key key = entry.getKey(); + String s = key.getColumnFamily().toString(); + int val = Integer.parseInt(key.getColumnQualifier().toString()); + + if(s.equals("subjectsubject")) { + Assert.assertTrue(val == 5); + } + if(s.equals("objectobject")) { + Assert.assertTrue(val == 8); + } + if(s.equals("objectsubjectsubjectpredicate")) { + Assert.assertTrue(val == 11); + } + if(s.equals("objectsubjectpredicateobject")) { + Assert.assertTrue(val == 15); + } + if(s.equals("objectsubjectobjectsubject")) { + Assert.assertTrue(val == 9); + } + if(s.equals("objectsubjectsubjectobject")) { + Assert.assertTrue(val == 10); + } + + j++; + } + Assert.assertTrue(j == 12); + + + + + + + + + } + + + + + + + + + + + + +} + + + + + + + \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/extras/rya.prospector/src/test/resources/stats_cluster_config.xml ---------------------------------------------------------------------- diff --git a/extras/rya.prospector/src/test/resources/stats_cluster_config.xml b/extras/rya.prospector/src/test/resources/stats_cluster_config.xml new file mode 100644 index 0000000..7287813 --- /dev/null +++ b/extras/rya.prospector/src/test/resources/stats_cluster_config.xml @@ -0,0 +1,76 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<configuration> + <!-- Accumulo info --> + + <property> + <name>mock</name> + <value>true</value> + </property> + <property> + <name>instance</name> + <value>accumulo</value> + </property> + <property> + <name>zookeepers</name> + <value>zoo1,zoo2,zoo3</value> + </property> + + <!-- User info --> + <property> + <name>username</name> + <value>user</value> + </property> + <property> + <name>password</name> + <value>pass</value> + </property> + + <!-- Rya info --> + <property> + <name>spo.table</name> + <value>rya_spo</value> + </property> + <property> + <name>prospects.table</name> + <value>rya_prospects</value> + </property> + <property> + <name>selectivity.table</name> + <value>rya_selectivity</value> + </property> + <property> + <name>auths</name> + <value>U</value> + </property> + <property> + <name>prospector.auths</name> + <value>U</value> + </property> + + <property> + <name>prospector.intable</name> + <value>rya_spo</value> + </property> + <property> + <name>prospector.outtable</name> + <value>rya_prospects</value> + </property> + + <property> + <name>inputpath</name> + <value>/tmp/RyaStats/JoinSelectStatisticsSumInput</value> + </property> + <property> + <name>outputpath</name> + <value>/tmp/RyaStats/JoinSelectStatisticsSumInput</value> + </property> + <property> + <name>prospects.outputpath</name> + <value>/tmp/RyaStats/ProspectsOutput</value> + </property> + <property> + <name>spo.outputpath</name> + <value>/tmp/RyaStats/SpoOutput</value> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/extras/tinkerpop.rya/pom.xml ---------------------------------------------------------------------- diff --git a/extras/tinkerpop.rya/pom.xml b/extras/tinkerpop.rya/pom.xml new file mode 100644 index 0000000..04db4ac --- /dev/null +++ b/extras/tinkerpop.rya/pom.xml @@ -0,0 +1,136 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <parent> + <groupId>mvm.rya</groupId> + <artifactId>rya.extras</artifactId> + <version>3.2.9</version> + </parent> + <modelVersion>4.0.0</modelVersion> + <name>${project.groupId}.${project.artifactId}</name> + + <artifactId>tinkerpop.rya</artifactId> + + <dependencies> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>mvm.rya</groupId> + <artifactId>rya.sail.impl</artifactId> + </dependency> + <dependency> + <groupId>mvm.rya</groupId> + <artifactId>accumulo.rya</artifactId> + </dependency> + <dependency> + <groupId>com.tinkerpop.gremlin</groupId> + <artifactId>gremlin-groovy</artifactId> + </dependency> + <dependency> + <groupId>com.tinkerpop.rexster</groupId> + <artifactId>rexster-server</artifactId> + <version>${blueprints.version}</version> + </dependency> + <dependency> + <groupId>com.tinkerpop.blueprints</groupId> + <artifactId>blueprints-sail-graph</artifactId> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.codehaus.gmaven</groupId> + <artifactId>gmaven-plugin</artifactId> + <version>1.3</version> + <dependencies> + <dependency> + <groupId>org.codehaus.groovy</groupId> + <artifactId>groovy-all</artifactId> + <version>1.8.6</version> + </dependency> + <dependency> + <groupId>org.codehaus.gmaven.runtime</groupId> + <artifactId>gmaven-runtime-1.7</artifactId> + <version>1.3</version> + <exclusions> + <exclusion> + <groupId>org.codehaus.groovy</groupId> + <artifactId>groovy-all</artifactId> + </exclusion> + </exclusions> + </dependency> + </dependencies> + <executions> + <execution> + <configuration> + <providerSelection>1.7</providerSelection> + </configuration> + <goals> + <goal>generateStubs</goal> + <goal>compile</goal> + <goal>generateTestStubs</goal> + <goal>testCompile</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> + + <profiles> + <profile> + <id>accumulo</id> + <activation> + <activeByDefault>true</activeByDefault> + </activation> + <dependencies> + <dependency> + <groupId>org.apache.accumulo</groupId> + <artifactId>accumulo-core</artifactId> + <optional>true</optional> + </dependency> + </dependencies> + </profile> + <profile> + <id>cloudbase</id> + <activation> + <activeByDefault>false</activeByDefault> + </activation> + <dependencies> + <dependency> + <groupId>com.texeltek</groupId> + <artifactId>accumulo-cloudbase-shim</artifactId> + <optional>true</optional> + </dependency> + </dependencies> + </profile> + <profile> + <id>uberjar</id> + <build> + <plugins> + <plugin> + <artifactId>maven-assembly-plugin</artifactId> + <configuration> + <descriptorRefs> + <descriptorRef>jar-with-dependencies</descriptorRef> + </descriptorRefs> + </configuration> + <executions> + <execution> + <id>make-assembly</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + </profiles> + +</project> http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/extras/tinkerpop.rya/src/main/groovy/mvm/rya/blueprints/config/RyaGraphConfiguration.groovy ---------------------------------------------------------------------- diff --git a/extras/tinkerpop.rya/src/main/groovy/mvm/rya/blueprints/config/RyaGraphConfiguration.groovy b/extras/tinkerpop.rya/src/main/groovy/mvm/rya/blueprints/config/RyaGraphConfiguration.groovy new file mode 100644 index 0000000..6559370 --- /dev/null +++ b/extras/tinkerpop.rya/src/main/groovy/mvm/rya/blueprints/config/RyaGraphConfiguration.groovy @@ -0,0 +1,84 @@ +package mvm.rya.blueprints.config + +import com.tinkerpop.blueprints.Direction; +import com.tinkerpop.rexster.config.GraphConfiguration +import com.tinkerpop.rexster.config.GraphConfigurationContext; + +import mvm.rya.accumulo.AccumuloRdfConfiguration +import mvm.rya.accumulo.AccumuloRyaDAO +import mvm.rya.blueprints.sail.RyaSailGraph +import mvm.rya.rdftriplestore.RdfCloudTripleStore +import mvm.rya.rdftriplestore.inference.InferenceEngine +import org.apache.commons.configuration.Configuration +import static mvm.rya.accumulo.mr.utils.MRUtils.* +import org.apache.commons.configuration.MapConfiguration +import mvm.rya.blueprints.sail.RyaSailEdge +import mvm.rya.blueprints.sail.RyaSailVertex +import org.apache.accumulo.core.client.mock.MockInstance +import org.apache.accumulo.core.client.ZooKeeperInstance + +/** + * Date: 5/8/12 + * Time: 5:38 PM + */ +class RyaGraphConfiguration implements GraphConfiguration { + + def instance, zk, user, pwd, tablePrefix, auths, cv, ttl, mock + + public static final RyaSailGraph createGraph(Map<String, String> props) { + if (props == null) props = [:] + def graphConfiguration = new RyaGraphConfiguration() + RyaGraphConfiguration.load() + return graphConfiguration.configureGraphInstance(new GraphConfigurationContext(new MapConfiguration(props), new HashMap())) + } + + public static void load() { + RyaSailEdge.metaClass.getSubj = { (delegate as RyaSailEdge).getVertex(Direction.OUT).id } + RyaSailEdge.metaClass.getPred = { (delegate as RyaSailEdge).label } + RyaSailEdge.metaClass.getObj = { (delegate as RyaSailEdge).getVertex(Direction.IN).id } + RyaSailEdge.metaClass.getCntxt = { (delegate as RyaSailEdge).namedGraph } + RyaSailEdge.metaClass.getStmt = { (delegate as RyaSailEdge).rawEdge } + } + + @Override + public RyaSailGraph configureGraphInstance(GraphConfigurationContext context) { + Configuration graphConfiguration = context.getProperties() + instance = graphConfiguration.getString(AC_INSTANCE_PROP) + zk = graphConfiguration.getString(AC_ZK_PROP) + user = graphConfiguration.getString(AC_USERNAME_PROP) + pwd = graphConfiguration.getString(AC_PWD_PROP) + mock = (graphConfiguration.containsKey(AC_MOCK_PROP)) ? (graphConfiguration.getBoolean(AC_MOCK_PROP)) : (null) + assert instance != null && (zk != null || mock != null) && user != null && pwd != null + + def ryaConfiguration = new AccumuloRdfConfiguration(); + //set other properties + graphConfiguration.keys.each { key -> + def val = graphConfiguration.getString(key) + if (val != null) { + ryaConfiguration.set(key, val) + } + } + //set table prefix + ryaConfiguration.setTablePrefix(ryaConfiguration.getTablePrefix()) + + def store = new RdfCloudTripleStore(); + store.setConf(ryaConfiguration); + def cryadao = new AccumuloRyaDAO(); + def connector = null + if (mock) { + connector = new MockInstance(instance).getConnector(user, pwd); + } else { + connector = new ZooKeeperInstance(instance, zk).getConnector(user, pwd); + } + cryadao.setConnector(connector); + store.setRyaDAO(cryadao); +// def ceval = new (); +// ceval.setConnector(connector); +// store.setRdfEvalStatsDAO(ceval); + def inferenceEngine = new InferenceEngine(); + inferenceEngine.setRyaDAO(cryadao); + store.setInferenceEngine(inferenceEngine); + + return new RyaSailGraph(store) + } +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/extras/tinkerpop.rya/src/main/groovy/mvm/rya/blueprints/sail/RyaSailEdge.groovy ---------------------------------------------------------------------- diff --git a/extras/tinkerpop.rya/src/main/groovy/mvm/rya/blueprints/sail/RyaSailEdge.groovy b/extras/tinkerpop.rya/src/main/groovy/mvm/rya/blueprints/sail/RyaSailEdge.groovy new file mode 100644 index 0000000..13a6470 --- /dev/null +++ b/extras/tinkerpop.rya/src/main/groovy/mvm/rya/blueprints/sail/RyaSailEdge.groovy @@ -0,0 +1,75 @@ +//package mvm.rya.blueprints.sail +// +//import com.tinkerpop.blueprints.pgm.impls.sail.SailEdge +//import org.openrdf.model.Statement +//import org.openrdf.model.impl.ContextStatementImpl +//import org.openrdf.model.impl.StatementImpl +// +///** +// * Blueprints Edge for Sail stores +// * outVertex edge inVertex +// * +// * Date: 5/9/12 +// * Time: 9:03 AM +// */ +//class RyaSailEdge extends SailEdge { +// +// public static final String SPLIT = "|" +// +// RyaSailEdge(Statement rawEdge, RyaSailGraph graph) { +// super(rawEdge, graph) +// } +// +// @Override +// Object getId() { +// def statement = this.getRawEdge() +// return formatId(statement); +// } +// +// /** +// * Returns a formatted id for a full statement. +// * @param statement +// * @return +// */ +// static String formatId(Statement statement) { +// if (null != statement.getContext()) +// return (new StringBuilder()).append(statement.getSubject()).append(SPLIT).append(statement.getPredicate()).append(SPLIT).append(statement.getObject()).append(SPLIT).append(statement.getContext()).toString(); +// else +// return (new StringBuilder()).append(statement.getSubject()).append(SPLIT).append(statement.getPredicate()).append(SPLIT).append(statement.getObject()).toString() +// } +// +//// public static RyaSailEdge fromId(String id, RyaSailGraph graph) { +//// def decodedId = URLDecoder.decode(id) +//// def statement = RdfIO.readStatement(ByteStreams.newDataInput(decodedId.bytes), RdfCloudTripleStoreConstants.VALUE_FACTORY) +//// println statement +//// return new RyaSailEdge(statement, graph) +//// } +// +// /** +// * +// * @param id formatted from getId method +// * @param graph +// * @return +// */ +// public static RyaSailEdge fromId(String id, RyaSailGraph graph) { +// assert id != null +// def split = id.split("\\|") +// if(split.length < 3) { +// return null +// } +// String subj_s = split[0].trim() +// def subj = graph.createValue(subj_s) +// String pred_s = split[1].trim() +// def pred = graph.createValue(pred_s) +// String obj_s = split[2].trim() +// def obj = graph.createValue(obj_s) +// if(split.length == 4) { +// //context available +// def context = graph.createValue(split[3]) +// return new RyaSailEdge(new ContextStatementImpl(subj, pred, obj, context), graph); +// } else { +// return new RyaSailEdge(new StatementImpl(subj, pred, obj), graph); +// } +// } +// +//} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/extras/tinkerpop.rya/src/main/groovy/mvm/rya/blueprints/sail/RyaSailEdgeSequence.groovy ---------------------------------------------------------------------- diff --git a/extras/tinkerpop.rya/src/main/groovy/mvm/rya/blueprints/sail/RyaSailEdgeSequence.groovy b/extras/tinkerpop.rya/src/main/groovy/mvm/rya/blueprints/sail/RyaSailEdgeSequence.groovy new file mode 100644 index 0000000..2bf26af --- /dev/null +++ b/extras/tinkerpop.rya/src/main/groovy/mvm/rya/blueprints/sail/RyaSailEdgeSequence.groovy @@ -0,0 +1,91 @@ +package mvm.rya.blueprints.sail + +import com.tinkerpop.blueprints.Edge +import info.aduna.iteration.Iteration +import info.aduna.iteration.Iterations +import info.aduna.iteration.IteratorIteration +import org.openrdf.model.Statement +import org.openrdf.sail.SailException + +/** + * Edge iterable that returns RyaSailEdge + * Date: 5/9/12 + * Time: 9:26 AM + */ +class RyaSailEdgeSequence implements Iterable<Edge>, Iterator<Edge> +{ + + protected Iteration<? extends Statement, SailException> statements; + protected RyaSailGraph graph; + + public RyaSailEdgeSequence(Iteration statements, RyaSailGraph graph) + { + this.statements = statements; + this.graph = graph; + } + + public RyaSailEdgeSequence(Iterator iterator, RyaSailGraph graph) { + this(new IteratorIteration(iterator), graph) + } + + public RyaSailEdgeSequence() + { + statements = null; + graph = null; + } + + public Iterator iterator() + { + return this; + } + + public void remove() + { + throw new UnsupportedOperationException(); + } + + public boolean hasNext() + { + if(null == statements) + return false; + try + { + if(statements.hasNext()) + return true; + } + catch(SailException e) + { + throw new RuntimeException(e.getMessage(), e); + } + Iterations.closeCloseable(statements); + return false; + } + + public Edge next() + { + if(null == statements) + throw new NoSuchElementException(); + try + { + def statement = (Statement) statements.next() + return new RyaSailEdge(statement, graph); + } + catch(SailException e) + { + throw new RuntimeException(e.getMessage()); + } + catch(NoSuchElementException e) + { + try + { + Iterations.closeCloseable(statements); + } + catch(SailException e2) + { + throw new RuntimeException(e2.getMessage(), e2); + } + throw e; + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/extras/tinkerpop.rya/src/main/groovy/mvm/rya/blueprints/sail/RyaSailGraph.groovy ---------------------------------------------------------------------- diff --git a/extras/tinkerpop.rya/src/main/groovy/mvm/rya/blueprints/sail/RyaSailGraph.groovy b/extras/tinkerpop.rya/src/main/groovy/mvm/rya/blueprints/sail/RyaSailGraph.groovy new file mode 100644 index 0000000..e5783e2 --- /dev/null +++ b/extras/tinkerpop.rya/src/main/groovy/mvm/rya/blueprints/sail/RyaSailGraph.groovy @@ -0,0 +1,112 @@ +package mvm.rya.blueprints.sail + +import com.tinkerpop.blueprints.impls.sail.SailGraph +import com.tinkerpop.blueprints.impls.sail.SailHelper +import com.tinkerpop.blueprints.impls.sail.SailTokens +import org.openrdf.model.Literal +import org.openrdf.model.Resource +import org.openrdf.model.URI +import org.openrdf.model.Value +import org.openrdf.model.impl.BNodeImpl +import org.openrdf.model.impl.URIImpl +import org.openrdf.sail.Sail +import org.openrdf.sail.SailConnection +import com.tinkerpop.blueprints.Edge +import com.tinkerpop.blueprints.Vertex +import com.tinkerpop.blueprints.util.MultiIterable +import org.openrdf.sail.SailException + +/** + * Blueprints Graph to interact with Sail stores + * + * Date: 5/8/12 + * Time: 5:52 PM + */ +class RyaSailGraph extends SailGraph { + + public static final Resource[] EMPTY_CONTEXT = new Resource[0] + + RyaSailGraph(Sail sail) { + super(sail) + } + + /** + * For some reason, the SailGraph does not implement this method. + * The id is the full formatted id of the edge (rdf statement) + * + * @param id + * @return + */ + @Override + Edge getEdge(Object id) { + assert id != null + return RyaSailEdge.fromId(id, this) + } + + @Override + Iterable<Edge> getEdges() { + return getEdgesSequence(); + } + + protected RyaSailEdgeSequence getEdgesSequence() { + return new RyaSailEdgeSequence(((SailConnection) sailConnection.get()).getStatements(null, null, null, false, new Resource[0]), this) + } + + @Override + Iterable<Vertex> getVertices() { + return new RyaSailVertexSequence(this.getEdgesSequence()) + } + + /** + * Utility method that can take a string and make it a Resource, Uri, or Literal + * @param resource + * @return + */ + public Value createValue(String resource) { + if (SailHelper.isBNode(resource)) + new BNodeImpl(resource.substring(2)); + Literal literal; + if ((literal = SailHelper.makeLiteral(resource, this)) != null) + return literal + if (resource.contains(":") || resource.contains("/") || resource.contains("#")) { + resource = expandPrefix(resource); + new URIImpl(resource); + } else { + throw new RuntimeException((new StringBuilder()).append(resource).append(" is not a valid URI, blank node, or literal value").toString()); + } + } + + public Vertex createVertex(String resource) { + return new RyaSailVertex(createValue(resource), this); + } + + @Override + public Vertex addVertex(Object id) { + if (null == id) + id = SailTokens.URN_UUID_PREFIX + UUID.randomUUID().toString(); + return createVertex(id.toString()); + } + + @Override + public Vertex getVertex(final Object id) { + if (null == id) + throw new IllegalArgumentException("Element identifier cannot be null"); + + try { + return createVertex(id.toString()); + } catch (RuntimeException re) { + return null; + } + } + + public Iterable<Edge> query(final String subj, final String pred, final String obj, final String cntxt) { + return new RyaSailEdgeSequence(sailConnection.get().getStatements( + (subj != null) ? (Resource) createValue(subj) : null, + (pred != null) ? (URI) createValue(pred) : null, + (obj != null) ? createValue(obj) : null, + false, + (cntxt != null) ? (Resource) createValue(cntxt) : EMPTY_CONTEXT), + this); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/extras/tinkerpop.rya/src/main/groovy/mvm/rya/blueprints/sail/RyaSailVertex.groovy ---------------------------------------------------------------------- diff --git a/extras/tinkerpop.rya/src/main/groovy/mvm/rya/blueprints/sail/RyaSailVertex.groovy b/extras/tinkerpop.rya/src/main/groovy/mvm/rya/blueprints/sail/RyaSailVertex.groovy new file mode 100644 index 0000000..3f6ec64 --- /dev/null +++ b/extras/tinkerpop.rya/src/main/groovy/mvm/rya/blueprints/sail/RyaSailVertex.groovy @@ -0,0 +1,70 @@ +//package mvm.rya.blueprints.sail +// +//import com.tinkerpop.blueprints.pgm.impls.MultiIterable +//import com.tinkerpop.blueprints.pgm.impls.sail.SailVertex +//import org.openrdf.model.Resource +//import org.openrdf.model.Value +//import org.openrdf.model.impl.URIImpl +//import org.openrdf.sail.SailException +//import com.tinkerpop.blueprints.pgm.Edge +// +///** +// * Extension to SailVertex to use RyaSailEdgeSequence underneath +// * Date: 5/9/12 +// * Time: 3:40 PM +// */ +//class RyaSailVertex extends SailVertex { +// +// def sailGraph +// +// RyaSailVertex(Value rawVertex, RyaSailGraph graph) { +// super(rawVertex, graph) +// sailGraph = graph +// } +// +// @Override +// public Iterable<Edge> getOutEdges(final String... labels) { +// def vertex = getRawVertex() +// if (vertex instanceof Resource) { +// try { +// if (labels.length == 0) { +// return new RyaSailEdgeSequence(sailGraph.getSailConnection().get().getStatements((Resource) vertex, null, null, false), sailGraph); +// } else if (labels.length == 1) { +// return new RyaSailEdgeSequence(sailGraph.getSailConnection().get().getStatements((Resource) vertex, new URIImpl(sailGraph.expandPrefix(labels[0])), null, false), sailGraph); +// } else { +// final List<Iterable<Edge>> edges = new ArrayList<Iterable<Edge>>(); +// for (final String label: labels) { +// edges.add(new RyaSailEdgeSequence(sailGraph.getSailConnection().get().getStatements((Resource) vertex, new URIImpl(sailGraph.expandPrefix(label)), null, false), sailGraph)); +// } +// return new MultiIterable<Edge>(edges); +// } +// } catch (SailException e) { +// throw new RuntimeException(e.getMessage(), e); +// } +// } else { +// return new RyaSailEdgeSequence(); +// } +// +// } +// +// @Override +// public Iterable<Edge> getInEdges(final String... labels) { +// try { +// def vertex = getRawVertex() +// if (labels.length == 0) { +// return new RyaSailEdgeSequence(sailGraph.getSailConnection().get().getStatements(null, null, vertex, false), sailGraph); +// } else if (labels.length == 1) { +// return new RyaSailEdgeSequence(sailGraph.getSailConnection().get().getStatements(null, new URIImpl(sailGraph.expandPrefix(labels[0])), vertex, false), sailGraph); +// } else { +// final List<Iterable<Edge>> edges = new ArrayList<Iterable<Edge>>(); +// for (final String label: labels) { +// edges.add(new RyaSailEdgeSequence(sailGraph.getSailConnection().get().getStatements(null, new URIImpl(sailGraph.expandPrefix(label)), vertex, false), sailGraph)); +// } +// return new MultiIterable<Edge>(edges); +// } +// } catch (SailException e) { +// throw new RuntimeException(e.getMessage(), e); +// } +// +// } +//} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/extras/tinkerpop.rya/src/main/groovy/mvm/rya/blueprints/sail/RyaSailVertexSequence.groovy ---------------------------------------------------------------------- diff --git a/extras/tinkerpop.rya/src/main/groovy/mvm/rya/blueprints/sail/RyaSailVertexSequence.groovy b/extras/tinkerpop.rya/src/main/groovy/mvm/rya/blueprints/sail/RyaSailVertexSequence.groovy new file mode 100644 index 0000000..e0d7479 --- /dev/null +++ b/extras/tinkerpop.rya/src/main/groovy/mvm/rya/blueprints/sail/RyaSailVertexSequence.groovy @@ -0,0 +1,75 @@ +package mvm.rya.blueprints.sail + +import com.google.common.collect.Iterators +import com.google.common.collect.PeekingIterator +import com.tinkerpop.blueprints.Edge +import com.tinkerpop.blueprints.Vertex +import org.openrdf.model.Statement + +/** + * Iterable that provides a distinct list of subjects or objects from statements + * Date: 5/8/12 + * Time: 5:56 PM + */ +class RyaSailVertexSequence implements Iterable<Vertex>, Iterator<Vertex> { + enum VERTEXSIDE { + SUBJECT, OBJECT + } + def PeekingIterator<Edge> iter + def RyaSailGraph graph + def previous + def vertexSide = VERTEXSIDE.SUBJECT + + RyaSailVertexSequence() { + } + + RyaSailVertexSequence(RyaSailEdgeSequence iter) { + this(iter, VERTEXSIDE.SUBJECT) + } + + RyaSailVertexSequence(RyaSailEdgeSequence iter, VERTEXSIDE vertexSide) { + this.iter = Iterators.peekingIterator(iter) + this.graph = iter.graph + this.vertexSide = vertexSide + } + + @Override + Iterator<Vertex> iterator() { + return this + } + + @Override + boolean hasNext() { + if (iter == null) { + return false + } + while (iter.hasNext()) { + def peek = (RyaSailEdge) iter.peek() + def subject = getVertexSide(peek.getRawEdge()) + if (!(subject.equals(previous))) { + return true + } + iter.next() //keep iterating + } + return false; + } + + @Override + Vertex next() { + if (!this.hasNext()) + throw new NoSuchElementException(); + def next = (RyaSailEdge) iter.next() + Statement statement = next.getRawEdge() + previous = getVertexSide(statement) + return new RyaSailVertex(previous, graph); + } + + def getVertexSide(Statement statement) { + return (VERTEXSIDE.SUBJECT.equals(vertexSide)) ? statement.getSubject() : statement.getObject() + } + + @Override + void remove() { + throw new UnsupportedOperationException(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/extras/tinkerpop.rya/src/main/java/mvm/rya/blueprints/sail/RyaSailEdge.java ---------------------------------------------------------------------- diff --git a/extras/tinkerpop.rya/src/main/java/mvm/rya/blueprints/sail/RyaSailEdge.java b/extras/tinkerpop.rya/src/main/java/mvm/rya/blueprints/sail/RyaSailEdge.java new file mode 100644 index 0000000..4345283 --- /dev/null +++ b/extras/tinkerpop.rya/src/main/java/mvm/rya/blueprints/sail/RyaSailEdge.java @@ -0,0 +1,100 @@ +package mvm.rya.blueprints.sail; + +/* + * #%L + * mvm.rya.tinkerpop.rya + * %% + * Copyright (C) 2014 Rya + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import com.tinkerpop.blueprints.impls.sail.SailEdge; +import org.openrdf.model.Resource; +import org.openrdf.model.Statement; +import org.openrdf.model.URI; +import org.openrdf.model.Value; +import org.openrdf.model.impl.ContextStatementImpl; +import org.openrdf.model.impl.StatementImpl; + +/** + * Blueprints Edge for Sail stores + * outVertex edge inVertex + * <br/> + * Groovy is doing something funky with properties and such here + * <p/> + * Date: 5/9/12 + * Time: 9:03 AM + */ +public class RyaSailEdge extends SailEdge { + + public static final String SPLIT = "|"; + + public RyaSailEdge(Statement rawEdge, RyaSailGraph graph) { + super(rawEdge, graph); + } + + @Override + public Object getId() { + Statement statement = this.getRawEdge(); + return formatId(statement); + } + + /** + * Returns a formatted id for a full statement. + * + * @param statement + * @return + */ + public static String formatId(Statement statement) { + if (null != statement.getContext()) + return (new StringBuilder()).append(statement.getSubject()).append(SPLIT).append(statement.getPredicate()).append(SPLIT).append(statement.getObject()).append(SPLIT).append(statement.getContext()).toString(); + else + return (new StringBuilder()).append(statement.getSubject()).append(SPLIT).append(statement.getPredicate()).append(SPLIT).append(statement.getObject()).toString(); + } + +// public static RyaSailEdge fromId(String id, RyaSailGraph graph) { +// def decodedId = URLDecoder.decode(id) +// def statement = RdfIO.readStatement(ByteStreams.newDataInput(decodedId.bytes), RdfCloudTripleStoreConstants.VALUE_FACTORY) +// println statement +// return new RyaSailEdge(statement, graph) +// } + + /** + * @param id formatted from getId method + * @param graph + * @return + */ + public static RyaSailEdge fromId(String id, RyaSailGraph graph) { + assert id != null; + String[] split = id.split("\\|"); + if (split.length < 3) { + return null; + } + String subj_s = split[0].trim(); + Value subj = graph.createValue(subj_s); + String pred_s = split[1].trim(); + Value pred = graph.createValue(pred_s); + String obj_s = split[2].trim(); + Value obj = graph.createValue(obj_s); + if (split.length == 4) { + //context available + Value context = graph.createValue(split[3]); + return new RyaSailEdge(new ContextStatementImpl((Resource) subj, (URI) pred, obj, (Resource) context), graph); + } else { + return new RyaSailEdge(new StatementImpl((Resource) subj, (URI) pred, obj), graph); + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/extras/tinkerpop.rya/src/main/java/mvm/rya/blueprints/sail/RyaSailVertex.java ---------------------------------------------------------------------- diff --git a/extras/tinkerpop.rya/src/main/java/mvm/rya/blueprints/sail/RyaSailVertex.java b/extras/tinkerpop.rya/src/main/java/mvm/rya/blueprints/sail/RyaSailVertex.java new file mode 100644 index 0000000..c02a376 --- /dev/null +++ b/extras/tinkerpop.rya/src/main/java/mvm/rya/blueprints/sail/RyaSailVertex.java @@ -0,0 +1,104 @@ +package mvm.rya.blueprints.sail; + +/* + * #%L + * mvm.rya.tinkerpop.rya + * %% + * Copyright (C) 2014 Rya + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import com.tinkerpop.blueprints.Direction; +import com.tinkerpop.blueprints.Edge; +import com.tinkerpop.blueprints.util.MultiIterable; +import com.tinkerpop.blueprints.impls.sail.SailGraph; +import com.tinkerpop.blueprints.impls.sail.SailVertex; +import org.openrdf.model.Resource; +import org.openrdf.model.Value; +import org.openrdf.model.impl.URIImpl; +import org.openrdf.sail.SailException; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * For some reason, the groovy class overwrites the metaclass and Gremlin.load will not change it for properties like outE, both, etc + * Date: 5/10/12 + * Time: 12:35 PM + */ +public class RyaSailVertex extends SailVertex { + + public RyaSailVertex(Value rawVertex, SailGraph graph) { + super(rawVertex, graph); + } + + + @Override + public Iterable<Edge> getEdges(Direction direction, final String... labels) { + if (direction.equals(Direction.OUT)) + return getOutEdges(labels); + if (direction.equals(Direction.IN)) { + return getInEdges(labels); + } + return new MultiIterable(Arrays.asList(new Iterable[] { getInEdges(labels), getOutEdges(labels) })); + } + + private Iterable<Edge> getOutEdges(final String... labels) { + Value vertex = getRawVertex(); + if (vertex instanceof Resource) { + try { + if (labels.length == 0) { + return new RyaSailEdgeSequence(getRyaSailGraph().getSailConnection().get().getStatements((Resource) vertex, null, null, false), getRyaSailGraph()); + } else if (labels.length == 1) { + return new RyaSailEdgeSequence(getRyaSailGraph().getSailConnection().get().getStatements((Resource) vertex, new URIImpl(getRyaSailGraph().expandPrefix(labels[0])), null, false), getRyaSailGraph()); + } else { + final List<Iterable<Edge>> edges = new ArrayList<Iterable<Edge>>(); + for (final String label: labels) { + edges.add(new RyaSailEdgeSequence(getRyaSailGraph().getSailConnection().get().getStatements((Resource) vertex, new URIImpl(getRyaSailGraph().expandPrefix(label)), null, false), getRyaSailGraph())); + } + return new MultiIterable<Edge>(edges); + } + } catch (SailException e) { + throw new RuntimeException(e.getMessage(), e); + } + } else { + return new RyaSailEdgeSequence(); + } + } + + private Iterable<Edge> getInEdges(final String... labels) { + try { + Value vertex = getRawVertex(); + if (labels.length == 0) { + return new RyaSailEdgeSequence(getRyaSailGraph().getSailConnection().get().getStatements(null, null, vertex, false), getRyaSailGraph()); + } else if (labels.length == 1) { + return new RyaSailEdgeSequence(getRyaSailGraph().getSailConnection().get().getStatements(null, new URIImpl(getRyaSailGraph().expandPrefix(labels[0])), vertex, false), getRyaSailGraph()); + } else { + final List<Iterable<Edge>> edges = new ArrayList<Iterable<Edge>>(); + for (final String label: labels) { + edges.add(new RyaSailEdgeSequence(getRyaSailGraph().getSailConnection().get().getStatements(null, new URIImpl(getRyaSailGraph().expandPrefix(label)), vertex, false),getRyaSailGraph())); + } + return new MultiIterable<Edge>(edges); + } + } catch (SailException e) { + throw new RuntimeException(e.getMessage(), e); + } + } + + public RyaSailGraph getRyaSailGraph() { + return (RyaSailGraph) graph; + } +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/extras/tinkerpop.rya/src/test/groovy/mvm/rya/blueprints/TstGremlinRya.groovy ---------------------------------------------------------------------- diff --git a/extras/tinkerpop.rya/src/test/groovy/mvm/rya/blueprints/TstGremlinRya.groovy b/extras/tinkerpop.rya/src/test/groovy/mvm/rya/blueprints/TstGremlinRya.groovy new file mode 100644 index 0000000..2914ac1 --- /dev/null +++ b/extras/tinkerpop.rya/src/test/groovy/mvm/rya/blueprints/TstGremlinRya.groovy @@ -0,0 +1,111 @@ +//package mvm.rya.blueprints +// +//import com.tinkerpop.blueprints.pgm.impls.sail.SailGraph +//import com.tinkerpop.blueprints.pgm.impls.sail.SailVertex +//import com.tinkerpop.gremlin.groovy.Gremlin +//import mvm.rya.accumulo.AccumuloRdfConfiguration +//import mvm.rya.accumulo.AccumuloRdfDAO +//import mvm.rya.accumulo.AccumuloRdfEvalStatsDAO +// +//import mvm.rya.blueprints.config.RyaGraphConfiguration +//import mvm.rya.rdftriplestore.RdfCloudTripleStore +//import mvm.rya.rdftriplestore.inference.InferenceEngine +//import org.apache.accumulo.core.client.ZooKeeperInstance +//import static mvm.rya.accumulo.mr.utils.MRUtils.* +//import static mvm.rya.api.RdfCloudTripleStoreConfiguration.CONF_QUERYPLAN_FLAG +//import static mvm.rya.api.RdfCloudTripleStoreConfiguration.CONF_TBL_PREFIX +// +///** +// * Date: 5/7/12 +// * Time: 5:39 PM +// */ +//class TstGremlinRya { +// public static void main(String[] args) { +// +// def conf = new AccumuloRdfConfiguration(); + +/* + * #%L + * mvm.rya.tinkerpop.rya + * %% + * Copyright (C) 2014 Rya + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ +// conf.setDisplayQueryPlan(true); +// def store = new RdfCloudTripleStore(); +// store.setConf(conf); +// def crdfdao = new AccumuloRdfDAO(); +// def connector = new ZooKeeperInstance("acu13", "stratus25:2181").getConnector("root", "secret"); +// crdfdao.setConnector(connector); +// conf.setTablePrefix("l_"); +// crdfdao.setConf(conf); +// store.setRdfDao(crdfdao); +// def ceval = new AccumuloRdfEvalStatsDAO(); +// ceval.setConnector(connector); +// ceval.setConf(conf); +// store.setRdfEvalStatsDAO(ceval); +// def inferenceEngine = new InferenceEngine(); +// inferenceEngine.setRdfDao(crdfdao); +// inferenceEngine.setConf(conf); +// store.setInferenceEngine(inferenceEngine); +// store.setConf(conf); +// +// Gremlin.load() +// def g = new SailGraph(store) +//// def g = RyaGraphConfiguration.createGraph([(AC_INSTANCE_PROP): "acu13", (AC_ZK_PROP): "stratus25:2181",(AC_USERNAME_PROP): "root", (AC_PWD_PROP): "secret", (CONF_TBL_PREFIX): "l_", (CONF_QUERYPLAN_FLAG): "true"]); +// +// def v = g.getVertex('http://www.Department0.University0.edu/GraduateCourse0'); +// def v2 = g.getVertex('http://www.Department0.University0.edu/GraduateCourse1'); +//// v.getInEdges().each { +//// println it +//// } +//// v.getInEdges('urn:lubm:rdfts#takesCourse').each { +//// println it +//// } +//// def gc0 = g.getVertex('http://www.Department0.University0.edu/GraduateCourse0') +//// gc0.getOutEdges().each { +//// println it.getInVertex() +//// } +//// +//// def gc0_lit = g.getVertex('\"GraduateCourse0\"') +//// println gc0_lit +// +// v = g.getVertex('http://dbpedia.org/resource/Albert_Camus') +// println v.outE.each { +// println it.label +// } +// +// g.shutdown() +// +// g = RyaGraphConfiguration.createGraph([(AC_INSTANCE_PROP): "acu13", (AC_ZK_PROP): "stratus25:2181",(AC_USERNAME_PROP): "root", (AC_PWD_PROP): "secret", (CONF_TBL_PREFIX): "l_", (CONF_QUERYPLAN_FLAG): "true"]); +// +// def rv = g.getVertex('http://dbpedia.org/resource/Albert_Camus') +// println rv.outE.each { +// println it.label +// } +// +// v = new SailVertex(rv.getRawVertex(), rv.sailGraph) +// println v.outE +// +// g.shutdown() +//// +//// def name = { +//// println it.name +//// } +//// println SailVertex.metaClass.properties.each(name) +//// println RyaSailVertex.metaClass.properties.each(name) +//// println RyaSailVertex.metaClass.properties.each(name) +// } +//} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/extras/tinkerpop.rya/src/test/groovy/mvm/rya/blueprints/config/RyaGraphConfigurationTest.groovy ---------------------------------------------------------------------- diff --git a/extras/tinkerpop.rya/src/test/groovy/mvm/rya/blueprints/config/RyaGraphConfigurationTest.groovy b/extras/tinkerpop.rya/src/test/groovy/mvm/rya/blueprints/config/RyaGraphConfigurationTest.groovy new file mode 100644 index 0000000..1de38f6 --- /dev/null +++ b/extras/tinkerpop.rya/src/test/groovy/mvm/rya/blueprints/config/RyaGraphConfigurationTest.groovy @@ -0,0 +1,129 @@ +package mvm.rya.blueprints.config + +import com.tinkerpop.blueprints.Vertex +import com.tinkerpop.blueprints.Direction +import junit.framework.TestCase +import mvm.rya.api.RdfCloudTripleStoreConstants +import mvm.rya.api.resolver.RdfToRyaConversions +import mvm.rya.blueprints.sail.RyaSailEdge +import org.openrdf.model.ValueFactory +import org.openrdf.model.impl.StatementImpl +import org.openrdf.model.impl.ValueFactoryImpl + +import static mvm.rya.api.RdfCloudTripleStoreConfiguration.CONF_TBL_PREFIX +import static mvm.rya.accumulo.mr.utils.MRUtils.* +import org.apache.accumulo.core.security.Authorizations +import org.apache.accumulo.core.client.Connector +import mvm.rya.accumulo.AccumuloRyaDAO +import mvm.rya.accumulo.AccumuloRdfConfiguration +import org.apache.accumulo.core.client.mock.MockInstance +import org.apache.accumulo.core.client.admin.SecurityOperations +import org.apache.accumulo.core.Constants +import org.apache.accumulo.core.security.TablePermission + +/** + * Date: 5/9/12 + * Time: 3:11 PM + */ +class RyaGraphConfigurationTest extends TestCase { + private String user = "user"; + +/* + * #%L + * mvm.rya.tinkerpop.rya + * %% + * Copyright (C) 2014 Rya + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + private String pwd = "pwd"; + private String instance = "myinstance"; + private String tablePrefix = "t_"; + private Authorizations auths = Constants.NO_AUTHS; + private Connector connector; + private AccumuloRyaDAO ryaDAO; + private ValueFactory vf = new ValueFactoryImpl(); + private String namespace = "urn:test#"; + private AccumuloRdfConfiguration conf; + + @Override + public void setUp() throws Exception { + super.setUp(); + connector = new MockInstance(instance).getConnector(user, pwd.getBytes()); + connector.tableOperations().create(tablePrefix + RdfCloudTripleStoreConstants.TBL_SPO_SUFFIX); + connector.tableOperations().create(tablePrefix + RdfCloudTripleStoreConstants.TBL_PO_SUFFIX); + connector.tableOperations().create(tablePrefix + RdfCloudTripleStoreConstants.TBL_OSP_SUFFIX); + connector.tableOperations().create(tablePrefix + RdfCloudTripleStoreConstants.TBL_NS_SUFFIX); + SecurityOperations secOps = connector.securityOperations(); + secOps.createUser(user, pwd.getBytes(), auths); + secOps.grantTablePermission(user, tablePrefix + RdfCloudTripleStoreConstants.TBL_SPO_SUFFIX, TablePermission.READ); + secOps.grantTablePermission(user, tablePrefix + RdfCloudTripleStoreConstants.TBL_PO_SUFFIX, TablePermission.READ); + secOps.grantTablePermission(user, tablePrefix + RdfCloudTripleStoreConstants.TBL_OSP_SUFFIX, TablePermission.READ); + secOps.grantTablePermission(user, tablePrefix + RdfCloudTripleStoreConstants.TBL_NS_SUFFIX, TablePermission.READ); + + conf = new AccumuloRdfConfiguration(); + ryaDAO = new AccumuloRyaDAO(); + ryaDAO.setConnector(connector); + conf.setTablePrefix(tablePrefix); + ryaDAO.setConf(conf); + ryaDAO.init(); + } + + @Override + public void tearDown() throws Exception { + super.tearDown(); + connector.tableOperations().delete(tablePrefix + RdfCloudTripleStoreConstants.TBL_SPO_SUFFIX); + connector.tableOperations().delete(tablePrefix + RdfCloudTripleStoreConstants.TBL_PO_SUFFIX); + connector.tableOperations().delete(tablePrefix + RdfCloudTripleStoreConstants.TBL_OSP_SUFFIX); + connector.tableOperations().delete(tablePrefix + RdfCloudTripleStoreConstants.TBL_NS_SUFFIX); + } + + public void testGraphConfiguration() { + def a = vf.createURI(namespace, "a") + def statement = new StatementImpl(a, vf.createURI(namespace, "p"), vf.createLiteral("l")) + def statement2 = new StatementImpl(a, vf.createURI(namespace, "p2"), vf.createLiteral("l")) + ryaDAO.add(RdfToRyaConversions.convertStatement(statement)); + ryaDAO.add(RdfToRyaConversions.convertStatement(statement2)); + ryaDAO.add(RdfToRyaConversions.convertStatement(new StatementImpl(vf.createURI(namespace, "b"), vf.createURI(namespace, "p"), vf.createLiteral("l")))); + ryaDAO.add(RdfToRyaConversions.convertStatement(new StatementImpl(vf.createURI(namespace, "c"), vf.createURI(namespace, "n"), vf.createLiteral("l")))); + + RyaGraphConfiguration.load() + + def graph = RyaGraphConfiguration.createGraph( + [(AC_INSTANCE_PROP): instance, + (AC_MOCK_PROP): "true", + (AC_USERNAME_PROP): user, + (AC_PWD_PROP): pwd, + (CONF_TBL_PREFIX): tablePrefix, +// (CONF_QUERYPLAN_FLAG): "true", + ] + ); + + def edge = graph.getEdge(RyaSailEdge.formatId(statement)) + assertNotNull(edge) + Vertex vertex = graph.getVertex(a.stringValue()) + assertNotNull(vertex) + def edges = vertex.getEdges(Direction.OUT).iterator().toList() + assertEquals(2, edges.size()) + assertNotNull edges[0].subj + assertNotNull edges[0].pred + assertNotNull edges[0].obj + assertNull edges[0].cntxt + + def queryEdges = graph.query(edges[0].subj, edges[0].pred, edges[0].obj, edges[0].cntxt) + assertEquals edges[0], queryEdges[0] + + graph.shutdown() + } +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/92ddfa59/extras/tinkerpop.rya/src/test/groovy/mvm/rya/blueprints/sail/RyaSailVertexSequenceTest.groovy ---------------------------------------------------------------------- diff --git a/extras/tinkerpop.rya/src/test/groovy/mvm/rya/blueprints/sail/RyaSailVertexSequenceTest.groovy b/extras/tinkerpop.rya/src/test/groovy/mvm/rya/blueprints/sail/RyaSailVertexSequenceTest.groovy new file mode 100644 index 0000000..9196eb7 --- /dev/null +++ b/extras/tinkerpop.rya/src/test/groovy/mvm/rya/blueprints/sail/RyaSailVertexSequenceTest.groovy @@ -0,0 +1,100 @@ +package mvm.rya.blueprints.sail + +import mvm.rya.api.utils.IteratorWrapper +import junit.framework.TestCase +import mvm.rya.blueprints.config.RyaGraphConfiguration +import org.openrdf.model.Statement +import static mvm.rya.accumulo.mr.utils.MRUtils.* +import static mvm.rya.api.RdfCloudTripleStoreConstants.VALUE_FACTORY + +/** + * Date: 5/10/12 + * Time: 8:55 AM + */ +class RyaSailVertexSequenceTest extends TestCase { + + public void testDistinctSubjects() { + def namespace = "urn:test#" + def vf = VALUE_FACTORY + def graph = RyaGraphConfiguration.createGraph( + [(AC_INSTANCE_PROP): "inst", + (AC_MOCK_PROP): "true", + (AC_USERNAME_PROP): "user", + (AC_PWD_PROP): "pwd", + ] + ); + +/* + * #%L + * mvm.rya.tinkerpop.rya + * %% + * Copyright (C) 2014 Rya + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + def a = vf.createURI(namespace, "a") + def b = vf.createURI(namespace, "b") + def c = vf.createURI(namespace, "c") + def statements = [ + vf.createStatement(a, vf.createURI(namespace, "p"), vf.createURI(namespace, "l1")), + vf.createStatement(a, vf.createURI(namespace, "p"), vf.createURI(namespace, "l2")), + vf.createStatement(a, vf.createURI(namespace, "p"), vf.createURI(namespace, "l3")), + vf.createStatement(b, vf.createURI(namespace, "p"), vf.createURI(namespace, "l1")), + vf.createStatement(c, vf.createURI(namespace, "p"), vf.createURI(namespace, "l1")), + vf.createStatement(c, vf.createURI(namespace, "p"), vf.createURI(namespace, "l2")), + vf.createStatement(c, vf.createURI(namespace, "p"), vf.createURI(namespace, "l3")), + ] + def edgeSeq = new RyaSailEdgeSequence(new IteratorWrapper<Statement>(statements.iterator()), graph) + def vertexSeq = new RyaSailVertexSequence(edgeSeq) + def expectedList = [a, b, c] + def list = vertexSeq.toList().collect { v -> + v.getRawVertex() + } + assertEquals(expectedList, list) + } + + public void testDistinctObjects() { + def namespace = "urn:test#" + def vf = VALUE_FACTORY + def graph = RyaGraphConfiguration.createGraph( + [(AC_INSTANCE_PROP): "inst", + (AC_MOCK_PROP): "true", + (AC_USERNAME_PROP): "user", + (AC_PWD_PROP): "pwd", + ] + ); + def a = vf.createURI(namespace, "a") + def b = vf.createURI(namespace, "b") + def c = vf.createURI(namespace, "c") + def l1 = vf.createURI(namespace, "l1") + def l2 = vf.createURI(namespace, "l2") + def l3 = vf.createURI(namespace, "l3") + def statements = [ + vf.createStatement(a, vf.createURI(namespace, "p"), l1), + vf.createStatement(b, vf.createURI(namespace, "p"), l1), + vf.createStatement(c, vf.createURI(namespace, "p"), l1), + vf.createStatement(a, vf.createURI(namespace, "p"), l2), + vf.createStatement(c, vf.createURI(namespace, "p"), l2), + vf.createStatement(a, vf.createURI(namespace, "p"), l3), + vf.createStatement(c, vf.createURI(namespace, "p"), l3), + ] + def edgeSeq = new RyaSailEdgeSequence(new IteratorWrapper<Statement>(statements.iterator()), graph) + def vertexSeq = new RyaSailVertexSequence(edgeSeq, RyaSailVertexSequence.VERTEXSIDE.OBJECT) + def expectedList = [l1, l2, l3] + def list = vertexSeq.toList().collect { v -> + v.getRawVertex() + } + assertEquals(expectedList, list) + } +}
