http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/5a03ef61/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/JoinSelectSpoTableOutput.java ---------------------------------------------------------------------- diff --git a/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/JoinSelectSpoTableOutput.java b/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/JoinSelectSpoTableOutput.java deleted file mode 100644 index f7b1672..0000000 --- a/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/JoinSelectSpoTableOutput.java +++ /dev/null @@ -1,126 +0,0 @@ -package mvm.rya.joinselect.mr; - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - - -import static mvm.rya.joinselect.mr.utils.JoinSelectConstants.AUTHS; -import static mvm.rya.joinselect.mr.utils.JoinSelectConstants.SPO_OUTPUTPATH; -import static mvm.rya.joinselect.mr.utils.JoinSelectConstants.SPO_TABLE; - -import java.io.IOException; - -import mvm.rya.accumulo.AccumuloRdfConfiguration; -import mvm.rya.api.RdfCloudTripleStoreConstants; -import mvm.rya.api.domain.RyaStatement; -import mvm.rya.api.resolver.RyaTripleContext; -import mvm.rya.api.resolver.triple.TripleRow; -import mvm.rya.api.resolver.triple.TripleRowResolverException; -import mvm.rya.joinselect.mr.utils.CompositeType; -import mvm.rya.joinselect.mr.utils.JoinSelectStatsUtil; -import mvm.rya.joinselect.mr.utils.TripleCard; -import mvm.rya.joinselect.mr.utils.TripleEntry; - -import org.apache.accumulo.core.data.Key; -import org.apache.accumulo.core.data.Value; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.conf.Configured; -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.MRJobConfig; -import org.apache.hadoop.mapreduce.Mapper; -import org.apache.hadoop.util.Tool; - -public class JoinSelectSpoTableOutput extends Configured implements Tool { - - public static class JoinSelectMapper extends Mapper<Key,Value,CompositeType,TripleCard> { - - private RyaTripleContext ryaContext; - private static final String DELIM = "\u0000"; - - public void map(Key row, Value data, Context context) throws IOException, InterruptedException { - try { - ryaContext = RyaTripleContext.getInstance(new AccumuloRdfConfiguration(context.getConfiguration())); - RyaStatement ryaStatement = ryaContext.deserializeTriple(RdfCloudTripleStoreConstants.TABLE_LAYOUT.SPO, new TripleRow(row.getRow().getBytes(), row - .getColumnFamily().getBytes(), row.getColumnQualifier().getBytes(), row.getTimestamp(), row.getColumnVisibility().getBytes(), data.get())); - - Text s = new Text(ryaStatement.getSubject().getData()); - Text p = new Text(ryaStatement.getPredicate().getData()); - Text o = new Text(ryaStatement.getObject().getData()); - Text sp = new Text(ryaStatement.getSubject().getData() + DELIM + ryaStatement.getPredicate().getData()); - Text po = new Text(ryaStatement.getPredicate().getData() + DELIM + ryaStatement.getObject().getData()); - Text so = new Text(ryaStatement.getSubject().getData() + DELIM + ryaStatement.getObject().getData()); - Text ps = new Text(ryaStatement.getPredicate().getData() + DELIM + ryaStatement.getSubject().getData()); - Text op = new Text(ryaStatement.getObject().getData() + DELIM + ryaStatement.getPredicate().getData()); - Text os = new Text(ryaStatement.getObject().getData() + DELIM + ryaStatement.getSubject().getData()); - - TripleEntry t1 = new TripleEntry(s, p, new Text("subject"), new Text("predicate"), new Text("object")); - TripleEntry t2 = new TripleEntry(p, o, new Text("predicate"), new Text("object"), new Text("subject")); - TripleEntry t3 = new TripleEntry(o, s, new Text("object"), new Text("subject"), new Text("predicate")); - TripleEntry t4 = new TripleEntry(s, new Text(""), new Text("subject"), new Text(""), new Text("predicateobject")); - TripleEntry t5 = new TripleEntry(p, new Text(""), new Text("predicate"), new Text(""), new Text("objectsubject")); - TripleEntry t6 = new TripleEntry(o, new Text(""), new Text("object"), new Text(""), new Text("subjectpredicate")); - TripleEntry t7 = new TripleEntry(s, new Text(""), new Text("subject"), new Text(""), new Text("objectpredicate")); - TripleEntry t8 = new TripleEntry(p, new Text(""), new Text("predicate"), new Text(""), new Text("subjectobject")); - TripleEntry t9 = new TripleEntry(o, new Text(""), new Text("object"), new Text(""), new Text("predicatesubject")); - - context.write(new CompositeType(o, new IntWritable(2)), new TripleCard(t1)); - context.write(new CompositeType(s, new IntWritable(2)), new TripleCard(t2)); - context.write(new CompositeType(p, new IntWritable(2)), new TripleCard(t3)); - context.write(new CompositeType(po, new IntWritable(2)), new TripleCard(t4)); - context.write(new CompositeType(so, new IntWritable(2)), new TripleCard(t5)); - context.write(new CompositeType(sp, new IntWritable(2)), new TripleCard(t6)); - context.write(new CompositeType(op, new IntWritable(2)), new TripleCard(t7)); - context.write(new CompositeType(os, new IntWritable(2)), new TripleCard(t8)); - context.write(new CompositeType(ps, new IntWritable(2)), new TripleCard(t9)); - - } catch (TripleRowResolverException e) { - e.printStackTrace(); - } - - } - - } - - @Override - public int run(String[] args) throws Exception { - - Configuration conf = getConf(); - String inTable = conf.get(SPO_TABLE); - String auths = conf.get(AUTHS); - String outPath = conf.get(SPO_OUTPUTPATH); - - assert inTable != null && outPath != null; - - Job job = new Job(conf, this.getClass().getSimpleName() + "_" + System.currentTimeMillis()); - job.setJarByClass(this.getClass()); - conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, true); - - JoinSelectStatsUtil.initTabToSeqFileJob(job, inTable, outPath, auths); - job.setMapperClass(JoinSelectMapper.class); - job.setNumReduceTasks(0); - job.waitForCompletion(true); - - return job.isSuccessful() ? 0 : 1; - - } - -}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/5a03ef61/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/JoinSelectStatisticsSum.java ---------------------------------------------------------------------- diff --git a/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/JoinSelectStatisticsSum.java b/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/JoinSelectStatisticsSum.java deleted file mode 100644 index ef271ff..0000000 --- a/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/JoinSelectStatisticsSum.java +++ /dev/null @@ -1,220 +0,0 @@ -package mvm.rya.joinselect.mr; - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - - -import static mvm.rya.joinselect.mr.utils.JoinSelectConstants.AUTHS; -import static mvm.rya.joinselect.mr.utils.JoinSelectConstants.INPUTPATH; -import static mvm.rya.joinselect.mr.utils.JoinSelectConstants.SELECTIVITY_TABLE; - -import java.io.IOException; - -import mvm.rya.joinselect.mr.utils.CardList; -import mvm.rya.joinselect.mr.utils.JoinSelectStatsUtil; -import mvm.rya.joinselect.mr.utils.TripleEntry; - -import org.apache.accumulo.core.client.AccumuloSecurityException; -import org.apache.accumulo.core.data.Mutation; -import org.apache.accumulo.core.data.Value; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.conf.Configured; -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.Mapper; -import org.apache.hadoop.mapreduce.Reducer; -import org.apache.hadoop.util.Tool; - -public class JoinSelectStatisticsSum extends Configured implements Tool { - - // TODO need to tweak this class to compute join cardinalities over more than one variable - - public static class CardinalityIdentityMapper extends Mapper<TripleEntry,CardList,TripleEntry,CardList> { - - public void map(TripleEntry key, CardList value, Context context) throws IOException, InterruptedException { - - // System.out.println("Keys are " + key + " and values are " + value); - - if (key.getSecond().toString().length() != 0 && key.getSecondPos().toString().length() != 0) { - TripleEntry te1 = new TripleEntry(key.getFirst(), new Text(""), key.getFirstPos(), new Text(""), key.getKeyPos()); - TripleEntry te2 = new TripleEntry(key.getSecond(), new Text(""), key.getSecondPos(), new Text(""), key.getKeyPos()); - - context.write(te1, value); - context.write(te2, value); - context.write(key, value); - // System.out.println("Output key values from mapper are " + te1 + " and " + value + "\n" - // + te2 + " and " + value + "\n" + key + " and " + value + "\n"); - } else if (key.getSecond().toString().length() == 0 && key.getSecondPos().toString().length() == 0) { - - context.write(key, value); - // System.out.println("Output key values from mapper are " + "\n" + key + " and " + value + "\n" + "\n"); - } - - } - - } - - public static class CardinalityIdentityReducer extends Reducer<TripleEntry,CardList,Text,Mutation> { - - private static final String DELIM = "\u0000"; - - public void reduce(TripleEntry te, Iterable<CardList> values, Context context) throws IOException, InterruptedException { - - CardList cl = new CardList(); - LongWritable s = new LongWritable(0); - LongWritable p = new LongWritable(0); - LongWritable o = new LongWritable(0); - LongWritable sp = new LongWritable(0); - LongWritable po = new LongWritable(0); - LongWritable so = new LongWritable(0); - - // System.out.println("***********************************************************\n" - // + "key is " + te); - - for (CardList val : values) { - // System.out.println("Value is " + val); - s.set(s.get() + val.getcardS().get()); - p.set(p.get() + val.getcardP().get()); - o.set(o.get() + val.getcardO().get()); - sp.set(sp.get() + val.getcardSP().get()); - po.set(po.get() + val.getcardPO().get()); - so.set(so.get() + val.getcardSO().get()); - } - cl.setCard(s, p, o, sp, po, so); - - Text row; - - if (te.getSecond().toString().length() > 0) { - row = new Text(te.getFirstPos().toString() + te.getSecondPos().toString() + DELIM + te.getFirst().toString() + DELIM + te.getSecond()); - } else { - row = new Text(te.getFirstPos().toString() + DELIM + te.getFirst().toString()); - } - - Mutation m1, m2, m3; - - if (te.getKeyPos().toString().equals("subject") || te.getKeyPos().toString().equals("predicate") || te.getKeyPos().toString().equals("object")) { - m1 = new Mutation(row); - m1.put(new Text(te.getKeyPos().toString() + "subject"), new Text(cl.getcardS().toString()), new Value(new byte[0])); - m2 = new Mutation(row); - m2.put(new Text(te.getKeyPos().toString() + "predicate"), new Text(cl.getcardP().toString()), new Value(new byte[0])); - m3 = new Mutation(row); - m3.put(new Text(te.getKeyPos().toString() + "object"), new Text(cl.getcardO().toString()), new Value(new byte[0])); - - } else if (te.getKeyPos().toString().equals("predicatesubject") || te.getKeyPos().toString().equals("objectpredicate") - || te.getKeyPos().toString().equals("subjectobject")) { - - String jOrder = reverseJoinOrder(te.getKeyPos().toString()); - - m1 = new Mutation(row); - m1.put(new Text(jOrder + "predicatesubject"), new Text(cl.getcardSP().toString()), new Value(new byte[0])); - m2 = new Mutation(row); - m2.put(new Text(jOrder + "objectpredicate"), new Text(cl.getcardPO().toString()), new Value(new byte[0])); - m3 = new Mutation(row); - m3.put(new Text(jOrder + "subjectobject"), new Text(cl.getcardSO().toString()), new Value(new byte[0])); - - } else { - - m1 = new Mutation(row); - m1.put(new Text(te.getKeyPos().toString() + "subjectpredicate"), new Text(cl.getcardSP().toString()), new Value(new byte[0])); - m2 = new Mutation(row); - m2.put(new Text(te.getKeyPos().toString() + "predicateobject"), new Text(cl.getcardPO().toString()), new Value(new byte[0])); - m3 = new Mutation(row); - m3.put(new Text(te.getKeyPos().toString() + "objectsubject"), new Text(cl.getcardSO().toString()), new Value(new byte[0])); - - } - - // TODO add the appropriate table name here - context.write(new Text(""), m1); - context.write(new Text(""), m2); - context.write(new Text(""), m3); - } - - private String reverseJoinOrder(String s) { - - if (s.equals("predicatesubject")) { - return "subjectpredicate"; - } else if (s.equals("objectpredicate")) { - return "predicateobject"; - } else if (s.equals("subjectobject")) { - return "objectsubject"; - } else { - throw new IllegalArgumentException("Invalid join type."); - } - - } - - } - - public static class CardinalityIdentityCombiner extends Reducer<TripleEntry,CardList,TripleEntry,CardList> { - - @Override - public void reduce(TripleEntry key, Iterable<CardList> values, Context context) throws IOException, InterruptedException { - - CardList cl = new CardList(); - LongWritable s = new LongWritable(0); - LongWritable p = new LongWritable(0); - LongWritable o = new LongWritable(0); - LongWritable sp = new LongWritable(0); - LongWritable po = new LongWritable(0); - LongWritable so = new LongWritable(0); - - for (CardList val : values) { - s.set(s.get() + val.getcardS().get()); - p.set(p.get() + val.getcardP().get()); - o.set(o.get() + val.getcardO().get()); - sp.set(sp.get() + val.getcardSP().get()); - po.set(po.get() + val.getcardPO().get()); - so.set(so.get() + val.getcardSO().get()); - } - - cl.setCard(s, p, o, sp, po, so); - context.write(key, cl); - - } - - } - - @Override - public int run(String[] args) throws AccumuloSecurityException, IOException, ClassNotFoundException, InterruptedException { - - Configuration conf = getConf(); - String outTable = conf.get(SELECTIVITY_TABLE); - String auths = conf.get(AUTHS); - String inPath = conf.get(INPUTPATH); - - assert outTable != null && inPath != null; - - Job job = new Job(getConf(), this.getClass().getSimpleName() + "_" + System.currentTimeMillis()); - job.setJarByClass(this.getClass()); - JoinSelectStatsUtil.initSumMRJob(job, inPath, outTable, auths); - - job.setMapperClass(CardinalityIdentityMapper.class); - job.setCombinerClass(CardinalityIdentityCombiner.class); - job.setReducerClass(CardinalityIdentityReducer.class); - job.setNumReduceTasks(32); - - job.waitForCompletion(true); - - return job.isSuccessful() ? 0 : 1; - - } - -} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/5a03ef61/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/CardList.java ---------------------------------------------------------------------- diff --git a/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/CardList.java b/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/CardList.java deleted file mode 100644 index b8fd274..0000000 --- a/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/CardList.java +++ /dev/null @@ -1,209 +0,0 @@ -package mvm.rya.joinselect.mr.utils; - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.WritableComparable; - -public class CardList implements WritableComparable<CardList> { - - private LongWritable cardS; - private LongWritable cardP; - private LongWritable cardO; - private LongWritable cardSP; - private LongWritable cardPO; - private LongWritable cardSO; - - public CardList() { - cardS = new LongWritable(); - cardP = new LongWritable(); - cardO = new LongWritable(); - cardSP = new LongWritable(); - cardSO = new LongWritable(); - cardPO = new LongWritable(); - - } - - public CardList(long cardS, long cardP, long cardO, long cardSP, long cardPO, long cardSO) { - this.cardS = new LongWritable(cardS); - this.cardP = new LongWritable(cardP); - this.cardO = new LongWritable(cardO); - this.cardSP = new LongWritable(cardSP); - this.cardSO = new LongWritable(cardSO); - this.cardPO = new LongWritable(cardPO); - } - - public CardList(LongWritable cardS, LongWritable cardP, LongWritable cardO, LongWritable cardSP, LongWritable cardPO, LongWritable cardSO) { - - this.cardS = cardS; - this.cardP = cardP; - this.cardO = cardO; - this.cardSP = cardSP; - this.cardPO = cardPO; - this.cardSO = cardSO; - - } - - public void setCard(LongWritable cardS, LongWritable cardP, LongWritable cardO, LongWritable cardSP, LongWritable cardPO, LongWritable cardSO) { - this.cardS = cardS; - this.cardP = cardP; - this.cardO = cardO; - this.cardSP = cardSP; - this.cardPO = cardPO; - this.cardSO = cardSO; - - } - - public void setSCard(long cardS) { - this.cardS = new LongWritable(cardS); - } - - public void setPCard(long cardP) { - this.cardP = new LongWritable(cardP); - } - - public void setOCard(long cardO) { - this.cardO = new LongWritable(cardO); - } - - public void setSPCard(long cardSP) { - this.cardSP = new LongWritable(cardSP); - } - - public void setSOCard(long cardSO) { - this.cardSO = new LongWritable(cardSO); - } - - public void setPOCard(long cardPO) { - this.cardPO = new LongWritable(cardPO); - } - - public LongWritable getcardS() { - return this.cardS; - } - - public LongWritable getcardP() { - return this.cardP; - } - - public LongWritable getcardO() { - return this.cardO; - } - - public LongWritable getcardPO() { - return this.cardPO; - } - - public LongWritable getcardSO() { - return this.cardSO; - } - - public LongWritable getcardSP() { - return this.cardSP; - } - - @Override - public void write(DataOutput out) throws IOException { - cardS.write(out); - cardP.write(out); - cardO.write(out); - cardSO.write(out); - cardPO.write(out); - cardSP.write(out); - - } - - @Override - public void readFields(DataInput in) throws IOException { - cardS.readFields(in); - cardP.readFields(in); - cardO.readFields(in); - cardSO.readFields(in); - cardPO.readFields(in); - cardSP.readFields(in); - - } - - @Override - public int hashCode() { - int result = 7; - result = result * 17 + cardS.hashCode(); - result = result * 17 + cardP.hashCode(); - result = result * 17 + cardO.hashCode(); - result = result * 17 + cardSP.hashCode(); - result = result * 17 + cardPO.hashCode(); - result = result * 17 + cardSO.hashCode(); - - return result; - - } - - @Override - public boolean equals(Object o) { - if (o instanceof CardList) { - CardList comp = (CardList) o; - return cardS.equals(comp.cardS) && cardP.equals(comp.cardP) && cardO.equals(comp.cardO) && cardSP.equals(comp.cardSP) && cardSO.equals(comp.cardSO) - && cardPO.equals(comp.cardPO); - - } - return false; - } - - @Override - public String toString() { - return cardS + "\t" + cardP + "\t" + cardO + "\t" + cardSP + "\t" + cardPO + "\t" + cardSO; - - } - - @Override - public int compareTo(CardList o) { - - int cmp = cardS.compareTo(o.cardS); - if (cmp != 0) { - return cmp; - } - cmp = cardP.compareTo(o.cardP); - if (cmp != 0) { - return cmp; - } - cmp = cardO.compareTo(o.cardO); - if (cmp != 0) { - return cmp; - } - cmp = cardSP.compareTo(o.cardSP); - if (cmp != 0) { - return cmp; - } - cmp = cardPO.compareTo(o.cardPO); - if (cmp != 0) { - return cmp; - } - - return cardSO.compareTo(o.cardSO); - - } - -} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/5a03ef61/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/CardinalityType.java ---------------------------------------------------------------------- diff --git a/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/CardinalityType.java b/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/CardinalityType.java deleted file mode 100644 index 924f596..0000000 --- a/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/CardinalityType.java +++ /dev/null @@ -1,149 +0,0 @@ -package mvm.rya.joinselect.mr.utils; - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - -import org.apache.hadoop.io.LongWritable; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.WritableComparable; - -public class CardinalityType implements WritableComparable<CardinalityType> { - - private LongWritable card; - private Text cardType; - private LongWritable ts; - - public CardinalityType() { - card = new LongWritable(); - cardType = new Text(); - ts = new LongWritable(); - } - - public CardinalityType(int card, String cardType, long ts) { - - this.card = new LongWritable(card); - this.cardType = new Text(cardType); - this.ts = new LongWritable(ts); - - } - - public CardinalityType(LongWritable card, Text cardType, LongWritable ts) { - - this.card = card; - this.ts = ts; - this.cardType = cardType; - - } - - public void set(CardinalityType ct) { - this.card.set(ct.card.get()); - this.ts.set(ct.ts.get()); - this.cardType.set(ct.cardType); - } - - public void setCard(LongWritable card) { - this.card = card; - - } - - public void setCardType(Text cardType) { - this.cardType = cardType; - } - - public void setTS(LongWritable ts) { - this.ts = ts; - } - - public LongWritable getCard() { - return this.card; - } - - public Text getCardType() { - return this.cardType; - } - - public LongWritable getTS() { - return this.ts; - } - - @Override - public void write(DataOutput out) throws IOException { - card.write(out); - cardType.write(out); - ts.write(out); - - } - - @Override - public void readFields(DataInput in) throws IOException { - card.readFields(in); - cardType.readFields(in); - ts.readFields(in); - - } - - @Override - public int hashCode() { - int result = 7; - result = result * 17 + card.hashCode(); - result = result * 17 + cardType.hashCode(); - result = result * 17 + ts.hashCode(); - - return result; - - } - - @Override - public boolean equals(Object o) { - if (o instanceof CardinalityType) { - CardinalityType trip = (CardinalityType) o; - return card.equals(trip.card) && cardType.equals(trip.cardType) && ts.equals(trip.ts); - - } - return false; - } - - @Override - public String toString() { - return card + " " + cardType + " " + ts; - - } - - @Override - public int compareTo(CardinalityType o) { - - int cmp = cardType.compareTo(o.cardType); - if (cmp != 0) { - return cmp; - } - cmp = ts.compareTo(o.ts); - if (cmp != 0) { - return cmp; - } - return card.compareTo(o.card); - - } - -} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/5a03ef61/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/CompositeType.java ---------------------------------------------------------------------- diff --git a/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/CompositeType.java b/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/CompositeType.java deleted file mode 100644 index 57e6ee2..0000000 --- a/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/CompositeType.java +++ /dev/null @@ -1,122 +0,0 @@ -package mvm.rya.joinselect.mr.utils; - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.WritableComparable; - -public class CompositeType implements WritableComparable<CompositeType> { - - private Text oldKey; - private IntWritable priority; - - public CompositeType() { - oldKey = new Text(); - priority = new IntWritable(); - } - - public CompositeType(String oldKey, int priority) { - this.oldKey = new Text(oldKey); - this.priority = new IntWritable(priority); - } - - public CompositeType(Text oldKey, IntWritable priority) { - - this.oldKey = oldKey; - this.priority = priority; - - } - - public void setOldKey(Text oldKey) { - this.oldKey = oldKey; - - } - - public void setPriority(IntWritable priority) { - this.priority = priority; - } - - public Text getOldKey() { - return this.oldKey; - } - - public IntWritable getPriority() { - return this.priority; - } - - @Override - public void write(DataOutput out) throws IOException { - oldKey.write(out); - priority.write(out); - - } - - @Override - public void readFields(DataInput in) throws IOException { - oldKey.readFields(in); - priority.readFields(in); - - } - - @Override - public int hashCode() { - int result = 7; - result = result * 17 + oldKey.hashCode(); - // result = result*17+ priority.hashCode(); - - return result; - - } - - @Override - public boolean equals(Object o) { - if (o instanceof CompositeType) { - CompositeType comp = (CompositeType) o; - return oldKey.equals(comp.oldKey) && priority.equals(comp.priority); - - } - return false; - } - - @Override - public String toString() { - return oldKey + "\t" + priority; - - } - - @Override - public int compareTo(CompositeType o) { - int compare = getOldKey().compareTo(o.getOldKey()); - if (compare != 0) { - return compare; - } - - return getPriority().compareTo(o.getPriority()); - - } - -} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/5a03ef61/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/JoinSelectConstants.java ---------------------------------------------------------------------- diff --git a/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/JoinSelectConstants.java b/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/JoinSelectConstants.java deleted file mode 100644 index 3ec34d0..0000000 --- a/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/JoinSelectConstants.java +++ /dev/null @@ -1,46 +0,0 @@ -package mvm.rya.joinselect.mr.utils; - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - - -public class JoinSelectConstants { - - public static final String COUNT = "count"; - public static final String METADATA = "metadata"; - public static final byte[] EMPTY = new byte[0]; - - // config properties - public static final String PERFORMANT = "performant"; - public static final String USERNAME = "username"; - public static final String PASSWORD = "password"; - public static final String INSTANCE = "instance"; - public static final String ZOOKEEPERS = "zookeepers"; - public static final String INPUTPATH = "inputpath"; - public static final String OUTPUTPATH = "outputpath"; - public static final String PROSPECTS_OUTPUTPATH = "prospects.outputpath"; - public static final String SPO_OUTPUTPATH = "spo.outputpath"; - public static final String AUTHS = "auths"; - public static final String PROSPECTS_TABLE = "prospects.table"; - public static final String SPO_TABLE = "spo.table"; - public static final String SELECTIVITY_TABLE = "selectivity.table"; - public static final String MOCK = "mock"; - -} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/5a03ef61/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/JoinSelectStatsUtil.java ---------------------------------------------------------------------- diff --git a/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/JoinSelectStatsUtil.java b/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/JoinSelectStatsUtil.java deleted file mode 100644 index cf8db40..0000000 --- a/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/JoinSelectStatsUtil.java +++ /dev/null @@ -1,183 +0,0 @@ -package mvm.rya.joinselect.mr.utils; - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - - -import static mvm.rya.accumulo.AccumuloRdfConstants.EMPTY_CV; -import static mvm.rya.accumulo.AccumuloRdfConstants.EMPTY_VALUE; -import static mvm.rya.api.RdfCloudTripleStoreConstants.EMPTY_TEXT; -import static mvm.rya.joinselect.mr.utils.JoinSelectConstants.INSTANCE; -import static mvm.rya.joinselect.mr.utils.JoinSelectConstants.PASSWORD; -import static mvm.rya.joinselect.mr.utils.JoinSelectConstants.USERNAME; -import static mvm.rya.joinselect.mr.utils.JoinSelectConstants.ZOOKEEPERS; - -import java.io.IOException; - -import mvm.rya.api.resolver.triple.TripleRow; - -import org.apache.accumulo.core.client.AccumuloSecurityException; -import org.apache.accumulo.core.client.mapreduce.AccumuloInputFormat; -import org.apache.accumulo.core.client.mapreduce.AccumuloOutputFormat; -import org.apache.accumulo.core.client.security.tokens.PasswordToken; -import org.apache.accumulo.core.data.Mutation; -import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.security.Authorizations; -import org.apache.accumulo.core.security.ColumnVisibility; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.io.IntWritable; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.mapreduce.Job; -import org.apache.hadoop.mapreduce.Mapper; -import org.apache.hadoop.mapreduce.lib.input.MultipleInputs; -import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat; -import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat; - -public class JoinSelectStatsUtil { - - public static void initSumMRJob(Job job, String inputPath, String outtable, String auths) throws AccumuloSecurityException, IOException { - Configuration conf = job.getConfiguration(); - String username = conf.get(USERNAME); - String password = conf.get(PASSWORD); - String instance = conf.get(INSTANCE); - String zookeepers = conf.get(ZOOKEEPERS); - - if (zookeepers != null) { - AccumuloOutputFormat.setConnectorInfo(job, username, new PasswordToken(password)); - AccumuloOutputFormat.setZooKeeperInstance(job, instance, zookeepers); - } else { - throw new IllegalArgumentException("Must specify zookeepers"); - } - - SequenceFileInputFormat.addInputPath(job, new Path(inputPath)); - job.setInputFormatClass(SequenceFileInputFormat.class); - job.setMapOutputKeyClass(TripleEntry.class); - job.setMapOutputValueClass(CardList.class); - - AccumuloOutputFormat.setDefaultTableName(job, outtable); - job.setOutputFormatClass(AccumuloOutputFormat.class); - job.setOutputKeyClass(Text.class); - job.setOutputValueClass(Mutation.class); - - } - - public static void initTableMRJob(Job job, String intable, String outtable, String auths) throws AccumuloSecurityException { - Configuration conf = job.getConfiguration(); - String username = conf.get(USERNAME); - String password = conf.get(PASSWORD); - String instance = conf.get(INSTANCE); - String zookeepers = conf.get(ZOOKEEPERS); - - System.out.println("Zookeepers are " + auths); - - if (zookeepers != null) { - AccumuloInputFormat.setZooKeeperInstance(job, instance, zookeepers); - AccumuloOutputFormat.setZooKeeperInstance(job, instance, zookeepers); - } else { - throw new IllegalArgumentException("Must specify either mock or zookeepers"); - } - - AccumuloInputFormat.setConnectorInfo(job, username, new PasswordToken(password)); - AccumuloInputFormat.setScanAuthorizations(job, new Authorizations(auths)); - AccumuloInputFormat.setInputTableName(job, intable); - job.setInputFormatClass(AccumuloInputFormat.class); - job.setMapOutputKeyClass(Text.class); - job.setMapOutputValueClass(IntWritable.class); - - // OUTPUT - AccumuloOutputFormat.setConnectorInfo(job, username, new PasswordToken(password)); - AccumuloOutputFormat.setDefaultTableName(job, outtable); - job.setOutputFormatClass(AccumuloOutputFormat.class); - job.setOutputKeyClass(Text.class); - job.setOutputValueClass(Mutation.class); - - } - - public static void initTabToSeqFileJob(Job job, String intable, String outpath, String auths) throws AccumuloSecurityException { - - Configuration conf = job.getConfiguration(); - String username = conf.get(USERNAME); - String password = conf.get(PASSWORD); - String instance = conf.get(INSTANCE); - String zookeepers = conf.get(ZOOKEEPERS); - - System.out.println("Zookeepers are " + auths); - - if (zookeepers != null) { - AccumuloInputFormat.setZooKeeperInstance(job, instance, zookeepers); - } else { - throw new IllegalArgumentException("Must specify either mock or zookeepers"); - } - - AccumuloInputFormat.setConnectorInfo(job, username, new PasswordToken(password)); - AccumuloInputFormat.setScanAuthorizations(job, new Authorizations(auths)); - AccumuloInputFormat.setInputTableName(job, intable); - job.setInputFormatClass(AccumuloInputFormat.class); - job.setMapOutputKeyClass(CompositeType.class); - job.setMapOutputValueClass(TripleCard.class); - - // OUTPUT - SequenceFileOutputFormat.setOutputPath(job, new Path(outpath)); - job.setOutputFormatClass(SequenceFileOutputFormat.class); - job.setOutputKeyClass(CompositeType.class); - job.setOutputValueClass(TripleCard.class); - - } - - public static void initJoinMRJob(Job job, String prospectsPath, String spoPath, Class<? extends Mapper<CompositeType,TripleCard,?,?>> mapperClass, - String outPath, String auths) throws AccumuloSecurityException { - - MultipleInputs.addInputPath(job, new Path(prospectsPath), SequenceFileInputFormat.class, mapperClass); - MultipleInputs.addInputPath(job, new Path(spoPath), SequenceFileInputFormat.class, mapperClass); - job.setMapOutputKeyClass(CompositeType.class); - job.setMapOutputValueClass(TripleCard.class); - - SequenceFileOutputFormat.setOutputPath(job, new Path(outPath)); - job.setOutputFormatClass(SequenceFileOutputFormat.class); - job.setOutputKeyClass(TripleEntry.class); - job.setOutputValueClass(CardList.class); - - } - - public static Mutation createMutation(TripleRow tripleRow) { - Mutation mutation = new Mutation(new Text(tripleRow.getRow())); - byte[] columnVisibility = tripleRow.getColumnVisibility(); - ColumnVisibility cv = columnVisibility == null ? EMPTY_CV : new ColumnVisibility(columnVisibility); - Long timestamp = tripleRow.getTimestamp(); - boolean hasts = timestamp != null; - timestamp = timestamp == null ? 0l : timestamp; - byte[] value = tripleRow.getValue(); - Value v = value == null ? EMPTY_VALUE : new Value(value); - byte[] columnQualifier = tripleRow.getColumnQualifier(); - Text cqText = columnQualifier == null ? EMPTY_TEXT : new Text(columnQualifier); - byte[] columnFamily = tripleRow.getColumnFamily(); - Text cfText = columnFamily == null ? EMPTY_TEXT : new Text(columnFamily); - - if (hasts) { - mutation.put(cfText, cqText, cv, timestamp, v); - } else { - mutation.put(cfText, cqText, cv, v); - - } - return mutation; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/5a03ef61/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/TripleCard.java ---------------------------------------------------------------------- diff --git a/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/TripleCard.java b/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/TripleCard.java deleted file mode 100644 index 467f754..0000000 --- a/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/TripleCard.java +++ /dev/null @@ -1,145 +0,0 @@ -package mvm.rya.joinselect.mr.utils; - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - -import org.apache.hadoop.io.WritableComparable; - -public class TripleCard implements WritableComparable<TripleCard> { - - private CardinalityType card = null; - private TripleEntry te = null; - - private CardinalityType tempCard = new CardinalityType(); - private TripleEntry tempTe = new TripleEntry(); - - public TripleCard() {} - - public TripleCard(CardinalityType card) { - this.setCard(card); - } - - public TripleCard(TripleEntry te) { - this.setTE(te); - } - - public void setCard(CardinalityType card) { - tempCard.set(card); - this.card = tempCard; - this.te = null; - } - - public void setTE(TripleEntry te) { - tempTe.setTE(te); - this.te = tempTe; - this.card = null; - } - - public CardinalityType getCard() { - return this.card; - } - - public TripleEntry getTE() { - return this.te; - } - - public boolean isCardNull() { - return (card == null); - } - - public boolean isTeNull() { - return (te == null); - } - - @Override - public void write(DataOutput out) throws IOException { - if (card != null) { - out.writeBoolean(true); - card.write(out); - } else { - out.writeBoolean(false); - te.write(out); - } - - } - - @Override - public void readFields(DataInput in) throws IOException { - if (in.readBoolean()) { - tempCard.readFields(in); - card = tempCard; - te = null; - } else { - tempTe.readFields(in); - te = tempTe; - card = null; - } - } - - @Override - public int hashCode() { - int result = 7; - if (card != null) { - result = result * 17 + card.hashCode(); - } else { - result = result * 17 + te.hashCode(); - } - return result; - - } - - @Override - public boolean equals(Object o) { - if (o instanceof TripleCard) { - TripleCard comp = (TripleCard) o; - if (card != null) { - return card.equals(comp.card); - } else { - return te.equals(comp.te); - } - } - return false; - } - - @Override - public String toString() { - if (card != null) { - return card.toString(); - } else { - return te.toString(); - } - } - - @Override - public int compareTo(TripleCard o) { - - if (card != null) { - return card.compareTo(o.card); - } else { - return te.compareTo(o.te); - } - } - -} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/5a03ef61/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/TripleEntry.java ---------------------------------------------------------------------- diff --git a/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/TripleEntry.java b/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/TripleEntry.java deleted file mode 100644 index 7deb346..0000000 --- a/extras/rya.prospector/src/main/java/mvm/rya/joinselect/mr/utils/TripleEntry.java +++ /dev/null @@ -1,180 +0,0 @@ -package mvm.rya.joinselect.mr.utils; - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - - -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - -import org.apache.hadoop.io.Text; -import org.apache.hadoop.io.WritableComparable; - -public class TripleEntry implements WritableComparable<TripleEntry> { - - private Text first; - private Text second; - private Text firstPos; - private Text secondPos; - private Text keyPos; - - public TripleEntry() { - - first = new Text(); - second = new Text(); - firstPos = new Text(); - secondPos = new Text(); - keyPos = new Text(); - - } - - public TripleEntry(String first, String second, String firstPos, String secondPos, String keyPos) { - this.first = new Text(first); - this.second = new Text(second); - this.firstPos = new Text(firstPos); - this.secondPos = new Text(secondPos); - this.keyPos = new Text(keyPos); - } - - public TripleEntry(Text first, Text second, Text firstPos, Text secondPos, Text keyPos) { - this.first = first; - this.second = second; - this.firstPos = firstPos; - this.secondPos = secondPos; - this.keyPos = keyPos; - } - - public void setEntry(Text first, Text second) { - this.first = first; - this.second = second; - } - - public void setPosition(Text firstPos, Text secondPos, Text keyPos) { - this.firstPos = firstPos; - this.secondPos = secondPos; - this.keyPos = keyPos; - } - - public void setTE(TripleEntry te) { - - this.first.set(te.first); - this.second.set(te.second); - this.firstPos.set(te.firstPos); - this.secondPos.set(te.secondPos); - this.keyPos.set(te.keyPos); - - } - - public Text getFirst() { - return this.first; - } - - public Text getSecond() { - return this.second; - } - - public Text getFirstPos() { - return this.firstPos; - } - - public Text getSecondPos() { - return this.secondPos; - } - - public Text getKeyPos() { - return this.keyPos; - } - - @Override - public void write(DataOutput out) throws IOException { - first.write(out); - second.write(out); - firstPos.write(out); - secondPos.write(out); - keyPos.write(out); - - } - - @Override - public void readFields(DataInput in) throws IOException { - first.readFields(in); - second.readFields(in); - firstPos.readFields(in); - secondPos.readFields(in); - keyPos.readFields(in); - - } - - @Override - public int hashCode() { - int result = 7; - result = result * 17 + first.hashCode(); - result = result * 17 + second.hashCode(); - result = result * 17 + firstPos.hashCode(); - result = result * 17 + secondPos.hashCode(); - result = result * 17 + keyPos.hashCode(); - - return result; - - } - - @Override - public boolean equals(Object o) { - if (o instanceof TripleEntry) { - TripleEntry trip = (TripleEntry) o; - return first.equals(trip.first) && second.equals(trip.second) && firstPos.equals(trip.firstPos) && secondPos.equals(trip.secondPos) - && keyPos.equals(trip.keyPos); - - } - return false; - } - - @Override - public String toString() { - return first + "\t" + firstPos + "\t" + second + "\t" + secondPos + "\t" + keyPos; - - } - - @Override - public int compareTo(TripleEntry o) { - - int cmp = first.compareTo(o.first); - if (cmp != 0) { - return cmp; - } - cmp = firstPos.compareTo(o.firstPos); - if (cmp != 0) { - return cmp; - } - cmp = second.compareTo(o.second); - if (cmp != 0) { - return cmp; - } - - cmp = secondPos.compareTo(o.secondPos); - if (cmp != 0) { - return cmp; - } - return keyPos.compareTo(o.keyPos); - - } - -} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/5a03ef61/extras/rya.prospector/src/main/resources/META-INF/services/mvm.rya.prospector.plans.IndexWorkPlan ---------------------------------------------------------------------- diff --git a/extras/rya.prospector/src/main/resources/META-INF/services/mvm.rya.prospector.plans.IndexWorkPlan b/extras/rya.prospector/src/main/resources/META-INF/services/mvm.rya.prospector.plans.IndexWorkPlan deleted file mode 100644 index 38258c1..0000000 --- a/extras/rya.prospector/src/main/resources/META-INF/services/mvm.rya.prospector.plans.IndexWorkPlan +++ /dev/null @@ -1 +0,0 @@ -mvm.rya.prospector.plans.impl.CountPlan \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/5a03ef61/extras/rya.prospector/src/test/groovy/mvm/rya/prospector/mr/ProspectorTest.groovy ---------------------------------------------------------------------- diff --git a/extras/rya.prospector/src/test/groovy/mvm/rya/prospector/mr/ProspectorTest.groovy b/extras/rya.prospector/src/test/groovy/mvm/rya/prospector/mr/ProspectorTest.groovy deleted file mode 100644 index 766a239..0000000 --- a/extras/rya.prospector/src/test/groovy/mvm/rya/prospector/mr/ProspectorTest.groovy +++ /dev/null @@ -1,178 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package mvm.rya.prospector.mr - -import com.google.common.collect.Iterators -import com.google.common.collect.Lists -import mvm.rya.accumulo.AccumuloRyaDAO -import mvm.rya.accumulo.AccumuloRdfConfiguration -import mvm.rya.api.persist.RdfEvalStatsDAO -import mvm.rya.api.domain.RyaStatement -import mvm.rya.api.domain.RyaType -import mvm.rya.api.domain.RyaURI -import mvm.rya.prospector.domain.IndexEntry -import mvm.rya.prospector.domain.TripleValueType -import mvm.rya.prospector.service.ProspectorService -import mvm.rya.prospector.service.ProspectorServiceEvalStatsDAO -import mvm.rya.prospector.utils.ProspectorConstants -import org.apache.accumulo.core.client.Instance -import org.apache.accumulo.core.client.mock.MockInstance -import org.apache.accumulo.core.security.Authorizations -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.Path -import org.apache.hadoop.util.ToolRunner -import org.junit.Test -import org.openrdf.model.vocabulary.XMLSchema -import org.openrdf.model.impl.URIImpl - -import static org.junit.Assert.assertEquals -import org.openrdf.model.impl.LiteralImpl -import org.openrdf.model.Value - -/** - * Date: 12/4/12 - * Time: 4:33 PM - */ -class ProspectorTest { - - @Test - public void testCount() throws Exception { - - Instance mock = new MockInstance("accumulo"); - - def connector = mock.getConnector("user", "pass".bytes) - def intable = "rya_spo" - def outtable = "rya_prospects" - if (connector.tableOperations().exists(outtable)) - connector.tableOperations().delete(outtable) - connector.tableOperations().create(outtable) - - AccumuloRyaDAO ryaDAO = new AccumuloRyaDAO(); - ryaDAO.setConnector(connector); - ryaDAO.init() - - ryaDAO.add(new RyaStatement(new RyaURI("urn:gem:etype#1234"), new RyaURI("urn:gem#pred"), new RyaType("mydata1"))) - ryaDAO.add(new RyaStatement(new RyaURI("urn:gem:etype#1234"), new RyaURI("urn:gem#pred"), new RyaType("mydata2"))) - ryaDAO.add(new RyaStatement(new RyaURI("urn:gem:etype#1234"), new RyaURI("urn:gem#pred"), new RyaType("12"))) - ryaDAO.add(new RyaStatement(new RyaURI("urn:gem:etype#1235"), new RyaURI("urn:gem#pred"), new RyaType(XMLSchema.INTEGER, "12"))) - ryaDAO.add(new RyaStatement(new RyaURI("urn:gem:etype#1235"), new RyaURI("urn:gem#pred1"), new RyaType("12"))) - - def confFile = "stats_cluster_config.xml" - def confPath = new Path(getClass().getClassLoader().getResource(confFile).toString()) - def args = (String[]) [confPath]; - ToolRunner.run(new Prospector(), args); - debugTable(connector, outtable) - - def scanner = connector.createScanner(outtable, new Authorizations("U", "FOUO")) - def iter = scanner.iterator() -// assertEquals(11, Iterators.size(iter)) - - ryaDAO.destroy() - - def conf = new Configuration() - conf.addResource(confPath) - // debugTable(mrInfo, outtable) - - def service = new ProspectorService(connector, outtable) - def auths = (String[]) ["U", "FOUO"] - def prospects = service.getProspects(auths) - def plist = Lists.newArrayList(prospects) - assertEquals(1, plist.size()) - - def rdfConf = new AccumuloRdfConfiguration(conf) - rdfConf.setAuths("U","FOUO") - - prospects = service.getProspectsInRange(System.currentTimeMillis() - 100000, System.currentTimeMillis() + 10000, auths) - plist = Lists.newArrayList(prospects) - assertEquals(1, plist.size()) - - List<String> queryTerms = new ArrayList<String>(); - queryTerms.add("urn:gem:etype"); - def query = service.query(plist, ProspectorConstants.COUNT, TripleValueType.entity.name(), queryTerms, XMLSchema.ANYURI.stringValue(), auths) - assertEquals(1, query.size()) -// assertEquals( -// new IndexEntry(index: ProspectorConstants.COUNT, data: "urn:gem:etype", dataType: XMLSchema.ANYURI.stringValue(), -// tripleValueType: TripleValueType.entity, visibility: "", count: -1, timestamp: plist.get(0)), -// query.get(0)) - - queryTerms = new ArrayList<String>(); - queryTerms.add("urn:gem:etype#1234"); - query = service.query(plist, ProspectorConstants.COUNT, TripleValueType.subject.name(), queryTerms, XMLSchema.ANYURI.stringValue(), auths) - assertEquals(1, query.size()) - - queryTerms = new ArrayList<String>(); - queryTerms.add("urn:gem#pred"); - query = service.query(plist, ProspectorConstants.COUNT, TripleValueType.predicate.name(), queryTerms, XMLSchema.ANYURI.stringValue(), auths) - assertEquals(1, query.size()) - assertEquals( - new IndexEntry(index: ProspectorConstants.COUNT, data: "urn:gem#pred", dataType: XMLSchema.ANYURI.stringValue(), - tripleValueType: TripleValueType.predicate, visibility: "", count: 4l, timestamp: plist.get(0)), - query.get(0)) - - queryTerms = new ArrayList<String>(); - queryTerms.add("mydata1"); - query = service.query(plist, ProspectorConstants.COUNT, TripleValueType.object.name(), queryTerms, XMLSchema.STRING.stringValue(), auths) - assertEquals(1, query.size()) -// assertEquals( -// new IndexEntry(index: ProspectorConstants.COUNT, data: "mydata1", dataType: XMLSchema.STRING.stringValue(), -// tripleValueType: TripleValueType.object, visibility: "", count: -1, timestamp: plist.get(0)), -// query.get(0)) - - queryTerms = new ArrayList<String>(); - queryTerms.add("urn:gem:etype#1234"); - queryTerms.add("urn:gem#pred"); - query = service.query(plist, ProspectorConstants.COUNT, TripleValueType.subjectpredicate.name(), queryTerms, XMLSchema.STRING.stringValue(), auths) - assertEquals(1, query.size()) -// assertEquals( -// new IndexEntry(index: ProspectorConstants.COUNT, data: "urn:gem:etype#1234" + "\u0000" + "urn:gem#pred", dataType: XMLSchema.STRING.stringValue(), -// tripleValueType: TripleValueType.subjectpredicate, visibility: "", count: -1, timestamp: plist.get(0)), -// query.get(0)) - - queryTerms = new ArrayList<String>(); - queryTerms.add("urn:gem#pred"); - queryTerms.add("12"); - query = service.query(plist, ProspectorConstants.COUNT, TripleValueType.predicateobject.name(), queryTerms, XMLSchema.STRING.stringValue(), auths) - assertEquals(1, query.size()) -// assertEquals( -// new IndexEntry(index: ProspectorConstants.COUNT, data: "urn:gem#pred" + "\u0000" + "12", dataType: XMLSchema.STRING.stringValue(), -// tripleValueType: TripleValueType.predicateobject, visibility: "", count: -1, timestamp: plist.get(0)), -// query.get(0)) - - queryTerms = new ArrayList<String>(); - queryTerms.add("urn:gem:etype#1234"); - queryTerms.add("mydata1"); - query = service.query(plist, ProspectorConstants.COUNT, TripleValueType.subjectobject.name(), queryTerms, XMLSchema.STRING.stringValue(), auths) - - assertEquals(1, query.size()) -// assertEquals( -// new IndexEntry(index: ProspectorConstants.COUNT, data: "urn:gem:etype#1234" + "\u0000" + "mydata1", dataType: XMLSchema.STRING.stringValue(), -// tripleValueType: TripleValueType.subjectobject, visibility: "", count: -1, timestamp: plist.get(0)), -// query.get(0)) - - //should be in a teardown method - connector.tableOperations().delete(outtable) - } - - private void debugTable(def connector, String table) { - connector.createScanner(table, new Authorizations((String[]) ["U", "FOUO"])).iterator().each { - println it - } - } -} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/5a03ef61/extras/rya.prospector/src/test/groovy/mvm/rya/prospector/service/ProspectorServiceEvalStatsDAOTest.groovy ---------------------------------------------------------------------- diff --git a/extras/rya.prospector/src/test/groovy/mvm/rya/prospector/service/ProspectorServiceEvalStatsDAOTest.groovy b/extras/rya.prospector/src/test/groovy/mvm/rya/prospector/service/ProspectorServiceEvalStatsDAOTest.groovy deleted file mode 100644 index 5bbbee8..0000000 --- a/extras/rya.prospector/src/test/groovy/mvm/rya/prospector/service/ProspectorServiceEvalStatsDAOTest.groovy +++ /dev/null @@ -1,182 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package mvm.rya.prospector.service - -import com.google.common.collect.Iterators -import mvm.rya.accumulo.AccumuloRdfConfiguration -import mvm.rya.accumulo.AccumuloRyaDAO -import mvm.rya.api.domain.RyaStatement -import mvm.rya.api.domain.RyaType -import mvm.rya.api.domain.RyaURI -import mvm.rya.api.persist.RdfEvalStatsDAO -import mvm.rya.prospector.mr.Prospector -import org.apache.accumulo.core.client.Instance -import org.apache.accumulo.core.client.mock.MockInstance -import org.apache.accumulo.core.security.Authorizations -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.Path -import org.apache.hadoop.util.ToolRunner -import org.junit.Test -import org.openrdf.model.impl.URIImpl -import org.openrdf.model.vocabulary.XMLSchema - -import static org.junit.Assert.assertEquals -import org.openrdf.model.impl.LiteralImpl -import org.openrdf.model.Value - -/** - * Date: 1/26/13 - * Time: 3:00 PM - */ -class ProspectorServiceEvalStatsDAOTest { - - @Test - public void testCount() throws Exception { - - Instance mock = new MockInstance("accumulo"); - - def connector = mock.getConnector("user", "pass".bytes) - def intable = "rya_spo" - def outtable = "rya_prospects" - if (connector.tableOperations().exists(outtable)) - connector.tableOperations().delete(outtable) - connector.tableOperations().create(outtable) - - AccumuloRyaDAO ryaDAO = new AccumuloRyaDAO(); - ryaDAO.setConnector(connector); - ryaDAO.init() - - ryaDAO.add(new RyaStatement(new RyaURI("urn:gem:etype#1234"), new RyaURI("urn:gem#pred"), new RyaType("mydata1"))) - ryaDAO.add(new RyaStatement(new RyaURI("urn:gem:etype#1234"), new RyaURI("urn:gem#pred"), new RyaType("mydata2"))) - ryaDAO.add(new RyaStatement(new RyaURI("urn:gem:etype#1234"), new RyaURI("urn:gem#pred"), new RyaType("12"))) - ryaDAO.add(new RyaStatement(new RyaURI("urn:gem:etype#1235"), new RyaURI("urn:gem#pred"), new RyaType(XMLSchema.INTEGER, "12"))) - ryaDAO.add(new RyaStatement(new RyaURI("urn:gem:etype#1235"), new RyaURI("urn:gem#pred1"), new RyaType("12"))) - - def confFile = "stats_cluster_config.xml" - def confPath = new Path(getClass().getClassLoader().getResource(confFile).toString()) - def args = (String[]) [confPath]; - ToolRunner.run(new Prospector(), args); - debugTable(connector, outtable) - - def scanner = connector.createScanner(outtable, new Authorizations("U", "FOUO")) - def iter = scanner.iterator() -// assertEquals(11, Iterators.size(iter)) - - ryaDAO.destroy() - - def conf = new Configuration() - conf.addResource(confPath) -// debugTable(connector, outtable) - - def rdfConf = new AccumuloRdfConfiguration(conf) - rdfConf.setAuths("U","FOUO") - def evalDao = new ProspectorServiceEvalStatsDAO(connector, rdfConf) - evalDao.init() - - List<Value> values = new ArrayList<Value>(); - values.add( new URIImpl("urn:gem#pred")); - - def count = evalDao.getCardinality(rdfConf, RdfEvalStatsDAO.CARDINALITY_OF.PREDICATE, values) - assertEquals(4.0, count, 0.001); - - values = new ArrayList<Value>(); - values.add( new LiteralImpl("mydata1")); - - count = evalDao.getCardinality(rdfConf, RdfEvalStatsDAO.CARDINALITY_OF.OBJECT, values); - assertEquals(1.0, count, 0.001); - - values = new ArrayList<Value>(); - values.add( new LiteralImpl("mydata3")); - - count = evalDao.getCardinality(rdfConf, RdfEvalStatsDAO.CARDINALITY_OF.OBJECT, values); - assertEquals(-1.0, count, 0.001); - - //should be in a teardown method - connector.tableOperations().delete(outtable) - } - - @Test - public void testNoAuthsCount() throws Exception { - - Instance mock = new MockInstance("accumulo"); - def connector = mock.getConnector("user", "pass".bytes) - def intable = "rya_spo" - def outtable = "rya_prospects" - if (connector.tableOperations().exists(outtable)) - connector.tableOperations().delete(outtable) - connector.tableOperations().create(outtable) - connector.securityOperations().createUser("user", "pass".bytes, new Authorizations("U", "FOUO")) - - AccumuloRyaDAO ryaDAO = new AccumuloRyaDAO(); - ryaDAO.setConnector(connector); - ryaDAO.init() - - ryaDAO.add(new RyaStatement(new RyaURI("urn:gem:etype#1234"), new RyaURI("urn:gem#pred"), new RyaType("mydata1"))) - ryaDAO.add(new RyaStatement(new RyaURI("urn:gem:etype#1234"), new RyaURI("urn:gem#pred"), new RyaType("mydata2"))) - ryaDAO.add(new RyaStatement(new RyaURI("urn:gem:etype#1234"), new RyaURI("urn:gem#pred"), new RyaType("12"))) - ryaDAO.add(new RyaStatement(new RyaURI("urn:gem:etype#1235"), new RyaURI("urn:gem#pred"), new RyaType(XMLSchema.INTEGER, "12"))) - ryaDAO.add(new RyaStatement(new RyaURI("urn:gem:etype#1235"), new RyaURI("urn:gem#pred1"), new RyaType("12"))) - - def confFile = "stats_cluster_config.xml" - def confPath = new Path(getClass().getClassLoader().getResource(confFile).toString()) - def args = (String[]) [confPath]; - ToolRunner.run(new Prospector(), args); - - def scanner = connector.createScanner(outtable, new Authorizations("U", "FOUO")) - def iter = scanner.iterator() -// assertEquals(11, Iterators.size(iter)) - - ryaDAO.destroy() - - def conf = new Configuration() - conf.addResource(confPath) - - def rdfConf = new AccumuloRdfConfiguration(conf) -// rdfConf.setAuths("U","FOUO") - def evalDao = new ProspectorServiceEvalStatsDAO(connector, rdfConf) - evalDao.init() - - - List<Value> values = new ArrayList<Value>(); - values.add( new URIImpl("urn:gem#pred")); - def count = evalDao.getCardinality(rdfConf, RdfEvalStatsDAO.CARDINALITY_OF.PREDICATE, values) - assertEquals(4.0, count, 0.001); - - values = new ArrayList<Value>(); - values.add( new LiteralImpl("mydata1")); - count = evalDao.getCardinality(rdfConf, RdfEvalStatsDAO.CARDINALITY_OF.OBJECT, values); - assertEquals(1.0, count, 0.001); - - values = new ArrayList<Value>(); - values.add( new LiteralImpl("mydata3")); - - count = evalDao.getCardinality(rdfConf, RdfEvalStatsDAO.CARDINALITY_OF.OBJECT, values); - assertEquals(-1.0, count, 0.001); - - //should be in a teardown method - connector.tableOperations().delete(outtable) - } - - private void debugTable(def connector, String table) { - connector.createScanner(table, new Authorizations((String[]) ["U", "FOUO"])).iterator().each { - println it - } - } -}
