Repository: incubator-rya Updated Branches: refs/heads/master 78f958d2f -> 8fca552e1
adding context;Closes #122 Project: http://git-wip-us.apache.org/repos/asf/incubator-rya/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-rya/commit/8fca552e Tree: http://git-wip-us.apache.org/repos/asf/incubator-rya/tree/8fca552e Diff: http://git-wip-us.apache.org/repos/asf/incubator-rya/diff/8fca552e Branch: refs/heads/master Commit: 8fca552e1aaff6a37e63f351faea26ee63465d06 Parents: 78f958d Author: pujav65 <puja...@gmail.com> Authored: Mon Nov 21 14:53:12 2016 -0500 Committer: pujav65 <puja...@gmail.com> Committed: Thu Dec 15 14:53:20 2016 -0500 ---------------------------------------------------------------------- common/rya.api/pom.xml | 5 ++ .../org/apache/rya/api/domain/RyaStatement.java | 36 +++++++++- .../rya/api/domain/StatementMetadata.java | 73 ++++++++++++++++++++ .../apache/rya/accumulo/AccumuloRyaDAOTest.java | 14 ++-- .../dao/SimpleMongoDBStorageStrategy.java | 29 ++++++-- .../RdfCloudTripleStoreConnection.java | 5 +- 6 files changed, 144 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/8fca552e/common/rya.api/pom.xml ---------------------------------------------------------------------- diff --git a/common/rya.api/pom.xml b/common/rya.api/pom.xml index 6728c7c..f73c006 100644 --- a/common/rya.api/pom.xml +++ b/common/rya.api/pom.xml @@ -58,6 +58,11 @@ under the License. <artifactId>guava</artifactId> </dependency> <dependency> + <groupId>com.google.code.gson</groupId> + <artifactId>gson</artifactId> + <version>2.8.0</version> + </dependency> + <dependency> <groupId>com.github.stephenc.findbugs</groupId> <artifactId>findbugs-annotations</artifactId> </dependency> http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/8fca552e/common/rya.api/src/main/java/org/apache/rya/api/domain/RyaStatement.java ---------------------------------------------------------------------- diff --git a/common/rya.api/src/main/java/org/apache/rya/api/domain/RyaStatement.java b/common/rya.api/src/main/java/org/apache/rya/api/domain/RyaStatement.java index 869c23b..de41be9 100644 --- a/common/rya.api/src/main/java/org/apache/rya/api/domain/RyaStatement.java +++ b/common/rya.api/src/main/java/org/apache/rya/api/domain/RyaStatement.java @@ -48,18 +48,25 @@ public class RyaStatement { this(subject, predicate, object, context, null); } + public RyaStatement(RyaURI subject, RyaURI predicate, RyaType object, RyaURI context, String qualifier) { - this(subject, predicate, object, context, qualifier, null); + this(subject, predicate, object, context, qualifier, new StatementMetadata()); + } + + public RyaStatement(RyaURI subject, RyaURI predicate, RyaType object, RyaURI context, String qualifier, StatementMetadata metadata) { + this(subject, predicate, object, context, qualifier, metadata, null); } - public RyaStatement(RyaURI subject, RyaURI predicate, RyaType object, RyaURI context, String qualifier, byte[] columnVisibility) { - this(subject, predicate, object, context, qualifier, columnVisibility, null); + public RyaStatement(RyaURI subject, RyaURI predicate, RyaType object, RyaURI context, String qualifier, StatementMetadata metadata, byte[] columnVisibility) { + this(subject, predicate, object, context, qualifier, columnVisibility, metadata.toBytes()); } + @Deprecated public RyaStatement(RyaURI subject, RyaURI predicate, RyaType object, RyaURI context, String qualifier, byte[] columnVisibility, byte[] value) { this(subject, predicate, object, context, qualifier, columnVisibility, value, null); } + @Deprecated public RyaStatement(RyaURI subject, RyaURI predicate, RyaType object, RyaURI context, String qualifier, byte[] columnVisibility, byte[] value, Long timestamp) { this.subject = subject; this.predicate = predicate; @@ -110,11 +117,28 @@ public class RyaStatement { public void setColumnVisibility(byte[] columnVisibility) { this.columnVisibility = columnVisibility; } + + public StatementMetadata getMetadata() { + // try to deserialize the value, if not assume that there was + // no explicit metadata + try { + return new StatementMetadata(value); + } + catch (Exception ex){ + return null; + } + } + + public void setStatementMetadata(StatementMetadata metadata){ + this.value = metadata.toBytes(); + } + @Deprecated public byte[] getValue() { return value; } + @Deprecated public void setValue(byte[] value) { this.value = value; } @@ -210,11 +234,17 @@ public class RyaStatement { return this; } + @Deprecated public RyaStatementBuilder setValue(byte[] value) { ryaStatement.setValue(value); return this; } + public RyaStatementBuilder setMetadata(StatementMetadata metadata) { + ryaStatement.setValue(metadata.toBytes()); + return this; + } + public RyaStatementBuilder setColumnVisibility(byte[] columnVisibility) { ryaStatement.setColumnVisibility(columnVisibility); return this; http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/8fca552e/common/rya.api/src/main/java/org/apache/rya/api/domain/StatementMetadata.java ---------------------------------------------------------------------- diff --git a/common/rya.api/src/main/java/org/apache/rya/api/domain/StatementMetadata.java b/common/rya.api/src/main/java/org/apache/rya/api/domain/StatementMetadata.java new file mode 100644 index 0000000..904f86c --- /dev/null +++ b/common/rya.api/src/main/java/org/apache/rya/api/domain/StatementMetadata.java @@ -0,0 +1,73 @@ +package org.apache.rya.api.domain; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.io.UnsupportedEncodingException; +import java.util.HashMap; +import java.util.Map; + +import org.apache.rya.api.persist.RdfDAOException; + +import com.google.gson.Gson; + +public class StatementMetadata { + + private static Gson gson = new Gson(); + public static StatementMetadata EMPTY_METADATA = new StatementMetadata(); + + private Map<String, String> metadataMap = new HashMap<String, String>(); + + public StatementMetadata() { + + } + + public void addMetadata(String key, String value){ + metadataMap.put(key, value); + } + + public StatementMetadata(byte[] value) throws RdfDAOException { + try { + // try to convert back to a json string and then back to the map. + String metadataString = new String(value, "UTF8"); + metadataMap = gson.fromJson(metadataString, HashMap.class); + } catch (UnsupportedEncodingException e) { + throw new RdfDAOException(e); + } + } + + public StatementMetadata(String statementMetadata) { + try { + metadataMap = gson.fromJson(statementMetadata, HashMap.class); + } catch (Exception e) { + throw new RdfDAOException(e); + } + } + + public String toString(){ + return gson.toJson(metadataMap); + } + + public byte[] toBytes() { + // convert the map to a json string + String metadataString = gson.toJson(metadataMap); + // TODO may want to cache this for performance reasons + return metadataString.getBytes(); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/8fca552e/dao/accumulo.rya/src/test/java/org/apache/rya/accumulo/AccumuloRyaDAOTest.java ---------------------------------------------------------------------- diff --git a/dao/accumulo.rya/src/test/java/org/apache/rya/accumulo/AccumuloRyaDAOTest.java b/dao/accumulo.rya/src/test/java/org/apache/rya/accumulo/AccumuloRyaDAOTest.java index f170a7e..e22bdde 100644 --- a/dao/accumulo.rya/src/test/java/org/apache/rya/accumulo/AccumuloRyaDAOTest.java +++ b/dao/accumulo.rya/src/test/java/org/apache/rya/accumulo/AccumuloRyaDAOTest.java @@ -36,6 +36,7 @@ import org.apache.rya.accumulo.query.AccumuloRyaQueryEngine; import org.apache.rya.api.domain.RyaStatement; import org.apache.rya.api.domain.RyaType; import org.apache.rya.api.domain.RyaURI; +import org.apache.rya.api.domain.StatementMetadata; import org.apache.rya.api.persist.RyaDAOException; import org.apache.rya.api.persist.query.RyaQuery; import org.apache.rya.api.resolver.RdfToRyaConversions; @@ -117,9 +118,9 @@ public class AccumuloRyaDAOTest { RyaURI cpu = RdfToRyaConversions.convertURI(vf.createURI(litdupsNS, "cpu")); RyaURI loadPerc = RdfToRyaConversions.convertURI(vf.createURI(litdupsNS, "loadPerc")); RyaURI uri1 = RdfToRyaConversions.convertURI(vf.createURI(litdupsNS, "uri1")); - RyaStatement stmt1 = new RyaStatement(cpu, loadPerc, uri1, null, "1", "vis1".getBytes()); + RyaStatement stmt1 = new RyaStatement(cpu, loadPerc, uri1, null, "1", new StatementMetadata(), "vis1".getBytes()); dao.add(stmt1); - RyaStatement stmt2 = new RyaStatement(cpu, loadPerc, uri1, null, "2", "vis2".getBytes()); + RyaStatement stmt2 = new RyaStatement(cpu, loadPerc, uri1, null, "2", new StatementMetadata(), "vis2".getBytes()); dao.add(stmt2); AccumuloRdfConfiguration cloneConf = conf.clone(); @@ -253,7 +254,8 @@ public class AccumuloRyaDAOTest { RyaURI loadPerc = new RyaURI(litdupsNS + "loadPerc"); RyaURI uri1 = new RyaURI(litdupsNS + "uri1"); String myval = "myval"; - dao.add(new RyaStatement(cpu, loadPerc, uri1, null, null, null, myval.getBytes())); + byte[] columnVis = null; + dao.add(new RyaStatement(cpu, loadPerc, uri1, null, null, columnVis, myval.getBytes())); AccumuloRyaQueryEngine queryEngine = dao.getQueryEngine(); CloseableIteration<RyaStatement, RyaDAOException> iter = queryEngine.query(new RyaStatement(cpu, loadPerc, null), conf); @@ -272,9 +274,9 @@ public class AccumuloRyaDAOTest { byte[] colVisABC = "A|B|C".getBytes(); byte[] colVisAB = "A|B".getBytes(); byte[] colVisA = "A".getBytes(); - dao.add(new RyaStatement(cpu, loadPerc, uri1, null, null, colVisABC)); - dao.add(new RyaStatement(cpu, loadPerc, uri2, null, null, colVisAB)); - dao.add(new RyaStatement(cpu, loadPerc, uri3, null, null, colVisA)); + dao.add(new RyaStatement(cpu, loadPerc, uri1, null, null, new StatementMetadata(), colVisABC)); + dao.add(new RyaStatement(cpu, loadPerc, uri2, null, null, new StatementMetadata(), colVisAB)); + dao.add(new RyaStatement(cpu, loadPerc, uri3, null, null, new StatementMetadata(), colVisA)); AccumuloRyaQueryEngine queryEngine = dao.getQueryEngine(); http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/8fca552e/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/dao/SimpleMongoDBStorageStrategy.java ---------------------------------------------------------------------- diff --git a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/dao/SimpleMongoDBStorageStrategy.java b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/dao/SimpleMongoDBStorageStrategy.java index 258d37f..5024110 100644 --- a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/dao/SimpleMongoDBStorageStrategy.java +++ b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/dao/SimpleMongoDBStorageStrategy.java @@ -27,6 +27,11 @@ import java.util.Map; import org.apache.commons.codec.binary.Hex; import org.apache.log4j.Logger; +import org.apache.rya.api.domain.RyaStatement; +import org.apache.rya.api.domain.RyaType; +import org.apache.rya.api.domain.RyaURI; +import org.apache.rya.api.domain.StatementMetadata; +import org.apache.rya.api.persist.query.RyaQuery; import org.openrdf.model.impl.ValueFactoryImpl; import org.openrdf.model.vocabulary.XMLSchema; @@ -34,11 +39,6 @@ import com.mongodb.BasicDBObject; import com.mongodb.DBCollection; import com.mongodb.DBObject; -import org.apache.rya.api.domain.RyaStatement; -import org.apache.rya.api.domain.RyaType; -import org.apache.rya.api.domain.RyaURI; -import org.apache.rya.api.persist.query.RyaQuery; - /** * Defines how {@link RyaStatement}s are stored in MongoDB. */ @@ -46,12 +46,13 @@ public class SimpleMongoDBStorageStrategy implements MongoDBStorageStrategy<RyaS private static final Logger LOG = Logger.getLogger(SimpleMongoDBStorageStrategy.class); protected static final String ID = "_id"; protected static final String OBJECT_TYPE = "objectType"; - protected static final String OBJECT_TYPE_VALUE = XMLSchema.ANYURI.stringValue(); + protected static final String URI_TYPE_VALUE = XMLSchema.ANYURI.stringValue(); protected static final String CONTEXT = "context"; protected static final String PREDICATE = "predicate"; protected static final String OBJECT = "object"; protected static final String SUBJECT = "subject"; public static final String TIMESTAMP = "insertTimestamp"; + protected static final String STATEMENT_METADATA = "statementMetadata"; protected ValueFactoryImpl factory = new ValueFactoryImpl(); @Override @@ -102,6 +103,7 @@ public class SimpleMongoDBStorageStrategy implements MongoDBStorageStrategy<RyaS final String predicate = (String) result.get(PREDICATE); final String context = (String) result.get(CONTEXT); final Long timestamp = (Long) result.get(TIMESTAMP); + final String statementMetadata = (String) result.get(STATEMENT_METADATA); RyaType objectRya = null; if (objectType.equalsIgnoreCase(ANYURI.stringValue())){ objectRya = new RyaURI(object); @@ -121,6 +123,15 @@ public class SimpleMongoDBStorageStrategy implements MongoDBStorageStrategy<RyaS if(timestamp != null) { statement.setTimestamp(timestamp); } + if(statementMetadata != null) { + try { + StatementMetadata metadata = new StatementMetadata(statementMetadata); + statement.setStatementMetadata(metadata); + } + catch (Exception ex){ + LOG.debug("Error deserializing metadata for statement", ex); + } + } return statement; } @@ -143,12 +154,16 @@ public class SimpleMongoDBStorageStrategy implements MongoDBStorageStrategy<RyaS } catch (final NoSuchAlgorithmException e) { LOG.error("Unable to perform SHA-1 on the ID, defaulting to raw bytes.", e); } - final BasicDBObject doc = new BasicDBObject(ID, new String(Hex.encodeHex(bytes))) + if (statement.getMetadata() == null){ + statement.setStatementMetadata(StatementMetadata.EMPTY_METADATA); + } + BasicDBObject doc = new BasicDBObject(ID, new String(Hex.encodeHex(bytes))) .append(SUBJECT, statement.getSubject().getData()) .append(PREDICATE, statement.getPredicate().getData()) .append(OBJECT, statement.getObject().getData()) .append(OBJECT_TYPE, statement.getObject().getDataType().toString()) .append(CONTEXT, context) + .append(STATEMENT_METADATA, statement.getMetadata().toString()) .append(TIMESTAMP, statement.getTimestamp()); return doc; http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/8fca552e/sail/src/main/java/org/apache/rya/rdftriplestore/RdfCloudTripleStoreConnection.java ---------------------------------------------------------------------- diff --git a/sail/src/main/java/org/apache/rya/rdftriplestore/RdfCloudTripleStoreConnection.java b/sail/src/main/java/org/apache/rya/rdftriplestore/RdfCloudTripleStoreConnection.java index e8e370e..85440fd 100644 --- a/sail/src/main/java/org/apache/rya/rdftriplestore/RdfCloudTripleStoreConnection.java +++ b/sail/src/main/java/org/apache/rya/rdftriplestore/RdfCloudTripleStoreConnection.java @@ -36,6 +36,7 @@ import org.apache.rya.api.RdfCloudTripleStoreConfiguration; import org.apache.rya.api.RdfCloudTripleStoreConstants; import org.apache.rya.api.domain.RyaStatement; import org.apache.rya.api.domain.RyaURI; +import org.apache.rya.api.domain.StatementMetadata; import org.apache.rya.api.persist.RdfEvalStatsDAO; import org.apache.rya.api.persist.RyaDAO; import org.apache.rya.api.persist.RyaDAOException; @@ -154,7 +155,7 @@ public class RdfCloudTripleStoreConnection extends SailConnectionBase { RdfToRyaConversions.convertURI(predicate), RdfToRyaConversions.convertValue(object), RdfToRyaConversions.convertResource(context), - null, cv); + null, new StatementMetadata(), cv); ryaDAO.add(statement); } @@ -163,7 +164,7 @@ public class RdfCloudTripleStoreConnection extends SailConnectionBase { RdfToRyaConversions.convertResource(subject), RdfToRyaConversions.convertURI(predicate), RdfToRyaConversions.convertValue(object), - null, null, cv); + null, null, new StatementMetadata(), cv); ryaDAO.add(statement); }
