http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/c2a09dec/extras/indexing/src/main/java/mvm/rya/indexing/entity/storage/mongo/RyaTypeDocumentConverter.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/entity/storage/mongo/RyaTypeDocumentConverter.java b/extras/indexing/src/main/java/mvm/rya/indexing/entity/storage/mongo/RyaTypeDocumentConverter.java new file mode 100644 index 0000000..ab6148b --- /dev/null +++ b/extras/indexing/src/main/java/mvm/rya/indexing/entity/storage/mongo/RyaTypeDocumentConverter.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package mvm.rya.indexing.entity.storage.mongo; + +import static java.util.Objects.requireNonNull; + +import javax.annotation.ParametersAreNonnullByDefault; + +import org.bson.Document; +import org.openrdf.model.ValueFactory; +import org.openrdf.model.impl.ValueFactoryImpl; + +import mvm.rya.api.domain.RyaType; + +/** + * Converts between {@link RyaType} and {@link Document}. + */ +@ParametersAreNonnullByDefault +public class RyaTypeDocumentConverter implements DocumentConverter<RyaType> { + + private static final ValueFactory VF = new ValueFactoryImpl(); + + public static final String DATA_TYPE = "dataType"; + public static final String VALUE = "value"; + + @Override + public Document toDocument(RyaType ryaType) { + requireNonNull(ryaType); + + return new Document() + .append(DATA_TYPE, ryaType.getDataType().toString()) + .append(VALUE, ryaType.getData()); + } + + @Override + public RyaType fromDocument(Document document) throws DocumentConverterException { + requireNonNull(document); + + if(!document.containsKey(DATA_TYPE)) { + throw new DocumentConverterException("Could not convert document '" + document + + "' because its '" + DATA_TYPE + "' field is missing."); + } + + if(!document.containsKey(VALUE)) { + throw new DocumentConverterException("Could not convert document '" + document + + "' because its '" + VALUE + "' field is missing."); + } + + return new RyaType( + VF.createURI( document.getString(DATA_TYPE) ), + document.getString(VALUE)); + } +} \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/c2a09dec/extras/indexing/src/main/java/mvm/rya/indexing/entity/storage/mongo/TypeDocumentConverter.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/entity/storage/mongo/TypeDocumentConverter.java b/extras/indexing/src/main/java/mvm/rya/indexing/entity/storage/mongo/TypeDocumentConverter.java new file mode 100644 index 0000000..2d171d7 --- /dev/null +++ b/extras/indexing/src/main/java/mvm/rya/indexing/entity/storage/mongo/TypeDocumentConverter.java @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package mvm.rya.indexing.entity.storage.mongo; + +import static java.util.Objects.requireNonNull; + +import java.util.ArrayList; +import java.util.List; + +import javax.annotation.ParametersAreNonnullByDefault; + +import org.bson.Document; + +import com.google.common.collect.ImmutableSet; + +import mvm.rya.api.domain.RyaURI; +import mvm.rya.indexing.entity.model.Type; + +/** + * Converts between {@link Type} and {@link Document}. + */ +@ParametersAreNonnullByDefault +public class TypeDocumentConverter implements DocumentConverter<Type> { + + public static final String ID = "_id"; + public static final String PROPERTY_NAMES = "propertyNames"; + + @Override + public Document toDocument(final Type type) { + requireNonNull(type); + + final Document doc = new Document(); + doc.append(ID, type.getId().getData()); + + final List<String> propertyNames = new ArrayList<>(); + type.getPropertyNames().forEach(field -> propertyNames.add(field.getData())); + doc.append(PROPERTY_NAMES, propertyNames); + + return doc; + } + + @Override + public Type fromDocument(final Document document) throws DocumentConverterException { + requireNonNull(document); + + if(!document.containsKey(ID)) { + throw new DocumentConverterException("Could not convert document '" + document + + "' because its '" + ID + "' field is missing."); + } + + if(!document.containsKey(PROPERTY_NAMES)) { + throw new DocumentConverterException("Could not convert document '" + document + + "' because its '" + PROPERTY_NAMES + "' field is missing."); + } + + final RyaURI typeId = new RyaURI( document.getString(ID) ); + + final ImmutableSet.Builder<RyaURI> propertyNames = ImmutableSet.builder(); + ((List<String>) document.get(PROPERTY_NAMES)) + .forEach(propertyName -> propertyNames.add(new RyaURI(propertyName))); + + return new Type(typeId, propertyNames.build()); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/c2a09dec/extras/indexing/src/main/java/mvm/rya/indexing/entity/update/BaseEntityIndexer.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/entity/update/BaseEntityIndexer.java b/extras/indexing/src/main/java/mvm/rya/indexing/entity/update/BaseEntityIndexer.java new file mode 100644 index 0000000..627b879 --- /dev/null +++ b/extras/indexing/src/main/java/mvm/rya/indexing/entity/update/BaseEntityIndexer.java @@ -0,0 +1,266 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package mvm.rya.indexing.entity.update; + +import static com.google.common.base.Preconditions.checkState; +import static java.util.Collections.singleton; +import static java.util.Objects.requireNonNull; +import static java.util.stream.Collectors.groupingBy; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; + +import javax.annotation.Nullable; +import javax.annotation.ParametersAreNonnullByDefault; + +import org.apache.hadoop.conf.Configuration; +import org.openrdf.model.URI; +import org.openrdf.model.vocabulary.RDF; + +import com.google.common.base.Objects; + +import mvm.rya.api.domain.RyaStatement; +import mvm.rya.api.domain.RyaType; +import mvm.rya.api.domain.RyaURI; +import mvm.rya.indexing.entity.model.Entity; +import mvm.rya.indexing.entity.model.Property; +import mvm.rya.indexing.entity.model.Type; +import mvm.rya.indexing.entity.storage.CloseableIterator; +import mvm.rya.indexing.entity.storage.EntityStorage; +import mvm.rya.indexing.entity.storage.EntityStorage.EntityStorageException; +import mvm.rya.indexing.entity.storage.TypeStorage; +import mvm.rya.indexing.entity.storage.TypeStorage.TypeStorageException; +import mvm.rya.mongodb.MongoDBRdfConfiguration; + +/** + * A base class that may be used to update an {@link EntityStorage} as new + * {@link RyaStatement}s are added to/removed from the Rya instance. + */ +@ParametersAreNonnullByDefault +public abstract class BaseEntityIndexer implements EntityIndexer { + + /** + * When this URI is the Predicate of a Statement, it indicates a {@link Type} for an {@link Entity}. + */ + private static final RyaURI TYPE_URI = new RyaURI( RDF.TYPE.toString() ); + + private final AtomicReference<MongoDBRdfConfiguration> configuration = new AtomicReference<>(); + private final AtomicReference<EntityStorage> entities = new AtomicReference<>(); + private final AtomicReference<TypeStorage> types = new AtomicReference<>(); + + /** + * Creates the {@link EntityStorage} that will be used by the indexer. + * + * @param conf - Indicates how the {@link EntityStorage} is initialized. (not null) + * @return The {@link EntityStorage} that will be used by this indexer. + */ + public abstract @Nullable EntityStorage getEntityStorage(Configuration conf); + + /** + * Creates the {@link TypeStorage} that will be used by the indexer. + * + * @param conf - Indicates how the {@link TypeStorage} is initialized. (not null) + * @return The {@link TypeStorage} that will be used by this indexer. + */ + public abstract @Nullable TypeStorage getTypeStorage(Configuration conf); + + @Override + public void setConf(Configuration conf) { + requireNonNull(conf); + entities.set( getEntityStorage(conf) ); + types.set( getTypeStorage(conf) ); + } + + @Override + public Configuration getConf() { + return configuration.get(); + } + + @Override + public void storeStatement(RyaStatement statement) throws IOException { + requireNonNull(statement); + storeStatements( singleton(statement) ); + } + + @Override + public void storeStatements(Collection<RyaStatement> statements) throws IOException { + requireNonNull(statements); + + final Map<RyaURI,List<RyaStatement>> groupedBySubject = statements.stream() + .collect(groupingBy(RyaStatement::getSubject)); + + for(final Entry<RyaURI, List<RyaStatement>> entry : groupedBySubject.entrySet()) { + try { + updateEntity(entry.getKey(), entry.getValue()); + } catch (final EntityStorageException e) { + throw new IOException("Failed to update the Entity index.", e); + } + } + } + + /** + * Updates a {@link Entity} to reflect new {@link RyaStatement}s. + * + * @param subject - The Subject of the {@link Entity} the statements are for. (not null) + * @param statements - Statements that the {@link Entity} will be updated with. (not null) + */ + private void updateEntity(final RyaURI subject, final Collection<RyaStatement> statements) throws EntityStorageException { + requireNonNull(subject); + requireNonNull(statements); + + final EntityStorage entities = this.entities.get(); + final TypeStorage types = this.types.get(); + checkState(entities != null, "Must set this indexers configuration before storing statements."); + checkState(types != null, "Must set this indexers configuration before storing statements."); + + new EntityUpdater(entities).update(subject, old -> { + // Create a builder with the updated Version. + final Entity.Builder updated; + if(!old.isPresent()) { + updated = Entity.builder() + .setSubject(subject) + .setVersion(0); + } else { + final int updatedVersion = old.get().getVersion() + 1; + updated = Entity.builder(old.get()) + .setVersion( updatedVersion ); + } + + // Update the entity based on the Statements. + for(final RyaStatement statement : statements) { + + // The Statement is setting an Explicit Type ID for the Entity. + if(Objects.equal(TYPE_URI, statement.getPredicate())) { + final RyaURI typeId = new RyaURI(statement.getObject().getData()); + updated.setExplicitType(typeId); + } + + // The Statement is adding a Property to the Entity. + else { + final RyaURI propertyName = statement.getPredicate(); + final RyaType propertyValue = statement.getObject(); + + try(final CloseableIterator<Type> typesIt = types.search(propertyName)) { + // Set the Property for each type that includes the Statement's predicate. + while(typesIt.hasNext()) { + final RyaURI typeId = typesIt.next().getId(); + updated.setProperty(typeId, new Property(propertyName, propertyValue)); + } + } catch (final TypeStorageException | IOException e) { + throw new RuntimeException("Failed to fetch Types that include the property name '" + + statement.getPredicate().getData() + "'.", e); + } + } + } + + return Optional.of( updated.build() ); + }); + } + + @Override + public void deleteStatement(RyaStatement statement) throws IOException { + requireNonNull(statement); + + final EntityStorage entities = this.entities.get(); + final TypeStorage types = this.types.get(); + checkState(entities != null, "Must set this indexers configuration before storing statements."); + checkState(types != null, "Must set this indexers configuration before storing statements."); + + try { + new EntityUpdater(entities).update(statement.getSubject(), old -> { + // If there is no Entity for the subject of the statement, then do nothing. + if(!old.isPresent()) { + return Optional.empty(); + } + + final Entity oldEntity = old.get(); + + // Increment the version of the Entity. + final Entity.Builder updated = Entity.builder(oldEntity); + updated.setVersion(oldEntity.getVersion() + 1); + + if(TYPE_URI.equals(statement.getPredicate())) { + // If the Type ID already isn't in the list of explicit types, then do nothing. + final RyaURI typeId = new RyaURI( statement.getObject().getData() ); + if(!oldEntity.getExplicitTypeIds().contains(typeId)) { + return Optional.empty(); + } + + // Otherwise remove it from the list. + updated.unsetExplicitType(typeId); + } else { + // If the deleted property appears within the old entity's properties, then remove it. + final RyaURI deletedPropertyName = statement.getPredicate(); + + boolean propertyWasPresent = false; + for(final RyaURI typeId : oldEntity.getProperties().keySet()) { + for(final RyaURI propertyName : oldEntity.getProperties().get(typeId).keySet()) { + if(deletedPropertyName.equals(propertyName)) { + propertyWasPresent = true; + updated.unsetProperty(typeId, deletedPropertyName); + } + } + } + + // If no properties were removed, then do nothing. + if(!propertyWasPresent) { + return Optional.empty(); + } + } + + return Optional.of( updated.build() ); + }); + } catch (final EntityStorageException e) { + throw new IOException("Failed to update the Entity index.", e); + } + } + + @Override + public String getTableName() { + // Storage details have been abstracted away from the indexer. + return null; + } + + @Override + public void flush() throws IOException { + // We do not need to do anything to flush since we do not batch work. + } + + @Override + public void close() throws IOException { + // Nothing to close. + } + + @Override + public void dropGraph(RyaURI... graphs) { + // We do not support graphs when performing entity centric indexing. + } + + @Override + public Set<URI> getIndexablePredicates() { + // This isn't used anywhere in Rya, so it will not be implemented. + return null; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/c2a09dec/extras/indexing/src/main/java/mvm/rya/indexing/entity/update/EntityIndexer.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/entity/update/EntityIndexer.java b/extras/indexing/src/main/java/mvm/rya/indexing/entity/update/EntityIndexer.java new file mode 100644 index 0000000..6d74836 --- /dev/null +++ b/extras/indexing/src/main/java/mvm/rya/indexing/entity/update/EntityIndexer.java @@ -0,0 +1,31 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package mvm.rya.indexing.entity.update; + +import mvm.rya.api.domain.RyaStatement; +import mvm.rya.api.persist.index.RyaSecondaryIndexer; +import mvm.rya.indexing.entity.storage.EntityStorage; + +/** + * Updates the {@link Entity}s that are in a {@link EntityStorage} when new + * {@link RyaStatement}s are added/removed from the Rya instance. + */ +public interface EntityIndexer extends RyaSecondaryIndexer { + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/c2a09dec/extras/indexing/src/main/java/mvm/rya/indexing/entity/update/EntityUpdater.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/entity/update/EntityUpdater.java b/extras/indexing/src/main/java/mvm/rya/indexing/entity/update/EntityUpdater.java new file mode 100644 index 0000000..6f883d4 --- /dev/null +++ b/extras/indexing/src/main/java/mvm/rya/indexing/entity/update/EntityUpdater.java @@ -0,0 +1,121 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package mvm.rya.indexing.entity.update; + +import static java.util.Objects.requireNonNull; + +import java.util.Optional; +import java.util.function.Function; + +import javax.annotation.ParametersAreNonnullByDefault; + +import mvm.rya.api.domain.RyaURI; +import mvm.rya.indexing.entity.model.Entity; +import mvm.rya.indexing.entity.storage.EntityStorage; +import mvm.rya.indexing.entity.storage.EntityStorage.EntityAlreadyExistsException; +import mvm.rya.indexing.entity.storage.EntityStorage.EntityStorageException; +import mvm.rya.indexing.entity.storage.EntityStorage.StaleUpdateException; + +/** + * Performs update operations over an {@link EntityStorage}. + */ +@ParametersAreNonnullByDefault +public class EntityUpdater { + + private final EntityStorage storage; + + /** + * Constructs an instance of {@link EntityUpdater}. + * + * @param storage - The storage this updater operates over. (not null) + */ + public EntityUpdater(EntityStorage storage) { + this.storage = requireNonNull(storage); + } + + /** + * Tries to updates the state of an {@link Entity} until the update succeeds + * or a non-recoverable exception is thrown. + * + * @param subject - The Subject of the {@link Entity} that will be updated. (not null) + * @param mutator - Performs the mutation on the old state of the Entity and returns + * the new state of the Entity. (not null) + * @throws EntityStorageException A non-recoverable error has caused the update to fail. + */ + public void update(RyaURI subject, EntityMutator mutator) throws EntityStorageException { + requireNonNull(subject); + requireNonNull(mutator); + + // Fetch the current state of the Entity. + boolean completed = false; + while(!completed) { + try { + final Optional<Entity> old = storage.get(subject); + final Optional<Entity> updated = mutator.apply(old); + + final boolean doWork = updated.isPresent(); + if(doWork) { + if(!old.isPresent()) { + storage.create(updated.get()); + } else { + storage.update(old.get(), updated.get()); + } + } + completed = true; + } catch(final EntityAlreadyExistsException | StaleUpdateException e) { + // These are recoverable exceptions. Try again. + } catch(final RuntimeException e) { + throw new EntityStorageException("Failed to update Entity with Subject '" + subject.getData() + "'.", e); + } + } + } + + /** + * Implementations of this interface are used to update the state of an + * {@link Entity} in unison with a {@link EntityUpdater}. + * </p> + * This table describes what the updater will do depending on if an Entity + * exists and if an updated Entity is returned. + * </p> + * <table border="1px"> + * <tr><th>Entity Provided</th><th>Update Returned</th><th>Effect</th></tr> + * <tr> + * <td>true</td> + * <td>true</td> + * <td>The old Entity will be updated using the returned state.</td> + * </tr> + * <tr> + * <td>true</td> + * <td>false</td> + * <td>No work is performed.</td> + * </tr> + * <tr> + * <td>false</td> + * <td>true</td> + * <td>A new Entity will be created using the returned state.</td> + * </tr> + * <tr> + * <td>false</td> + * <td>false</td> + * <td>No work is performed.</td> + * </tr> + * </table> + */ + public interface EntityMutator extends Function<Optional<Entity>, Optional<Entity>> { } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/c2a09dec/extras/indexing/src/main/java/mvm/rya/indexing/entity/update/mongo/MongoEntityIndexer.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/entity/update/mongo/MongoEntityIndexer.java b/extras/indexing/src/main/java/mvm/rya/indexing/entity/update/mongo/MongoEntityIndexer.java new file mode 100644 index 0000000..94550ac --- /dev/null +++ b/extras/indexing/src/main/java/mvm/rya/indexing/entity/update/mongo/MongoEntityIndexer.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package mvm.rya.indexing.entity.update.mongo; + +import javax.annotation.ParametersAreNonnullByDefault; + +import org.apache.hadoop.conf.Configuration; + +import com.mongodb.MongoClient; + +import mvm.rya.indexing.entity.storage.EntityStorage; +import mvm.rya.indexing.entity.storage.TypeStorage; +import mvm.rya.indexing.entity.storage.mongo.MongoEntityStorage; +import mvm.rya.indexing.entity.storage.mongo.MongoTypeStorage; +import mvm.rya.indexing.entity.update.BaseEntityIndexer; +import mvm.rya.indexing.entity.update.EntityIndexer; +import mvm.rya.mongodb.MongoConnectorFactory; +import mvm.rya.mongodb.MongoDBRdfConfiguration; + +/** + * A Mongo DB implementation of {@link EntityIndexer}. + */ +@ParametersAreNonnullByDefault +public class MongoEntityIndexer extends BaseEntityIndexer { + + @Override + public EntityStorage getEntityStorage(Configuration conf) { + final MongoClient mongoClient = MongoConnectorFactory.getMongoClient(conf); + final String ryaInstanceName = new MongoDBRdfConfiguration(conf).getMongoDBName(); + return new MongoEntityStorage(mongoClient, ryaInstanceName); + } + + @Override + public TypeStorage getTypeStorage(Configuration conf) { + final MongoClient mongoClient = MongoConnectorFactory.getMongoClient(conf); + final String ryaInstanceName = new MongoDBRdfConfiguration(conf).getMongoDBName(); + return new MongoTypeStorage(mongoClient, ryaInstanceName); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/c2a09dec/extras/indexing/src/test/java/mvm/rya/indexing/entity/query/EntityQueryNodeTest.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/test/java/mvm/rya/indexing/entity/query/EntityQueryNodeTest.java b/extras/indexing/src/test/java/mvm/rya/indexing/entity/query/EntityQueryNodeTest.java new file mode 100644 index 0000000..ab263e0 --- /dev/null +++ b/extras/indexing/src/test/java/mvm/rya/indexing/entity/query/EntityQueryNodeTest.java @@ -0,0 +1,194 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package mvm.rya.indexing.entity.query; + +import static org.mockito.Mockito.mock; + +import java.util.List; + +import org.junit.Test; +import org.openrdf.model.vocabulary.RDF; +import org.openrdf.query.MalformedQueryException; +import org.openrdf.query.algebra.StatementPattern; +import org.openrdf.query.algebra.helpers.StatementPatternCollector; +import org.openrdf.query.parser.sparql.SPARQLParser; + +import com.google.common.collect.ImmutableSet; + +import mvm.rya.api.domain.RyaURI; +import mvm.rya.indexing.entity.model.Type; +import mvm.rya.indexing.entity.storage.EntityStorage; + +/** + * Unit tests the methods of {@link EntityQueryNode}. + */ +public class EntityQueryNodeTest { + + private static final Type PERSON_TYPE = + new Type(new RyaURI("urn:person"), + ImmutableSet.<RyaURI>builder() + .add(new RyaURI("urn:name")) + .add(new RyaURI("urn:age")) + .add(new RyaURI("urn:eye")) + .build()); + + private static final Type EMPLOYEE_TYPE = + new Type(new RyaURI("urn:employee"), + ImmutableSet.<RyaURI>builder() + .add(new RyaURI("urn:name")) + .add(new RyaURI("urn:hoursPerWeek")) + .build()); + + @Test(expected = IllegalStateException.class) + public void constructor_differentSubjects() throws Exception { + // A pattern that has two different subjects. + final List<StatementPattern> patterns = getSPs( + "SELECT * WHERE { " + + "<urn:SSN:111-11-1111> <" + RDF.TYPE + "> <urn:person> ."+ + "<urn:SSN:111-11-1111> <urn:age> ?age . " + + "<urn:SSN:111-11-1111> <urn:eye> ?eye . " + + "<urn:SSN:111-11-1111> <urn:name> ?name . " + + "<urn:SSN:222-22-2222> <urn:age> ?age . " + + "<urn:SSN:222-22-2222> <urn:eye> ?eye . " + + "<urn:SSN:222-22-2222> <urn:name> ?name . " + + "}"); + + + // This will fail. + new EntityQueryNode(PERSON_TYPE, patterns, mock(EntityStorage.class)); + } + + @Test(expected = IllegalStateException.class) + public void constructor_variablePredicate() throws Exception { + // A pattern that has a variable for its predicate. + final List<StatementPattern> patterns = getSPs( + "SELECT * WHERE { " + + "?subject <" + RDF.TYPE + "> <urn:person> ."+ + "?subject ?variableProperty ?value . " + + "?subject <urn:eye> ?eye . " + + "?subject <urn:name> ?name . " + + "}"); + + + // This will fail. + new EntityQueryNode(PERSON_TYPE, patterns, mock(EntityStorage.class)); + } + + @Test(expected = IllegalStateException.class) + public void constructor_predicateNotPartOfType() throws Exception { + // A pattern that does uses a predicate that is not part of the type. + final List<StatementPattern> patterns = getSPs( + "SELECT * WHERE { " + + "?subject <" + RDF.TYPE + "> <urn:person> ."+ + "?subject <urn:age> ?age . " + + "?subject <urn:eye> ?eye . " + + "?subject <urn:name> ?name . " + + "?subject <urn:notPartOfType> ?value . " + + "}"); + + // This will fail. + new EntityQueryNode(PERSON_TYPE, patterns, mock(EntityStorage.class)); + } + + @Test(expected = IllegalStateException.class) + public void constructor_typeMissing() throws Exception { + // A pattern that does uses a predicate that is not part of the type. + final List<StatementPattern> patterns = getSPs( + "SELECT * WHERE { " + + "?subject <urn:age> ?age . " + + "?subject <urn:eye> ?eye . " + + "?subject <urn:name> ?name . " + + "}"); + + // This will fail. + new EntityQueryNode(PERSON_TYPE, patterns, mock(EntityStorage.class)); + } + + @Test(expected = IllegalStateException.class) + public void constructor_wrongType() throws Exception { + // A pattern that does uses a predicate that is not part of the type. + final List<StatementPattern> patterns = getSPs( + "SELECT * WHERE { " + + "?subject <" + RDF.TYPE + "> <urn:person> ."+ + "?subject <urn:age> ?age . " + + "?subject <urn:eye> ?eye . " + + "?subject <urn:name> ?name . " + + "}"); + + // This will fail. + new EntityQueryNode(EMPLOYEE_TYPE, patterns, mock(EntityStorage.class)); + } + + // Happy path test. + + // TODO test for all of the types of preconditions + // test when a binding set can join + // test when a binding set does not join + // test when there are constants that are part of the query + // test when there are variables that are part of the query. + + @Test + public void evaluate_constantSubject() throws Exception { + // A set of patterns that match a sepecific Entity subject. + final List<StatementPattern> patterns = getSPs( + "SELECT * WHERE { " + + "<urn:SSN:111-11-1111> <" + RDF.TYPE + "> <urn:person> ."+ + "<urn:SSN:111-11-1111> <urn:age> ?age . " + + "<urn:SSN:111-11-1111> <urn:eye> ?eye . " + + "<urn:SSN:111-11-1111> <urn:name> ?name . " + + "}"); + + new EntityQueryNode(PERSON_TYPE, patterns, mock(EntityStorage.class)); + + + // TODO implement + } + + @Test + public void evaluate_variableSubject() throws Exception { + // A set of patterns that matches a variable Entity subject. + final List<StatementPattern> patterns = getSPs( + "SELECT * WHERE { " + + "?subject <" + RDF.TYPE + "> <urn:person> ."+ + "?subject <urn:age> ?age . " + + "?subject <urn:eye> ?eye . " + + "?subject <urn:name> ?name . " + + "}"); + + new EntityQueryNode(PERSON_TYPE, patterns, mock(EntityStorage.class)); + + + // TODO implement + } + + + + /** + * TODO doc + * + * @param sparql + * @return + * @throws MalformedQueryException + */ + private static List<StatementPattern> getSPs(String sparql) throws MalformedQueryException { + final StatementPatternCollector spCollector = new StatementPatternCollector(); + new SPARQLParser().parseQuery(sparql, null).getTupleExpr().visit(spCollector); + return spCollector.getStatementPatterns(); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/c2a09dec/extras/indexing/src/test/java/mvm/rya/indexing/entity/storage/mongo/EntityDocumentConverterTest.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/test/java/mvm/rya/indexing/entity/storage/mongo/EntityDocumentConverterTest.java b/extras/indexing/src/test/java/mvm/rya/indexing/entity/storage/mongo/EntityDocumentConverterTest.java new file mode 100644 index 0000000..bbd3f06 --- /dev/null +++ b/extras/indexing/src/test/java/mvm/rya/indexing/entity/storage/mongo/EntityDocumentConverterTest.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package mvm.rya.indexing.entity.storage.mongo; + +import static org.junit.Assert.assertEquals; + +import org.bson.Document; +import org.junit.Test; +import org.openrdf.model.vocabulary.XMLSchema; + +import mvm.rya.api.domain.RyaType; +import mvm.rya.api.domain.RyaURI; +import mvm.rya.indexing.entity.model.Entity; +import mvm.rya.indexing.entity.model.Property; +import mvm.rya.indexing.entity.storage.mongo.DocumentConverter.DocumentConverterException; + +/** + * Tests the methods of {@link EntityDocumentConverter}. + */ +public class EntityDocumentConverterTest { + + @Test + public void to_and_from_document() throws DocumentConverterException { + // Convert an Entity into a Document. + final Entity entity = Entity.builder() + .setSubject(new RyaURI("urn:alice")) + // Add some explicily typed properties. + .setExplicitType(new RyaURI("urn:person")) + .setProperty(new RyaURI("urn:person"), new Property(new RyaURI("urn:age"), new RyaType(XMLSchema.INT, "30"))) + .setProperty(new RyaURI("urn:person"), new Property(new RyaURI("urn:eye"), new RyaType("blue"))) + // Add some implicitly typed properties. + .setProperty(new RyaURI("urn:employee"), new Property(new RyaURI("urn:hours"), new RyaType(XMLSchema.INT, "40"))) + .setProperty(new RyaURI("urn:employee"), new Property(new RyaURI("urn:employer"), new RyaType("Burger Joint"))) + .build(); + + final Document document = new EntityDocumentConverter().toDocument(entity); + + // Convert the Document back into an Entity. + final Entity converted = new EntityDocumentConverter().fromDocument(document); + + // Ensure the original matches the round trip converted Entity. + assertEquals(entity, converted); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/c2a09dec/extras/indexing/src/test/java/mvm/rya/indexing/entity/storage/mongo/MongoEntityStorageIT.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/test/java/mvm/rya/indexing/entity/storage/mongo/MongoEntityStorageIT.java b/extras/indexing/src/test/java/mvm/rya/indexing/entity/storage/mongo/MongoEntityStorageIT.java new file mode 100644 index 0000000..cc821c3 --- /dev/null +++ b/extras/indexing/src/test/java/mvm/rya/indexing/entity/storage/mongo/MongoEntityStorageIT.java @@ -0,0 +1,391 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package mvm.rya.indexing.entity.storage.mongo; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; + +import org.junit.Test; +import org.openrdf.model.vocabulary.XMLSchema; + +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; + +import mvm.rya.api.domain.RyaType; +import mvm.rya.api.domain.RyaURI; +import mvm.rya.indexing.entity.model.Entity; +import mvm.rya.indexing.entity.model.Property; +import mvm.rya.indexing.entity.model.Type; +import mvm.rya.indexing.entity.model.TypedEntity; +import mvm.rya.indexing.entity.storage.CloseableIterator; +import mvm.rya.indexing.entity.storage.EntityStorage; +import mvm.rya.indexing.entity.storage.EntityStorage.EntityAlreadyExistsException; +import mvm.rya.indexing.entity.storage.EntityStorage.EntityStorageException; +import mvm.rya.indexing.entity.storage.EntityStorage.StaleUpdateException; + +/** + * Integration tests the methods of {@link MongoEntityStorage}. + */ +public class MongoEntityStorageIT extends MongoITBase { + + private static final String RYA_INSTANCE_NAME = "testInstance"; + + @Test + public void create_and_get() throws EntityStorageException { + // An Entity that will be stored. + final Entity entity = Entity.builder() + .setSubject(new RyaURI("urn:GTIN-14/00012345600012")) + .setExplicitType(new RyaURI("urn:icecream")) + .setProperty(new RyaURI("urn:icecream"), new Property(new RyaURI("urn:brand"), new RyaType(XMLSchema.STRING, "Awesome Icecream"))) + .setProperty(new RyaURI("urn:icecream"), new Property(new RyaURI("urn:flavor"), new RyaType(XMLSchema.STRING, "Chocolate"))) + .build(); + + // Create it. + final EntityStorage storage = new MongoEntityStorage(super.getMongoClient(), RYA_INSTANCE_NAME); + storage.create(entity); + + // Get it. + final Optional<Entity> storedEntity = storage.get(new RyaURI("urn:GTIN-14/00012345600012")); + + // Verify the correct value was returned. + assertEquals(entity, storedEntity.get()); + } + + @Test + public void can_not_create_with_same_subject() throws EntityStorageException { + // A Type that will be stored. + final Entity entity = Entity.builder() + .setSubject(new RyaURI("urn:GTIN-14/00012345600012")) + .setExplicitType(new RyaURI("urn:icecream")) + .setProperty(new RyaURI("urn:icecream"), new Property(new RyaURI("urn:brand"), new RyaType(XMLSchema.STRING, "Awesome Icecream"))) + .setProperty(new RyaURI("urn:icecream"), new Property(new RyaURI("urn:flavor"), new RyaType(XMLSchema.STRING, "Chocolate"))) + .build(); + + // Create it. + final EntityStorage storage = new MongoEntityStorage(super.getMongoClient(), RYA_INSTANCE_NAME); + storage.create(entity); + + // Try to create it again. This will fail. + boolean failed = false; + try { + storage.create(entity); + } catch(final EntityAlreadyExistsException e) { + failed = true; + } + assertTrue(failed); + } + + @Test + public void get_noneExisting() throws EntityStorageException { + // Get a Type that hasn't been created. + final EntityStorage storage = new MongoEntityStorage(super.getMongoClient(), RYA_INSTANCE_NAME); + final Optional<Entity> storedEntity = storage.get(new RyaURI("urn:GTIN-14/00012345600012")); + + // Verify nothing was returned. + assertFalse(storedEntity.isPresent()); + } + + @Test + public void delete() throws EntityStorageException { + // An Entity that will be stored. + final Entity entity = Entity.builder() + .setSubject(new RyaURI("urn:GTIN-14/00012345600012")) + .setExplicitType(new RyaURI("urn:icecream")) + .setProperty(new RyaURI("urn:icecream"), new Property(new RyaURI("urn:brand"), new RyaType(XMLSchema.STRING, "Awesome Icecream"))) + .setProperty(new RyaURI("urn:icecream"), new Property(new RyaURI("urn:flavor"), new RyaType(XMLSchema.STRING, "Chocolate"))) + .build(); + + // Create it. + final EntityStorage storage = new MongoEntityStorage(super.getMongoClient(), RYA_INSTANCE_NAME); + storage.create(entity); + + // Delete it. + final boolean deleted = storage.delete( new RyaURI("urn:GTIN-14/00012345600012") ); + + // Verify a document was deleted. + assertTrue( deleted ); + } + + @Test + public void delete_nonExisting() throws EntityStorageException { + // Delete an Entity that has not been created. + final EntityStorage storage = new MongoEntityStorage(super.getMongoClient(), RYA_INSTANCE_NAME); + final boolean deleted = storage.delete( new RyaURI("urn:GTIN-14/00012345600012") ); + + // Verify no document was deleted. + assertFalse( deleted ); + } + + @Test + public void search_byDataType() throws Exception { + final EntityStorage storage = new MongoEntityStorage(super.getMongoClient(), RYA_INSTANCE_NAME); + + // The Type we will search by. + final Type icecreamType = new Type(new RyaURI("urn:icecream"), + ImmutableSet.<RyaURI>builder() + .add(new RyaURI("urn:brand")) + .add(new RyaURI("urn:flavor")) + .add(new RyaURI("urn:cost")) + .build()); + + // Some Person typed entities. + final Entity alice = Entity.builder() + .setSubject( new RyaURI("urn:SSN/111-11-1111") ) + .setExplicitType(new RyaURI("urn:person")) + .setProperty(new RyaURI("urn:person"), new Property(new RyaURI("urn:name"), new RyaType(XMLSchema.STRING, "Alice"))) + .setProperty(new RyaURI("urn:person"), new Property(new RyaURI("urn:age"), new RyaType(XMLSchema.INT, "30"))) + .setProperty(new RyaURI("urn:person"), new Property(new RyaURI("urn:eye"), new RyaType(XMLSchema.STRING, "blue"))) + .build(); + + final Entity bob = Entity.builder() + .setSubject( new RyaURI("urn:SSN/222-22-2222") ) + .setExplicitType(new RyaURI("urn:person")) + .setProperty(new RyaURI("urn:person"), new Property(new RyaURI("urn:name"), new RyaType(XMLSchema.STRING, "Bob"))) + .setProperty(new RyaURI("urn:person"), new Property(new RyaURI("urn:age"), new RyaType(XMLSchema.INT, "57"))) + .setProperty(new RyaURI("urn:person"), new Property(new RyaURI("urn:eye"), new RyaType(XMLSchema.STRING, "blue"))) + .build(); + + // Some Icecream typed objects. + final Entity chocolateIcecream = Entity.builder() + .setSubject(new RyaURI("urn:GTIN-14/00012345600012")) + .setExplicitType(new RyaURI("urn:icecream")) + .setProperty(new RyaURI("urn:icecream"), new Property(new RyaURI("urn:brand"), new RyaType(XMLSchema.STRING, "Awesome Icecream"))) + .setProperty(new RyaURI("urn:icecream"), new Property(new RyaURI("urn:flavor"), new RyaType(XMLSchema.STRING, "Chocolate"))) + .build(); + + final Entity vanillaIcecream = Entity.builder() + .setSubject( new RyaURI("urn:GTIN-14/22356325213432") ) + .setExplicitType(new RyaURI("urn:icecream")) + .setProperty(new RyaURI("urn:icecream"), new Property(new RyaURI("urn:brand"), new RyaType(XMLSchema.STRING, "Awesome Icecream"))) + .setProperty(new RyaURI("urn:icecream"), new Property(new RyaURI("urn:flavor"), new RyaType(XMLSchema.STRING, "Vanilla"))) + .build(); + + + final Entity strawberryIcecream = Entity.builder() + .setSubject( new RyaURI("urn:GTIN-14/77544325436721") ) + .setProperty(new RyaURI("urn:icecream"), new Property(new RyaURI("urn:brand"), new RyaType(XMLSchema.STRING, "Awesome Icecream"))) + .setProperty(new RyaURI("urn:icecream"), new Property(new RyaURI("urn:flavor"), new RyaType(XMLSchema.STRING, "Strawberry"))) + .build(); + + // Create the objects in the storage. + storage.create(alice); + storage.create(bob); + storage.create(chocolateIcecream); + storage.create(vanillaIcecream); + storage.create(strawberryIcecream); + + // Search for all icecreams. + final Set<TypedEntity> objects = new HashSet<>(); + try(final CloseableIterator<TypedEntity> it = storage.search(icecreamType, new HashSet<>())) { + while(it.hasNext()) { + objects.add(it.next()); + } + } + + // Verify the expected results were returned. + final Set<TypedEntity> expected = Sets.newHashSet( + chocolateIcecream.makeTypedEntity(new RyaURI("urn:icecream")).get(), + vanillaIcecream.makeTypedEntity(new RyaURI("urn:icecream")).get()); + assertEquals(expected, objects); + } + + @Test + public void search_byFields() throws Exception { + final EntityStorage storage = new MongoEntityStorage(super.getMongoClient(), RYA_INSTANCE_NAME); + + // A Type that defines a Person. + final Type personType = new Type(new RyaURI("urn:person"), + ImmutableSet.<RyaURI>builder() + .add(new RyaURI("urn:name")) + .add(new RyaURI("urn:age")) + .add(new RyaURI("urn:eye")) + .build()); + + // Some Person typed objects. + final Entity alice = Entity.builder() + .setSubject( new RyaURI("urn:SSN/111-11-1111") ) + .setExplicitType(new RyaURI("urn:person")) + .setProperty(new RyaURI("urn:person"), new Property(new RyaURI("urn:name"), new RyaType(XMLSchema.STRING, "Alice"))) + .setProperty(new RyaURI("urn:person"), new Property(new RyaURI("urn:age"), new RyaType(XMLSchema.INT, "30"))) + .setProperty(new RyaURI("urn:person"), new Property(new RyaURI("urn:eye"), new RyaType(XMLSchema.STRING, "blue"))) + .build(); + + final Entity bob = Entity.builder() + .setSubject( new RyaURI("urn:SSN/222-22-2222") ) + .setExplicitType(new RyaURI("urn:person")) + .setProperty(new RyaURI("urn:person"), new Property(new RyaURI("urn:name"), new RyaType(XMLSchema.STRING, "Bob"))) + .setProperty(new RyaURI("urn:person"), new Property(new RyaURI("urn:age"), new RyaType(XMLSchema.INT, "57"))) + .setProperty(new RyaURI("urn:person"), new Property(new RyaURI("urn:eye"), new RyaType(XMLSchema.STRING, "blue"))) + .build(); + + + final Entity charlie = Entity.builder() + .setSubject( new RyaURI("urn:SSN/333-33-3333") ) + .setExplicitType( new RyaURI("urn:person") ) + .setProperty(new RyaURI("urn:person"), new Property(new RyaURI("urn:name"), new RyaType(XMLSchema.STRING, "Charlie"))) + .setProperty(new RyaURI("urn:person"), new Property(new RyaURI("urn:age"), new RyaType(XMLSchema.INT, "30"))) + .setProperty(new RyaURI("urn:person"), new Property(new RyaURI("urn:eye"), new RyaType(XMLSchema.STRING, "blue"))) + .build(); + + final Entity david = Entity.builder() + .setSubject( new RyaURI("urn:SSN/444-44-4444") ) + .setExplicitType( new RyaURI("urn:person") ) + .setProperty(new RyaURI("urn:person"), new Property(new RyaURI("urn:name"), new RyaType(XMLSchema.STRING, "David"))) + .setProperty(new RyaURI("urn:person"), new Property(new RyaURI("urn:age"), new RyaType(XMLSchema.INT, "30"))) + .setProperty(new RyaURI("urn:person"), new Property(new RyaURI("urn:eye"), new RyaType(XMLSchema.STRING, "brown"))) + .build(); + + final Entity eve = Entity.builder() + .setSubject( new RyaURI("urn:SSN/555-55-5555") ) + .setExplicitType( new RyaURI("urn:person") ) + .setProperty(new RyaURI("urn:person"), new Property(new RyaURI("urn:name"), new RyaType(XMLSchema.STRING, "Eve"))) + .setProperty(new RyaURI("urn:person"), new Property(new RyaURI("urn:age"), new RyaType(XMLSchema.INT, "30"))) + .build(); + + final Entity frank = Entity.builder() + .setSubject( new RyaURI("urn:SSN/666-66-6666") ) + .setExplicitType( new RyaURI("urn:person") ) + .setProperty(new RyaURI("urn:person"), new Property(new RyaURI("urn:name"), new RyaType(XMLSchema.STRING, "Frank"))) + .setProperty(new RyaURI("urn:person"), new Property(new RyaURI("urn:eye"), new RyaType(XMLSchema.STRING, "blue"))) + .setProperty(new RyaURI("urn:someOtherType"), new Property(new RyaURI("urn:age"), new RyaType(XMLSchema.INT, "30"))) + .build(); + + final Entity george = Entity.builder() + .setSubject( new RyaURI("urn:SSN/777-77-7777") ) + .setProperty(new RyaURI("urn:person"), new Property(new RyaURI("urn:name"), new RyaType(XMLSchema.STRING, "George"))) + .setProperty(new RyaURI("urn:person"), new Property(new RyaURI("urn:age"), new RyaType(XMLSchema.INT, "30"))) + .setProperty(new RyaURI("urn:person"), new Property(new RyaURI("urn:eye"), new RyaType(XMLSchema.STRING, "blue"))) + .build(); + + // Create the objects in the storage. + storage.create(alice); + storage.create(bob); + storage.create(charlie); + storage.create(david); + storage.create(eve); + storage.create(frank); + storage.create(george); + + // Search for all people who are 30 and have blue eyes. + final Set<TypedEntity> objects = new HashSet<>(); + + final Set<Property> searchValues = Sets.newHashSet( + new Property(new RyaURI("urn:eye"), new RyaType(XMLSchema.STRING, "blue")), + new Property(new RyaURI("urn:age"), new RyaType(XMLSchema.INT, "30"))); + + try(final CloseableIterator<TypedEntity> it = storage.search(personType, searchValues)) { + while(it.hasNext()) { + objects.add(it.next()); + } + } + + // Verify the expected results were returned. + assertEquals(2, objects.size()); + assertTrue(objects.contains(alice.makeTypedEntity(new RyaURI("urn:person")).get())); + assertTrue(objects.contains(charlie.makeTypedEntity(new RyaURI("urn:person")).get())); + } + + @Test + public void update() throws EntityStorageException { + final EntityStorage storage = new MongoEntityStorage(super.getMongoClient(), RYA_INSTANCE_NAME); + + // Store Alice in the repository. + final Entity alice = Entity.builder() + .setSubject( new RyaURI("urn:SSN/111-11-1111") ) + .setExplicitType(new RyaURI("urn:person")) + .setProperty(new RyaURI("urn:person"), new Property(new RyaURI("urn:name"), new RyaType(XMLSchema.STRING, "Alice"))) + .setProperty(new RyaURI("urn:person"), new Property(new RyaURI("urn:age"), new RyaType(XMLSchema.INT, "30"))) + .setProperty(new RyaURI("urn:person"), new Property(new RyaURI("urn:eye"), new RyaType(XMLSchema.STRING, "blue"))) + .build(); + + storage.create(alice); + + // Show Alice was stored. + Optional<Entity> latest = storage.get(new RyaURI("urn:SSN/111-11-1111")); + assertEquals(alice, latest.get()); + + // Change Alice's eye color to brown. + final Entity updated = Entity.builder(alice) + .setVersion(latest.get().getVersion() + 1) + .setProperty(new RyaURI("urn:person"), new Property(new RyaURI("urn:eye"), new RyaType(XMLSchema.STRING, "brown"))) + .build(); + + storage.update(alice, updated); + + // Fetch the Alice object and ensure it has the new value. + latest = storage.get(new RyaURI("urn:SSN/111-11-1111")); + + assertEquals(updated, latest.get()); + } + + @Test(expected = StaleUpdateException.class) + public void update_stale() throws EntityStorageException { + final EntityStorage storage = new MongoEntityStorage(super.getMongoClient(), RYA_INSTANCE_NAME); + + // Store Alice in the repository. + final Entity alice = Entity.builder() + .setSubject( new RyaURI("urn:SSN/111-11-1111") ) + .setExplicitType(new RyaURI("urn:person")) + .setProperty(new RyaURI("urn:person"), new Property(new RyaURI("urn:name"), new RyaType(XMLSchema.STRING, "Alice"))) + .setProperty(new RyaURI("urn:person"), new Property(new RyaURI("urn:age"), new RyaType(XMLSchema.INT, "30"))) + .setProperty(new RyaURI("urn:person"), new Property(new RyaURI("urn:eye"), new RyaType(XMLSchema.STRING, "blue"))) + .build(); + + storage.create(alice); + + // Show Alice was stored. + final Optional<Entity> latest = storage.get(new RyaURI("urn:SSN/111-11-1111")); + assertEquals(alice, latest.get()); + + // Create the wrong old state and try to change Alice's eye color to brown. + final Entity wrongOld = Entity.builder(alice) + .setVersion(500) + .build(); + + final Entity updated = Entity.builder(alice) + .setVersion(501) + .setProperty(new RyaURI("urn:person"), new Property(new RyaURI("urn:eye"), new RyaType(XMLSchema.STRING, "brown"))) + .build(); + + storage.update(wrongOld, updated); + } + + @Test(expected = EntityStorageException.class) + public void update_differentSubjects() throws StaleUpdateException, EntityStorageException { + // Two objects that do not have the same Subjects. + final Entity old = Entity.builder() + .setSubject( new RyaURI("urn:SSN/111-11-1111") ) + .setExplicitType( new RyaURI("urn:person") ) + .build(); + + final Entity updated = Entity.builder() + .setSubject( new RyaURI("urn:SSN/222-22-2222") ) + .setExplicitType( new RyaURI("urn:person") ) + .build(); + + // The update will fail. + final EntityStorage storage = new MongoEntityStorage(super.getMongoClient(), RYA_INSTANCE_NAME); + storage.update(old, updated); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/c2a09dec/extras/indexing/src/test/java/mvm/rya/indexing/entity/storage/mongo/MongoITBase.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/test/java/mvm/rya/indexing/entity/storage/mongo/MongoITBase.java b/extras/indexing/src/test/java/mvm/rya/indexing/entity/storage/mongo/MongoITBase.java new file mode 100644 index 0000000..760a816 --- /dev/null +++ b/extras/indexing/src/test/java/mvm/rya/indexing/entity/storage/mongo/MongoITBase.java @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package mvm.rya.indexing.entity.storage.mongo; + +import java.net.UnknownHostException; +import java.util.HashSet; +import java.util.Set; + +import org.apache.hadoop.conf.Configuration; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; + +import com.mongodb.MongoClient; +import com.mongodb.MongoException; + +import mvm.rya.mongodb.MongoConnectorFactory; +import mvm.rya.mongodb.MongoDBRdfConfiguration; + +/** + * A base class that may be used when implementing Mongo DB integration tests that + * use the JUnit framework. + */ +public class MongoITBase { + + private MongoClient mongoClient = null; + private Set<String> originalDbNames = null; + + @Before + public void setupTest() throws UnknownHostException, MongoException { + final MongoDBRdfConfiguration conf = new MongoDBRdfConfiguration( new Configuration() ); + conf.setUseTestMongo(true); + conf.setMongoDBName("testDB"); + + mongoClient = MongoConnectorFactory.getMongoClient(conf); + + // Store the names of the DBs that are present before running the test. + originalDbNames = new HashSet<>(); + for(final String name : mongoClient.listDatabaseNames()) { + originalDbNames.add(name); + } + } + + @After + public void cleanupTest() { + // Remove any DBs that were created by the test. + for(final String dbName : mongoClient.listDatabaseNames()) { + if(!originalDbNames.contains(dbName)) { + mongoClient.dropDatabase(dbName); + } + } + } + + @AfterClass + public static void shutdown() { + MongoConnectorFactory.shutdown(); + } + + /** + * @return A {@link MongoClient} that is connected to the embedded instance of Mongo DB. + */ + public MongoClient getMongoClient() { + return mongoClient; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/c2a09dec/extras/indexing/src/test/java/mvm/rya/indexing/entity/storage/mongo/MongoTypeStorageIT.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/test/java/mvm/rya/indexing/entity/storage/mongo/MongoTypeStorageIT.java b/extras/indexing/src/test/java/mvm/rya/indexing/entity/storage/mongo/MongoTypeStorageIT.java new file mode 100644 index 0000000..6809247 --- /dev/null +++ b/extras/indexing/src/test/java/mvm/rya/indexing/entity/storage/mongo/MongoTypeStorageIT.java @@ -0,0 +1,174 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package mvm.rya.indexing.entity.storage.mongo; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.HashSet; +import java.util.Set; + +import org.junit.Test; + +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; + +import mvm.rya.api.domain.RyaURI; +import mvm.rya.indexing.entity.model.Type; +import mvm.rya.indexing.entity.storage.CloseableIterator; +import mvm.rya.indexing.entity.storage.TypeStorage; +import mvm.rya.indexing.entity.storage.TypeStorage.TypeStorageException; + +/** + * Integration tests the methods of {@link MongoTypeStorage}. + */ +public class MongoTypeStorageIT extends MongoITBase { + + private static final String RYA_INSTANCE_NAME = "testInstance"; + + @Test + public void create_and_get() throws TypeStorageException { + // A Type that will be stored. + final Type type = new Type(new RyaURI("urn:icecream"), + ImmutableSet.<RyaURI>builder() + .add(new RyaURI("urn:brand")) + .add(new RyaURI("urn:flavor")) + .add(new RyaURI("urn:cost")) + .build()); + + // Create it. + final TypeStorage storage = new MongoTypeStorage(super.getMongoClient(), RYA_INSTANCE_NAME); + storage.create(type); + + // Get it. + final Optional<Type> storedType = storage.get(new RyaURI("urn:icecream")); + + // Verify the correct value was returned. + assertEquals(type, storedType.get()); + } + + @Test + public void can_not_create_with_same_id() throws TypeStorageException { + // A Type that will be stored. + final Type type = new Type(new RyaURI("urn:icecream"), + ImmutableSet.<RyaURI>builder() + .add(new RyaURI("urn:brand")) + .add(new RyaURI("urn:flavor")) + .add(new RyaURI("urn:cost")) + .build()); + + // Create it. + final TypeStorage storage = new MongoTypeStorage(super.getMongoClient(), RYA_INSTANCE_NAME); + storage.create(type); + + // Try to create it again. This will fail. + boolean failed = false; + try { + storage.create(type); + } catch(final TypeStorageException e) { + failed = true; + } + assertTrue(failed); + } + + @Test + public void get_nonexisting() throws TypeStorageException { + // Get a Type that hasn't been created. + final TypeStorage storage = new MongoTypeStorage(super.getMongoClient(), RYA_INSTANCE_NAME); + final Optional<Type> storedType = storage.get(new RyaURI("urn:icecream")); + + // Verify nothing was returned. + assertFalse(storedType.isPresent()); + } + + @Test + public void delete() throws TypeStorageException { + // An Type that will be stored. + final Type type = new Type(new RyaURI("urn:icecream"), + ImmutableSet.<RyaURI>builder() + .add(new RyaURI("urn:brand")) + .add(new RyaURI("urn:flavor")) + .add(new RyaURI("urn:cost")) + .build()); + + // Create it. + final TypeStorage storage = new MongoTypeStorage(super.getMongoClient(), RYA_INSTANCE_NAME); + storage.create(type); + + // Delete it. + final boolean deleted = storage.delete( new RyaURI("urn:icecream") ); + + // Verify a document was deleted. + assertTrue( deleted ); + } + + @Test + public void delete_nonexisting() throws TypeStorageException { + // Delete an Type that has not been created. + final TypeStorage storage = new MongoTypeStorage(super.getMongoClient(), RYA_INSTANCE_NAME); + final boolean deleted = storage.delete( new RyaURI("urn:icecream") ); + + // Verify no document was deleted. + assertFalse( deleted ); + } + + @Test + public void search() throws Exception { + // Add some Types to the storage. + final Type cat = new Type(new RyaURI("urn:cat"), + ImmutableSet.<RyaURI>builder() + .add(new RyaURI("urn:numLegs")) + .add(new RyaURI("urn:eye")) + .add(new RyaURI("urn:species")) + .build()); + + final Type dog = new Type(new RyaURI("urn:dog"), + ImmutableSet.<RyaURI>builder() + .add(new RyaURI("urn:numLegs")) + .add(new RyaURI("urn:eye")) + .add(new RyaURI("urn:species")) + .build()); + + final Type icecream = new Type(new RyaURI("urn:icecream"), + ImmutableSet.<RyaURI>builder() + .add(new RyaURI("urn:brand")) + .add(new RyaURI("urn:flavor")) + .add(new RyaURI("urn:cost")) + .build()); + + final TypeStorage storage = new MongoTypeStorage(super.getMongoClient(), RYA_INSTANCE_NAME); + storage.create(cat); + storage.create(dog); + storage.create(icecream); + + // Search for all Types that have the 'urn:eye' property. + final CloseableIterator<Type> typeIt = storage.search(new RyaURI("urn:eye")); + + final Set<Type> types = new HashSet<>(); + while(typeIt.hasNext()) { + types.add( typeIt.next() ); + } + + // Verify the correct types were returned. + final Set<Type> expected = Sets.newHashSet(cat, dog); + assertEquals(expected, types); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/c2a09dec/extras/indexing/src/test/java/mvm/rya/indexing/entity/storage/mongo/RyaTypeDocumentConverterTest.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/test/java/mvm/rya/indexing/entity/storage/mongo/RyaTypeDocumentConverterTest.java b/extras/indexing/src/test/java/mvm/rya/indexing/entity/storage/mongo/RyaTypeDocumentConverterTest.java new file mode 100644 index 0000000..0626009 --- /dev/null +++ b/extras/indexing/src/test/java/mvm/rya/indexing/entity/storage/mongo/RyaTypeDocumentConverterTest.java @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package mvm.rya.indexing.entity.storage.mongo; + +import static org.junit.Assert.assertEquals; + +import org.bson.Document; +import org.junit.Test; +import org.openrdf.model.impl.ValueFactoryImpl; +import org.openrdf.model.vocabulary.XMLSchema; + +import mvm.rya.api.domain.RyaType; +import mvm.rya.api.resolver.RdfToRyaConversions; +import mvm.rya.indexing.entity.storage.mongo.DocumentConverter.DocumentConverterException; + +/** + * Tests the methods of {@link RyaTypeDocumentConverter}. + */ +public class RyaTypeDocumentConverterTest { + + @Test + public void toDocument() { + // Convert the RyaType into a Document. + final RyaType ryaType = RdfToRyaConversions.convertLiteral( new ValueFactoryImpl().createLiteral( 4.5 ) ); + final Document document = new RyaTypeDocumentConverter().toDocument( ryaType ); + + // Show the document has the correct structure. + final Document expected = new Document() + .append(RyaTypeDocumentConverter.DATA_TYPE, XMLSchema.DOUBLE.toString()) + .append(RyaTypeDocumentConverter.VALUE, "4.5"); + assertEquals(expected, document); + } + + @Test + public void fromDocument() throws DocumentConverterException { + // Convert a document into a RyaType + final Document document = new Document() + .append(RyaTypeDocumentConverter.DATA_TYPE, XMLSchema.DOUBLE.toString()) + .append(RyaTypeDocumentConverter.VALUE, "4.5"); + final RyaType ryaType = new RyaTypeDocumentConverter().fromDocument( document ); + + // Show the converted value has the expected structure. + final RyaType expected = RdfToRyaConversions.convertLiteral( new ValueFactoryImpl().createLiteral( 4.5 ) ); + assertEquals(expected, ryaType); + } + + @Test(expected = DocumentConverterException.class) + public void fromDocument_noDataType() throws DocumentConverterException { + // A document that does not have a data type. + final Document document = new Document() + .append(RyaTypeDocumentConverter.VALUE, "4.5"); + + // The conversion will fail. + new RyaTypeDocumentConverter().fromDocument(document); + } + + @Test(expected = DocumentConverterException.class) + public void fromDocument_noValue() throws DocumentConverterException { + // A document that does not have a value. + final Document document = new Document() + .append(RyaTypeDocumentConverter.DATA_TYPE, XMLSchema.DOUBLE.toString()); + + // The conversion will fail. + new RyaTypeDocumentConverter().fromDocument(document); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/c2a09dec/extras/indexing/src/test/java/mvm/rya/indexing/entity/storage/mongo/TypeDocumentConverterTest.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/test/java/mvm/rya/indexing/entity/storage/mongo/TypeDocumentConverterTest.java b/extras/indexing/src/test/java/mvm/rya/indexing/entity/storage/mongo/TypeDocumentConverterTest.java new file mode 100644 index 0000000..8b5bc7f --- /dev/null +++ b/extras/indexing/src/test/java/mvm/rya/indexing/entity/storage/mongo/TypeDocumentConverterTest.java @@ -0,0 +1,93 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package mvm.rya.indexing.entity.storage.mongo; + +import static org.junit.Assert.assertEquals; + +import org.bson.Document; +import org.junit.Test; + +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Lists; + +import mvm.rya.api.domain.RyaURI; +import mvm.rya.indexing.entity.model.Type; +import mvm.rya.indexing.entity.storage.mongo.DocumentConverter.DocumentConverterException; + +/** + * Tests the methods of {@link TypeDocumentConverter}. + */ +public class TypeDocumentConverterTest { + + @Test + public void toDocument() { + // Convert a Type into a Document. + final Type type = new Type(new RyaURI("urn:icecream"), + ImmutableSet.<RyaURI>builder() + .add(new RyaURI("urn:brand")) + .add(new RyaURI("urn:flavor")) + .add(new RyaURI("urn:cost")) + .build()); + final Document document = new TypeDocumentConverter().toDocument(type); + + // Show the document has the correct values. + final Document expected = new Document() + .append(TypeDocumentConverter.ID, "urn:icecream") + .append(TypeDocumentConverter.PROPERTY_NAMES, Lists.newArrayList("urn:brand", "urn:flavor", "urn:cost")); + assertEquals(expected, document); + } + + @Test + public void fromDocument() throws DocumentConverterException { + // Convert a Document into a Type. + final Document document = new Document() + .append(TypeDocumentConverter.ID, "urn:icecream") + .append(TypeDocumentConverter.PROPERTY_NAMES, Lists.newArrayList("urn:brand", "urn:flavor", "urn:cost")); + final Type type = new TypeDocumentConverter().fromDocument(document); + + // Show the converted value has the expected structure. + final Type expected = new Type(new RyaURI("urn:icecream"), + ImmutableSet.<RyaURI>builder() + .add(new RyaURI("urn:brand")) + .add(new RyaURI("urn:flavor")) + .add(new RyaURI("urn:cost")) + .build()); + assertEquals(expected, type); + } + + @Test(expected = DocumentConverterException.class) + public void fromDocument_noId() throws DocumentConverterException { + // A document that does not have a Data Type ID. + final Document document = new Document() + .append(TypeDocumentConverter.PROPERTY_NAMES, Lists.newArrayList("urn:brand", "urn:flavor", "urn:cost")); + + // The conversion will fail. + new TypeDocumentConverter().fromDocument(document); + } + + @Test(expected = DocumentConverterException.class) + public void fromDocument_noOptionalFieldNames() throws DocumentConverterException { + // A document that does not have an Optional Field Names. + final Document document = new Document() + .append(TypeDocumentConverter.ID, "urn:icecream"); + + // The conversion will fail. + new TypeDocumentConverter().fromDocument(document); + } +} \ No newline at end of file
