RYA-250 Added data duplication detection methods to Smart URI/Entities. These use configured tolerances for each data type to decide if an Entity is considered nearly equal. Also, string terms that are considered equivalent can be configured. Closes #153.
Project: http://git-wip-us.apache.org/repos/asf/incubator-rya/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-rya/commit/b319365e Tree: http://git-wip-us.apache.org/repos/asf/incubator-rya/tree/b319365e Diff: http://git-wip-us.apache.org/repos/asf/incubator-rya/diff/b319365e Branch: refs/heads/master Commit: b319365e8c06fe5f19c6ada611f596be2695930c Parents: 7cd70bf Author: eric.white <eric.wh...@parsons.com> Authored: Wed Apr 12 11:15:03 2017 -0400 Committer: Caleb Meier <caleb.me...@parsons.com> Committed: Mon Aug 21 13:39:38 2017 -0700 ---------------------------------------------------------------------- .../org/apache/rya/api/domain/RyaTypeUtils.java | 54 +- extras/indexing/README.md | 212 ++ .../conf/duplicate_data_detection_config.xml | 80 + .../indexing/entity/EntityIndexOptimizer.java | 7 +- .../rya/indexing/entity/model/Entity.java | 2 +- .../storage/mongo/MongoEntityStorage.java | 131 +- .../entity/update/BaseEntityIndexer.java | 9 +- .../indexing/entity/update/EntityIndexer.java | 4 +- .../entity/update/mongo/MongoEntityIndexer.java | 3 +- .../rya/indexing/mongodb/MongoDbSmartUri.java | 5 +- .../rya/indexing/smarturi/SmartUriAdapter.java | 88 +- .../indexing/smarturi/SmartUriException.java | 8 + .../duplication/ApproxEqualsDetector.java | 78 + .../duplication/DuplicateDataDetector.java | 1066 +++++++++ .../EntityNearDuplicateException.java | 47 + .../smarturi/duplication/Tolerance.java | 67 + .../smarturi/duplication/ToleranceType.java | 50 + .../duplication/conf/DuplicateDataConfig.java | 337 +++ .../rya/indexing/mongo/MongoDbSmartUriTest.java | 1 - .../duplication/DuplicateDataDetectorTest.java | 2053 ++++++++++++++++++ 20 files changed, 4210 insertions(+), 92 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/b319365e/common/rya.api/src/main/java/org/apache/rya/api/domain/RyaTypeUtils.java ---------------------------------------------------------------------- diff --git a/common/rya.api/src/main/java/org/apache/rya/api/domain/RyaTypeUtils.java b/common/rya.api/src/main/java/org/apache/rya/api/domain/RyaTypeUtils.java index 8ca65fc..6f9902e 100644 --- a/common/rya.api/src/main/java/org/apache/rya/api/domain/RyaTypeUtils.java +++ b/common/rya.api/src/main/java/org/apache/rya/api/domain/RyaTypeUtils.java @@ -24,12 +24,44 @@ import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.joda.time.format.ISODateTimeFormat; import org.openrdf.model.URI; +import org.openrdf.model.impl.URIImpl; import org.openrdf.model.vocabulary.XMLSchema; +import com.google.common.collect.ImmutableMap; + /** * Utility methods for using {@link RyaType}. */ public final class RyaTypeUtils { + private static final ImmutableMap<Class<?>, RyaTypeMethod> METHOD_MAP = + ImmutableMap.<Class<?>, RyaTypeMethod>builder() + .put(Boolean.class, (v) -> booleanRyaType((Boolean) v)) + .put(Byte.class, (v) -> byteRyaType((Byte) v)) + .put(Date.class, (v) -> dateRyaType((Date) v)) + .put(DateTime.class, (v) -> dateRyaType((DateTime) v)) + .put(Double.class, (v) -> doubleRyaType((Double) v)) + .put(Float.class, (v) -> floatRyaType((Float) v)) + .put(Integer.class, (v) -> intRyaType((Integer) v)) + .put(Long.class, (v) -> longRyaType((Long) v)) + .put(Short.class, (v) -> shortRyaType((Short) v)) + .put(String.class, (v) -> stringRyaType((String) v)) + .put(URI.class, (v) -> uriRyaType((URI) v)) + .put(URIImpl.class, (v) -> uriRyaType((URIImpl) v)) + .build(); + + /** + * Represents a method inside the {@link RyaTypeUtils} class that can be + * called. + */ + private static interface RyaTypeMethod { + /** + * Calls the method within {@link RyaTypeUtils} with the supplied value. + * @param value the object value. + * @return the {@link RyaType}. + */ + public RyaType callRyaTypeMethod(final Object value); + } + /** * Private constructor to prevent instantiation. */ @@ -66,7 +98,10 @@ public final class RyaTypeUtils { */ public static RyaType dateRyaType(final Date value) { final DateTime dateTime = new DateTime(value.getTime()); - return dateRyaType(dateTime); + final StringBuffer sb = new StringBuffer(); + ISODateTimeFormat.dateTime().withZone(DateTimeZone.UTC).printTo(sb, dateTime.getMillis()); + final String formattedDate = sb.toString(); + return new RyaType(XMLSchema.DATE, formattedDate); } /** @@ -154,4 +189,21 @@ public final class RyaTypeUtils { public static RyaType uriRyaType(final URI value) { return new RyaType(XMLSchema.ANYURI, value.stringValue()); } + + /** + * Calls the appropriate {@link RyaTypeUtils} method based on the class + * specified and initializes it with the supplied value. + * @param classType the {@link Class} of {@link RyaType} to find. + * @param value the value to initialize the {@link RyaType} with. + * @return the {@link RyaType} or {@code null} if none could be found for + * the specified {@code classType}. + */ + public static RyaType getRyaTypeForClass(final Class<?> classType, final Object value) { + final RyaTypeMethod method = METHOD_MAP.get(classType); + RyaType ryaType = null; + if (method != null) { + ryaType = method.callRyaTypeMethod(value); + } + return ryaType; + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/b319365e/extras/indexing/README.md ---------------------------------------------------------------------- diff --git a/extras/indexing/README.md b/extras/indexing/README.md new file mode 100644 index 0000000..a2f7497 --- /dev/null +++ b/extras/indexing/README.md @@ -0,0 +1,212 @@ +<!-- Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. --> + +# Rya Indexing + +___ + +This project contains implementations of Rya's Indexing components. + +Rya Indexing supports the following datastores: + + * Accumulo + + * MongoDB + +## Entity Indexing +An Entity is a named concept that has at least one defined structure +and a bunch of values that fit within each of those structures. A structure is +defined by a Type. A value that fits within that Type is a Property. +</p> +For example, suppose we want to represent a type of icecream as an Entity. +First we must define what properties an icecream entity may have: +``` + Type ID: <urn:icecream> + Properties: <urn:brand> + <urn:flavor> + <urn:ingredients> + <urn:nutritionalInformation> +``` +Now we can represent our icecream whose brand is "Awesome Icecream" and whose +flavor is "Chocolate", but has no ingredients or nutritional information, as +an Entity by doing the following: +``` +final Entity entity = Entity.builder() + .setSubject(new RyaURI("urn:GTIN-14/00012345600012")) + .setExplicitType(new RyaURI("urn:icecream")) + .setProperty(new RyaURI("urn:icecream"), new Property(new RyaURI("urn:brand"), new RyaType(XMLSchema.STRING, "Awesome Icecream"))) + .setProperty(new RyaURI("urn:icecream"), new Property(new RyaURI("urn:flavor"), new RyaType(XMLSchema.STRING, "Chocolate"))) + .build(); +``` +The two types of Entities that may be created are implicit and explicit. +An implicit Entity is one who has at least one Property that matches +the Type, but nothing has explicitly indicated it is of that Type. +Once something has done so, it is an explicitly typed Entity. + +### Smart URI +A Smart URI is an indentifier of a specific Entity with all its data fields and associated values. Smart URI's are only currently supported by MongoDB instances of Rya. + +The Smart URI format: + +[subject URI]?ryaTypes=[encoded map of type URIs to type names]&[type1.propertyName1]=[value1]&[type1.propertyName2]=[value2]&[type2.propertyName1]=[value3] + +The "subject URI" can be any URI. The Entities' properties are expressed as query parameters in the Smart URI. Since the Smart URI may represent an Entity of multiple Types, the "ryaTypes" query parameter is used to keep track of which property belongs to which Type. The Type's mapped short name is appended to each property name and each name/value pair has their characters escaped properly to produce a valid URI. This means all foreign character and special characters will will be encoded so that they conform to [RFC-3986](https://www.ietf.org/rfc/rfc3986.txt). + +Using the "icecream" Type Entity example from the "Entity Indexing" section above, the Smart URI representation would be: + +`urn://GTIN-14/00012345600012?ryaTypes=urn%3A%2F%2FentityTypeMap%3Furn%253Aicecream%3Dicecream&icecream.brand=Awesome+Icecream&icecream.flavor=Chocolate` + +As an example of an Entity with multiple Types, lets consider the Entity as also being part of the following "dessert" type: +```xml + Type ID: <urn:dessert> + Properties: <urn:storageNeeded> + <urn:utensilUsed> +``` + +The Smart URI representation would be: + +`urn://GTIN-14/00012345600012?ryaTypes=urn%3A%2F%2FentityTypeMap%3Furn%253Aicecream%3Dicecream%26urn%253Adessert%3Ddessert&dessert.storageNeeded=Freezer&dessert.utensilUsed=Spoon&icecream.brand=Awesome+Icecream&icecream.flavor=Chocolate` + +#### Smart URI Entity Duplication Detection + +In some cases, data that is close enough together to be considered nearly identical should be treated as duplicate Entities. Duplicate data detection can be enabled so that newly found Entities that appear close enough to existing Entities should not be created. + +##### Configuring Duplication Detection + +This sections discusses how to configure the various options of Smart URI Entity Duplication Detection. To edit Duplication Detection, create or modify the `duplicate_data_detection_config.xml` in the `conf` directory. + +It should look similar to this: +```xml +<duplicateDataDetectionConfiguration> + <enableDetection>true</enableDetection> + <tolerances> + <booleanTolerance> + <value>0</value> + <type>DIFFERENCE</type> + </booleanTolerance> + <byteTolerance> + <value>0</value> + <type>DIFFERENCE</type> + </byteTolerance> + <dateTolerance> + <value>500</value> + <type>DIFFERENCE</type> + </dateTolerance> + <doubleTolerance> + <value>0.01%</value> + <type>PERCENTAGE</type> + </doubleTolerance> + <floatTolerance> + <value>0.01%</value> + <type>PERCENTAGE</type> + </floatTolerance> + <integerTolerance> + <value>1</value> + <type>DIFFERENCE</type> + </integerTolerance> + <longTolerance> + <value>1</value> + <type>DIFFERENCE</type> + </longTolerance> + <shortTolerance> + <value>1</value> + <type>DIFFERENCE</type> + </shortTolerance> + <stringTolerance> + <value>1</value> + <type>DIFFERENCE</type> + </stringTolerance> + <uriTolerance> + <value>1</value> + <type>DIFFERENCE</type> + </uriTolerance> + </tolerances> + <termMappings> + <termMapping> + <term>example</term> + <equivalents> + <equivalent>sample</equivalent> + <equivalent>case</equivalent> + </equivalents> + </termMapping> + <termMapping> + <term>test</term> + <equivalents> + <equivalent>exam</equivalent> + <equivalent>quiz</equivalent> + </equivalents> + </termMapping> + </termMappings> +</duplicateDataDetectionConfiguration> +``` + +###### Enabling/Disabling Duplication Detection + +To enable detection, set `<enableDetection>` to "true". Setting to "false" will disable it. + +###### Tolerance Type Configuration + +Each data type can have a tolerance type set for it (either "PERCENTAGE" or "DIFFERENCE"). If "DIFFERENCE" is selected then the data types are considered duplicates if they are within the `<value>` specified. The `<value>` must be positive. So, the two integer values 50000 and 50001 are considered duplicates when configured like this: +```xml + <integerTolerance> + <value>1</value> + <type>DIFFERENCE</type> + </integerTolerance> +``` + +If "PERCENTAGE" is selected then the values must be within a certain percent to one another to be considered nearly identical duplicates. This is useful when dealing with really small numbers or really large numbers where an exact absolute difference is hard to determine to consider them nearly equal. The `<value>` can be expressed in percent or decimal form. So, one percent can be entered as "1%" or "0.01" in the `<value>` field. The `<value>` must be positive and cannot be greater than 100%. + +If a Type's tolerance is not specified it defers to a default value. +The default values are: + +| **Class** | **Tolerance Type** | **Value** | +|-----------|--------------------|-----------| +| boolean | DIFFERENCE | 0 | +| byte | DIFFERENCE | 0 | +| date | DIFFERENCE | 500 (ms) | +| double | PERCENTAGE | 0.01% | +| float | PERCENTAGE | 0.01% | +| integer | DIFFERENCE | 1 | +| long | DIFFERENCE | 1 | +| short | DIFFERENCE | 1 | +| string | PERCENTAGE | 5% | +| uri | DIFFERENCE | 1 | + +###### Equivalent Terms Configuration + +Words that one wants to consider equivalent can be inserted into a map under the `<termMappings>` part of the configuration file. An example `<termMapping>` entry is shown below: + +```xml + <termMapping> + <term>example</term> + <equivalents> + <equivalent>sample</equivalent> + <equivalent>case</equivalent> + </equivalents> + </termMapping> +``` + +This `<termMapping>` means that a new Entity with a field value of "sample" or "case" would be equivalent to an existing field value of "example" but this relationship is only one way. If a field value of "case" already exists then a new field value "example" is not equivalent. The relationship can be made bidirectional by adding another `<termMapping>`. + +```xml + <termMapping> + <term>case</term> + <equivalents> + <equivalent>example</equivalent> + </equivalents> + </termMapping> +``` http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/b319365e/extras/indexing/conf/duplicate_data_detection_config.xml ---------------------------------------------------------------------- diff --git a/extras/indexing/conf/duplicate_data_detection_config.xml b/extras/indexing/conf/duplicate_data_detection_config.xml new file mode 100644 index 0000000..7855ca7 --- /dev/null +++ b/extras/indexing/conf/duplicate_data_detection_config.xml @@ -0,0 +1,80 @@ +<?xml version="1.0" encoding="UTF-8" standalone="yes"?> +<!-- Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. --> +<duplicateDataDetectionConfiguration> + <enableDetection>true</enableDetection> + <tolerances> + <booleanTolerance> + <value>0</value> + <type>DIFFERENCE</type> + </booleanTolerance> + <byteTolerance> + <value>0</value> + <type>DIFFERENCE</type> + </byteTolerance> + <dateTolerance> + <value>500</value> + <type>DIFFERENCE</type> + </dateTolerance> + <doubleTolerance> + <value>0.01%</value> + <type>PERCENTAGE</type> + </doubleTolerance> + <floatTolerance> + <value>0.01%</value> + <type>PERCENTAGE</type> + </floatTolerance> + <integerTolerance> + <value>1</value> + <type>DIFFERENCE</type> + </integerTolerance> + <longTolerance> + <value>1</value> + <type>DIFFERENCE</type> + </longTolerance> + <shortTolerance> + <value>1</value> + <type>DIFFERENCE</type> + </shortTolerance> + <stringTolerance> + <value>1</value> + <type>DIFFERENCE</type> + </stringTolerance> + <uriTolerance> + <value>1</value> + <type>DIFFERENCE</type> + </uriTolerance> + </tolerances> + <!-- Uncomment to define equivalent terms + <termMappings> + <termMapping> + <term>example</term> + <equivalents> + <equivalent>sample</equivalent> + <equivalent>case</equivalent> + </equivalents> + </termMapping> + <termMapping> + <term>test</term> + <equivalents> + <equivalent>exam</equivalent> + <equivalent>quiz</equivalent> + </equivalents> + </termMapping> + </termMappings> + --> +</duplicateDataDetectionConfiguration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/b319365e/extras/indexing/src/main/java/org/apache/rya/indexing/entity/EntityIndexOptimizer.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/org/apache/rya/indexing/entity/EntityIndexOptimizer.java b/extras/indexing/src/main/java/org/apache/rya/indexing/entity/EntityIndexOptimizer.java index 5a45dc8..cd5278e 100644 --- a/extras/indexing/src/main/java/org/apache/rya/indexing/entity/EntityIndexOptimizer.java +++ b/extras/indexing/src/main/java/org/apache/rya/indexing/entity/EntityIndexOptimizer.java @@ -26,6 +26,7 @@ import org.apache.log4j.Logger; import org.apache.rya.indexing.entity.model.Entity; import org.apache.rya.indexing.entity.query.EntityQueryNode; import org.apache.rya.indexing.entity.storage.EntityStorage; +import org.apache.rya.indexing.entity.storage.EntityStorage.EntityStorageException; import org.apache.rya.indexing.entity.storage.TypeStorage; import org.apache.rya.indexing.entity.update.mongo.MongoEntityIndexer; import org.apache.rya.indexing.external.matching.AbstractExternalSetOptimizer; @@ -71,7 +72,11 @@ public class EntityIndexOptimizer extends AbstractExternalSetOptimizer<EntityQue indexer.setConf(conf); typeStorage = indexer.getTypeStorage(conf); - entityStorage = indexer.getEntityStorage(conf); + try { + entityStorage = indexer.getEntityStorage(conf); + } catch (final EntityStorageException e) { + log.error("Error getting entity storage", e); + } provider = new EntityIndexSetProvider(typeStorage, entityStorage); } http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/b319365e/extras/indexing/src/main/java/org/apache/rya/indexing/entity/model/Entity.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/org/apache/rya/indexing/entity/model/Entity.java b/extras/indexing/src/main/java/org/apache/rya/indexing/entity/model/Entity.java index a90e469..3804de4 100644 --- a/extras/indexing/src/main/java/org/apache/rya/indexing/entity/model/Entity.java +++ b/extras/indexing/src/main/java/org/apache/rya/indexing/entity/model/Entity.java @@ -208,7 +208,7 @@ public class Entity { final ImmutableMap<RyaURI, Property> typePropertyMap = properties.get(typeRyaUri); Optional<Property> property = Optional.empty(); if (typePropertyMap != null) { - property = Optional.of(typePropertyMap.get(propertyRyaUri)); + property = Optional.ofNullable(typePropertyMap.get(propertyRyaUri)); } return property; } http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/b319365e/extras/indexing/src/main/java/org/apache/rya/indexing/entity/storage/mongo/MongoEntityStorage.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/org/apache/rya/indexing/entity/storage/mongo/MongoEntityStorage.java b/extras/indexing/src/main/java/org/apache/rya/indexing/entity/storage/mongo/MongoEntityStorage.java index a71d673..87634d7 100644 --- a/extras/indexing/src/main/java/org/apache/rya/indexing/entity/storage/mongo/MongoEntityStorage.java +++ b/extras/indexing/src/main/java/org/apache/rya/indexing/entity/storage/mongo/MongoEntityStorage.java @@ -20,25 +20,35 @@ package org.apache.rya.indexing.entity.storage.mongo; import static java.util.Objects.requireNonNull; +import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; +import java.util.List; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; +import org.apache.commons.configuration.ConfigurationException; +import org.apache.log4j.Logger; import org.apache.rya.api.domain.RyaURI; import org.apache.rya.indexing.entity.model.Entity; import org.apache.rya.indexing.entity.model.Property; import org.apache.rya.indexing.entity.model.Type; import org.apache.rya.indexing.entity.model.TypedEntity; import org.apache.rya.indexing.entity.storage.EntityStorage; +import org.apache.rya.indexing.entity.storage.TypeStorage.TypeStorageException; import org.apache.rya.indexing.entity.storage.mongo.ConvertingCursor.Converter; import org.apache.rya.indexing.entity.storage.mongo.DocumentConverter.DocumentConverterException; import org.apache.rya.indexing.entity.storage.mongo.key.MongoDbSafeKey; +import org.apache.rya.indexing.smarturi.SmartUriException; +import org.apache.rya.indexing.smarturi.duplication.DuplicateDataDetector; +import org.apache.rya.indexing.smarturi.duplication.EntityNearDuplicateException; import org.bson.Document; import org.bson.conversions.Bson; import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; import com.mongodb.ErrorCategory; import com.mongodb.MongoClient; import com.mongodb.MongoException; @@ -54,6 +64,7 @@ import edu.umd.cs.findbugs.annotations.NonNull; */ @DefaultAnnotation(NonNull.class) public class MongoEntityStorage implements EntityStorage { + private static final Logger log = Logger.getLogger(MongoEntityStorage.class); protected static final String COLLECTION_NAME = "entity-entities"; @@ -69,15 +80,40 @@ public class MongoEntityStorage implements EntityStorage { */ protected final String ryaInstanceName; + private final DuplicateDataDetector duplicateDataDetector; + private MongoTypeStorage mongoTypeStorage = null; + + /** + * Constructs an instance of {@link MongoEntityStorage}. + * + * @param mongo - A client connected to the Mongo instance that hosts the Rya instance. (not null) + * @param ryaInstanceName - The name of the Rya instance the {@link TypedEntity}s are for. (not null) + * @throws ConfigurationException + */ + public MongoEntityStorage(final MongoClient mongo, final String ryaInstanceName) throws EntityStorageException { + this(mongo, ryaInstanceName, null); + } + /** * Constructs an instance of {@link MongoEntityStorage}. * * @param mongo - A client connected to the Mongo instance that hosts the Rya instance. (not null) * @param ryaInstanceName - The name of the Rya instance the {@link TypedEntity}s are for. (not null) + * @param duplicateDataDetector - The {@link DuplicateDataDetector}. + * @throws EntityStorageException */ - public MongoEntityStorage(final MongoClient mongo, final String ryaInstanceName) { + public MongoEntityStorage(final MongoClient mongo, final String ryaInstanceName, final DuplicateDataDetector duplicateDataDetector) throws EntityStorageException { this.mongo = requireNonNull(mongo); this.ryaInstanceName = requireNonNull(ryaInstanceName); + if (duplicateDataDetector == null) { + try { + this.duplicateDataDetector = new DuplicateDataDetector(); + } catch (final ConfigurationException e) { + throw new EntityStorageException("Could not create duplicate data detector.", e); + } + } else { + this.duplicateDataDetector = duplicateDataDetector; + } } @Override @@ -85,10 +121,15 @@ public class MongoEntityStorage implements EntityStorage { requireNonNull(entity); try { - mongo.getDatabase(ryaInstanceName) - .getCollection(COLLECTION_NAME) - .insertOne( ENTITY_CONVERTER.toDocument(entity) ); - + final boolean hasDuplicate = detectDuplicates(entity); + + if (!hasDuplicate) { + mongo.getDatabase(ryaInstanceName) + .getCollection(COLLECTION_NAME) + .insertOne( ENTITY_CONVERTER.toDocument(entity) ); + } else { + throw new EntityNearDuplicateException("Duplicate data found and will not be inserted for Entity with Subject: " + entity); + } } catch(final MongoException e) { final ErrorCategory category = ErrorCategory.fromErrorCode( e.getCode() ); if(category == ErrorCategory.DUPLICATE_KEY) { @@ -242,4 +283,84 @@ public class MongoEntityStorage implements EntityStorage { return Stream.of(dataTypeFilter, valueFilter); } + + private boolean detectDuplicates(final Entity entity) throws EntityStorageException { + boolean hasDuplicate = false; + if (duplicateDataDetector.isDetectionEnabled()) { + // Grab all entities that have all the same explicit types as our + // original Entity. + final List<Entity> comparisonEntities = searchHasAllExplicitTypes(entity.getExplicitTypeIds()); + + // Now that we have our set of potential duplicates, compare them. + // We can stop when we find one duplicate. + for (final Entity compareEntity : comparisonEntities) { + try { + hasDuplicate = duplicateDataDetector.compareEntities(entity, compareEntity); + } catch (final SmartUriException e) { + throw new EntityStorageException("Encountered an error while comparing entities.", e); + } + if (hasDuplicate) { + break; + } + } + } + return hasDuplicate; + } + + /** + * Searches the Entity storage for all Entities that contain all the + * specified explicit type IDs. + * @param explicitTypeIds the {@link ImmutableList} of {@link RyaURI}s that + * are being searched for. + * @return the {@link List} of {@link Entity}s that have all the specified + * explicit type IDs. If nothing was found an empty {@link List} is + * returned. + * @throws EntityStorageException + */ + private List<Entity> searchHasAllExplicitTypes(final ImmutableList<RyaURI> explicitTypeIds) throws EntityStorageException { + final List<Entity> hasAllExplicitTypesEntities = new ArrayList<>(); + if (!explicitTypeIds.isEmpty()) { + // Grab the first type from the explicit type IDs. + final RyaURI firstType = explicitTypeIds.get(0); + + // Check if that type exists anywhere in storage. + final List<RyaURI> subjects = new ArrayList<>(); + Optional<Type> type; + try { + if (mongoTypeStorage == null) { + mongoTypeStorage = new MongoTypeStorage(mongo, ryaInstanceName); + } + type = mongoTypeStorage.get(firstType); + } catch (final TypeStorageException e) { + throw new EntityStorageException("Unable to get entity type: " + firstType, e); + } + if (type.isPresent()) { + // Grab the subjects for all the types we found matching "firstType" + final ConvertingCursor<TypedEntity> cursor = search(Optional.empty(), type.get(), Collections.emptySet()); + while (cursor.hasNext()) { + final TypedEntity typedEntity = cursor.next(); + final RyaURI subject = typedEntity.getSubject(); + subjects.add(subject); + } + } + + // Now grab all the Entities that have the subjects we found. + for (final RyaURI subject : subjects) { + final Optional<Entity> entityFromSubject = get(subject); + if (entityFromSubject.isPresent()) { + final Entity candidateEntity = entityFromSubject.get(); + // Filter out any entities that don't have all the same + // types associated with them as our original list of + // explicit type IDs. We already know the entities we found + // have "firstType" but now we have access to all the other + // types they have. + if (candidateEntity.getExplicitTypeIds().containsAll(explicitTypeIds)) { + hasAllExplicitTypesEntities.add(candidateEntity); + } + } + } + } + + return hasAllExplicitTypesEntities; + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/b319365e/extras/indexing/src/main/java/org/apache/rya/indexing/entity/update/BaseEntityIndexer.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/org/apache/rya/indexing/entity/update/BaseEntityIndexer.java b/extras/indexing/src/main/java/org/apache/rya/indexing/entity/update/BaseEntityIndexer.java index 84b0bdc..e73eeb3 100644 --- a/extras/indexing/src/main/java/org/apache/rya/indexing/entity/update/BaseEntityIndexer.java +++ b/extras/indexing/src/main/java/org/apache/rya/indexing/entity/update/BaseEntityIndexer.java @@ -33,6 +33,7 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicReference; import org.apache.hadoop.conf.Configuration; +import org.apache.log4j.Logger; import org.apache.rya.api.domain.RyaStatement; import org.apache.rya.api.domain.RyaType; import org.apache.rya.api.domain.RyaURI; @@ -40,6 +41,7 @@ import org.apache.rya.indexing.entity.model.Entity; import org.apache.rya.indexing.entity.model.Property; import org.apache.rya.indexing.entity.model.Type; import org.apache.rya.indexing.entity.storage.EntityStorage; +import org.apache.rya.indexing.entity.storage.EntityStorage.EntityStorageException; import org.apache.rya.indexing.entity.storage.TypeStorage; import org.apache.rya.indexing.entity.storage.TypeStorage.TypeStorageException; import org.apache.rya.indexing.entity.storage.mongo.ConvertingCursor; @@ -60,6 +62,7 @@ import edu.umd.cs.findbugs.annotations.NonNull; */ @DefaultAnnotation(NonNull.class) public abstract class BaseEntityIndexer implements EntityIndexer, MongoSecondaryIndex { + private static final Logger log = Logger.getLogger(BaseEntityIndexer.class); /** * When this URI is the Predicate of a Statement, it indicates a {@link Type} for an {@link Entity}. @@ -73,7 +76,11 @@ public abstract class BaseEntityIndexer implements EntityIndexer, MongoSecondary @Override public void setConf(final Configuration conf) { requireNonNull(conf); - entities.set( getEntityStorage(conf) ); + try { + entities.set( getEntityStorage(conf) ); + } catch (final EntityStorageException e) { + log.error("Unable to set entity storage."); + } types.set( getTypeStorage(conf) ); } http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/b319365e/extras/indexing/src/main/java/org/apache/rya/indexing/entity/update/EntityIndexer.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/org/apache/rya/indexing/entity/update/EntityIndexer.java b/extras/indexing/src/main/java/org/apache/rya/indexing/entity/update/EntityIndexer.java index 48cb0b1..aeb5a41 100644 --- a/extras/indexing/src/main/java/org/apache/rya/indexing/entity/update/EntityIndexer.java +++ b/extras/indexing/src/main/java/org/apache/rya/indexing/entity/update/EntityIndexer.java @@ -22,6 +22,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.rya.api.domain.RyaStatement; import org.apache.rya.api.persist.index.RyaSecondaryIndexer; import org.apache.rya.indexing.entity.storage.EntityStorage; +import org.apache.rya.indexing.entity.storage.EntityStorage.EntityStorageException; import org.apache.rya.indexing.entity.storage.TypeStorage; import edu.umd.cs.findbugs.annotations.Nullable; @@ -37,8 +38,9 @@ public interface EntityIndexer extends RyaSecondaryIndexer { * * @param conf - Indicates how the {@link EntityStorage} is initialized. (not null) * @return The {@link EntityStorage} that will be used by this indexer. + * @throws EntityStorageException */ - public @Nullable EntityStorage getEntityStorage(Configuration conf); + public @Nullable EntityStorage getEntityStorage(Configuration conf) throws EntityStorageException; /** * Creates the {@link TypeStorage} that will be used by the indexer. http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/b319365e/extras/indexing/src/main/java/org/apache/rya/indexing/entity/update/mongo/MongoEntityIndexer.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/org/apache/rya/indexing/entity/update/mongo/MongoEntityIndexer.java b/extras/indexing/src/main/java/org/apache/rya/indexing/entity/update/mongo/MongoEntityIndexer.java index 84eebaa..1ab48b6 100644 --- a/extras/indexing/src/main/java/org/apache/rya/indexing/entity/update/mongo/MongoEntityIndexer.java +++ b/extras/indexing/src/main/java/org/apache/rya/indexing/entity/update/mongo/MongoEntityIndexer.java @@ -20,6 +20,7 @@ package org.apache.rya.indexing.entity.update.mongo; import org.apache.hadoop.conf.Configuration; import org.apache.rya.indexing.entity.storage.EntityStorage; +import org.apache.rya.indexing.entity.storage.EntityStorage.EntityStorageException; import org.apache.rya.indexing.entity.storage.TypeStorage; import org.apache.rya.indexing.entity.storage.mongo.MongoEntityStorage; import org.apache.rya.indexing.entity.storage.mongo.MongoTypeStorage; @@ -41,7 +42,7 @@ public class MongoEntityIndexer extends BaseEntityIndexer { private MongoClient client; @Override - public EntityStorage getEntityStorage(final Configuration conf) { + public EntityStorage getEntityStorage(final Configuration conf) throws EntityStorageException { final MongoDBRdfConfiguration mongoConf = (MongoDBRdfConfiguration) conf; if (client == null) { if(mongoConf.getMongoClient() != null) { http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/b319365e/extras/indexing/src/main/java/org/apache/rya/indexing/mongodb/MongoDbSmartUri.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/org/apache/rya/indexing/mongodb/MongoDbSmartUri.java b/extras/indexing/src/main/java/org/apache/rya/indexing/mongodb/MongoDbSmartUri.java index cbc8796..b40c9b6 100644 --- a/extras/indexing/src/main/java/org/apache/rya/indexing/mongodb/MongoDbSmartUri.java +++ b/extras/indexing/src/main/java/org/apache/rya/indexing/mongodb/MongoDbSmartUri.java @@ -135,7 +135,7 @@ public class MongoDbSmartUri implements SmartUriStorage { if (!isInit) { try { setupClient(conf); - } catch (final UnknownHostException | MongoException e) { + } catch (final UnknownHostException | MongoException | EntityStorageException e) { throw new SmartUriException("Failed to setup MongoDB client", e); } } @@ -146,8 +146,9 @@ public class MongoDbSmartUri implements SmartUriStorage { * @param conf the {@link Configuration}. * @throws UnknownHostException * @throws MongoException + * @throws EntityStorageException */ - private void setupClient(final Configuration conf) throws UnknownHostException, MongoException { + private void setupClient(final Configuration conf) throws UnknownHostException, MongoException, EntityStorageException { final MongoDBRdfConfiguration mongoConf = (MongoDBRdfConfiguration) conf; mongoClient = mongoConf.getMongoClient(); if (mongoClient == null) { http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/b319365e/extras/indexing/src/main/java/org/apache/rya/indexing/smarturi/SmartUriAdapter.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/org/apache/rya/indexing/smarturi/SmartUriAdapter.java b/extras/indexing/src/main/java/org/apache/rya/indexing/smarturi/SmartUriAdapter.java index d6a5e8a..f637d0d 100644 --- a/extras/indexing/src/main/java/org/apache/rya/indexing/smarturi/SmartUriAdapter.java +++ b/extras/indexing/src/main/java/org/apache/rya/indexing/smarturi/SmartUriAdapter.java @@ -21,8 +21,6 @@ package org.apache.rya.indexing.smarturi; import java.io.UnsupportedEncodingException; import java.net.URISyntaxException; import java.net.URLDecoder; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedHashMap; @@ -98,40 +96,23 @@ public class SmartUriAdapter { } private static String getShortNameForType(final RyaURI type) throws SmartUriException { - String typeUriString; - try { - typeUriString = new java.net.URI(type.getData()).getRawSchemeSpecificPart(); - } catch (final URISyntaxException e) { - throw new SmartUriException("Unable to get create URI for type", e); - } - final Path path = Paths.get(typeUriString); - final String shortName = path.getFileName().toString(); + final String shortName = new URIImpl(type.getData()).getLocalName(); return shortName; } private static String addTypePrefixToUri(final String uriString, final String typePrefix) { - int location = StringUtils.lastIndexOf(uriString, "#"); - if (location == - 1) { - location = StringUtils.lastIndexOf(uriString, "/"); - } - - final String lastSegment = uriString.substring(location + 1); - - final String formattedUriString = uriString.substring(0, location + 1) + typePrefix + lastSegment; + final String localName = new URIImpl(uriString).getLocalName(); + final String beginning = StringUtils.removeEnd(uriString, localName); + final String formattedUriString = beginning + typePrefix + localName; return formattedUriString; } private static String removeTypePrefixFromUri(final String uriString, final String typePrefix) { - int location = StringUtils.lastIndexOf(uriString, "#"); - if (location == - 1) { - location = StringUtils.lastIndexOf(uriString, "/"); - } - - final String lastSegment = uriString.substring(location + 1); - final String replacement = lastSegment.replaceFirst(typePrefix + ".", ""); - - final String formattedUriString = uriString.substring(0, location + 1) + replacement; + final String localName = new URIImpl(uriString).getLocalName(); + final String beginning = StringUtils.removeEnd(uriString, localName); + final String replacement = localName.replaceFirst(typePrefix + ".", ""); + final String formattedUriString = beginning + replacement; return formattedUriString; } @@ -310,7 +291,7 @@ public class SmartUriAdapter { String formattedKey = key.getData(); if (StringUtils.isNotBlank(typeShortName)) { - formattedKey = addTypePrefixToUri(key.getData(), typeShortName); + formattedKey = addTypePrefixToUri(formattedKey, typeShortName); } final URI uri = new URIImpl(formattedKey); objectMap.put(uri, value); @@ -343,7 +324,7 @@ public class SmartUriAdapter { if (fragmentPosition > -1) { uriBuilder = new URIBuilder(new java.net.URI("urn://" + fragment)); } else { - uriBuilder = new URIBuilder(new java.net.URI(subjectData)); + uriBuilder = new URIBuilder(new java.net.URI(subjectData.replaceFirst(":", "://"))); } } catch (final URISyntaxException e) { throw new SmartUriException("Unable to serialize a Smart URI from the provided properties", e); @@ -447,55 +428,6 @@ public class SmartUriAdapter { return map; } -// public static Map<URI, Value> deserializeUri(final URI uri) throws SmartUriException { -// final String uriString = uri.stringValue(); -// final int fragmentPosition = uriString.indexOf("#"); -// String prefix = uriString.substring(0, fragmentPosition + 1); -// if (fragmentPosition == -1) { -// prefix = uriString.split("\\?", 2)[0]; -// } -// final String fragment = uriString.substring(fragmentPosition + 1, uriString.length()); -// java.net.URI queryUri; -// -// URIBuilder uriBuilder = null; -// try { -// if (fragmentPosition > -1) { -// queryUri = new java.net.URI("urn://" + fragment); -// } else { -// queryUri = new java.net.URI(uriString); -// } -// uriBuilder = new URIBuilder(queryUri); -// } catch (final URISyntaxException e) { -// throw new SmartUriException("Unable to deserialize Smart URI", e); -// } -// final Map<URI, Value> map = new HashMap<>(); -// final List<NameValuePair> parameters = uriBuilder.getQueryParams(); -// Map<RyaURI, String> entityTypeMap = new LinkedHashMap<>(); -// for (final NameValuePair pair : parameters) { -// final String keyString = pair.getName(); -// final String valueString = pair.getValue(); -// -// final URI keyUri = new URIImpl(prefix + keyString); -// final URI type = TypeDeterminer.determineType(valueString); -// if (type == XMLSchema.ANYURI) { -// final String decoded; -// try { -// decoded = URLDecoder.decode(valueString, Charsets.UTF_8.name()); -// } catch (final UnsupportedEncodingException e) { -// throw new SmartUriException("", e); -// } -// entityTypeMap = convertUriToTypeMap(new URIImpl(decoded)); -// } else { -// final RyaType ryaType = new RyaType(type, valueString); -// -// final Value value = RyaToRdfConversions.convertValue(ryaType); -// -// map.put(keyUri, value); -// } -// } -// return map; -// } - public static Entity deserializeUriEntity(final URI uri) throws SmartUriException { final String uriString = uri.stringValue(); final int fragmentPosition = uriString.indexOf("#"); http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/b319365e/extras/indexing/src/main/java/org/apache/rya/indexing/smarturi/SmartUriException.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/org/apache/rya/indexing/smarturi/SmartUriException.java b/extras/indexing/src/main/java/org/apache/rya/indexing/smarturi/SmartUriException.java index 979b6b9..6ec1e40 100644 --- a/extras/indexing/src/main/java/org/apache/rya/indexing/smarturi/SmartUriException.java +++ b/extras/indexing/src/main/java/org/apache/rya/indexing/smarturi/SmartUriException.java @@ -28,6 +28,14 @@ public class SmartUriException extends Exception { /** * Creates a new instance of {@link SmartUriException}. * @param message the message to be displayed by the exception. + */ + public SmartUriException(final String message) { + super(message); + } + + /** + * Creates a new instance of {@link SmartUriException}. + * @param message the message to be displayed by the exception. * @param throwable the source {#link Throwable} cause of the exception. */ public SmartUriException(final String message, final Throwable throwable) { http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/b319365e/extras/indexing/src/main/java/org/apache/rya/indexing/smarturi/duplication/ApproxEqualsDetector.java ---------------------------------------------------------------------- diff --git a/extras/indexing/src/main/java/org/apache/rya/indexing/smarturi/duplication/ApproxEqualsDetector.java b/extras/indexing/src/main/java/org/apache/rya/indexing/smarturi/duplication/ApproxEqualsDetector.java new file mode 100644 index 0000000..c450951 --- /dev/null +++ b/extras/indexing/src/main/java/org/apache/rya/indexing/smarturi/duplication/ApproxEqualsDetector.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.rya.indexing.smarturi.duplication; + +import org.apache.rya.indexing.smarturi.SmartUriException; +import org.openrdf.model.URI; + +/** + * Interface for detecting if two objects of type {@code T} are considered + * approximately equal to each other. + * @param <T> the type of object the implementation of + * {@link ApproxEqualsDetector} handles. + */ +public interface ApproxEqualsDetector<T> { + /** + * Checks if two objects are approximately equal. + * @param lhs the left hand side object. + * @param rhs the right hand side object. + * @return {@code true} if the two objects are considered approximately + * equals. {@code false} otherwise. + */ + public boolean areObjectsApproxEquals(final T lhs, final T rhs); + + /** + * @return the default tolerance for the type. + */ + public Tolerance getDefaultTolerance(); + + /** + * Converts a string representation of the object into the object + * represented by the class {@link #getTypeClass()}. + * @param string the {@link String} to convert to an object. + * @return the object. + * @throws SmartUriException + */ + public T convertStringToObject(final String string) throws SmartUriException; + + /** + * @return the object {@link Class} this detector is used for. + */ + public Class<?> getTypeClass(); + + /** + * @return the {@link URI} for the XML schema type this detector is used + * for. + */ + public URI getXmlSchemaUri(); + + /** + * Checks if two string representations of objects are approximately equal. + * @param lhs the left hand side string object representation. + * @param rhs the right hand side string object representation. + * @return {@code true} if the two string object representations are + * considered approximately equals. {@code false} otherwise. + * @throws SmartUriException + */ + public default boolean areApproxEquals(final String lhs, final String rhs) throws SmartUriException { + final T object1 = convertStringToObject(lhs); + final T object2 = convertStringToObject(rhs); + return areObjectsApproxEquals(object1, object2); + } +} \ No newline at end of file
