Improve consistency when returning entities/ids loaded from unique valuie index when entity repair ( removal of stale entities) has been executed.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/8982ddd3 Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/8982ddd3 Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/8982ddd3 Branch: refs/heads/master Commit: 8982ddd3a12541e988e802c3fc1dd5feca44ad6d Parents: 02e1619 Author: Michael Russo <[email protected]> Authored: Sun Jul 3 15:25:22 2016 -0700 Committer: Michael Russo <[email protected]> Committed: Sun Jul 3 15:25:22 2016 -0700 ---------------------------------------------------------------------- .../corepersistence/CpEntityManager.java | 44 ++++++++++++++------ .../persistence/collection/FieldSet.java | 12 ++++++ .../impl/EntityCollectionManagerImpl.java | 23 +--------- .../serialization/SerializationFig.java | 4 -- .../serialization/impl/MutableFieldSet.java | 12 ++++++ 5 files changed, 57 insertions(+), 38 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/8982ddd3/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java index 1ed05bd..318bba2 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java @@ -860,8 +860,6 @@ public class CpEntityManager implements EntityManager { Timer.Context repairedEntityGet = entGetRepairedEntityTimer.time(); - // TODO: can't we just sub in the getEntityRepair method here - // so for every read of a uniqueEntityField we can verify it is correct? StringField uniqueLookupRepairField = new StringField( propertyName, aliasType.toString()); @@ -877,11 +875,22 @@ public class CpEntityManager implements EntityManager { return null; } - FieldSet fieldSet = fieldSetObservable - .doOnError( t -> - logger.error("Unable to retrieve unique values due to: {}", t.getMessage()) - ) - .toBlocking().last(); + FieldSet fieldSet = fieldSetObservable.toBlocking().last(); + + // do a re-load if we know an entity repair was executed + // a hit to performance, but we need to return consistent success respones if we're repairing data + if(fieldSet.getEntityRepairExecuted()){ + + if(logger.isTraceEnabled()){ + logger.trace("One or more entities were repaired ( removed ) during loading of unique field [{}={}], " + + "executing the unique value lookup once more for consistency", uniqueLookupRepairField.getName(), + uniqueLookupRepairField.getValue()); + } + + fieldSet = ecm.getEntitiesFromFields( + Inflector.getInstance().singularize( collectionType ), + Collections.singletonList(uniqueLookupRepairField), uniqueIndexRepair).toBlocking().last(); + } repairedEntityGet.stop(); @@ -912,11 +921,22 @@ public class CpEntityManager implements EntityManager { return null; } - FieldSet fieldSet = fieldSetObservable - .doOnError( t -> - logger.error("Unable to retrieve unique values due to: {}", t.getMessage()) - ) - .toBlocking().last(); + FieldSet fieldSet = fieldSetObservable.toBlocking().last(); + + // do a re-load if we know an entity repair was executed + // a hit to performance, but we need to return consistent success respones if we're repairing data + if(fieldSet.getEntityRepairExecuted()){ + + if(logger.isTraceEnabled()){ + logger.trace("One or more entities were repaired ( removed ) during loading of unique field [{}={}], " + + "executing the unique value lookup once more for consistency", uniqueLookupRepairField.getName(), + uniqueLookupRepairField.getValue()); + } + + fieldSet = ecm.getEntitiesFromFields( + Inflector.getInstance().singularize( collectionType ), + Collections.singletonList(uniqueLookupRepairField), uniqueIndexRepair).toBlocking().last(); + } if(fieldSet == null || fieldSet.isEmpty()) { return null; http://git-wip-us.apache.org/repos/asf/usergrid/blob/8982ddd3/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/FieldSet.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/FieldSet.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/FieldSet.java index 43f470d..1fabd5a 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/FieldSet.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/FieldSet.java @@ -45,4 +45,16 @@ public interface FieldSet { * @return */ public boolean isEmpty(); + + /** + * Return true if there was at least entity repaired ( removed ) during the building of this set. + * @return + */ + public boolean getEntityRepairExecuted(); + + /** + * Return true if there was at least entity repaired ( removed ) during the building of this set. + * @return + */ + public void setEntityRepairExecuted(final boolean entityRepairExecuted); } http://git-wip-us.apache.org/repos/asf/usergrid/blob/8982ddd3/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java index 5bdcf1d..9261b56 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java @@ -413,32 +413,11 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager { if ( deleteBatch.getRowCount() > 0 ) { + response.setEntityRepairExecuted(true); deleteBatch.execute(); - - // optionally sleep after read repair as some tasks immediately try to write after the delete - if ( serializationFig.getReadRepairDelay() > 0 ){ - - try { - - if(logger.isTraceEnabled()) { - logger.trace("Sleeping {}ms after unique value read repair execution", - serializationFig.getReadRepairDelay()); - } - - Thread.sleep(Math.min(serializationFig.getReadRepairDelay(), 200L)); - - } catch (InterruptedException e) { - - // do nothing if sleep fails; log and continue on - logger.warn("Sleep during unique value read repair failed."); - } - - } - } - return response; } catch ( ConnectionException e ) { http://git-wip-us.apache.org/repos/asf/usergrid/blob/8982ddd3/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java index 12033fe..bdd71c7 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java @@ -58,8 +58,4 @@ public interface SerializationFig extends GuicyFig { @Default( "150" ) int getUniqueVerifyPoolSize(); - - @Key ( "collection.readrepair.delay" ) - @Default( "0" ) // in milliseconds - int getReadRepairDelay(); } http://git-wip-us.apache.org/repos/asf/usergrid/blob/8982ddd3/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MutableFieldSet.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MutableFieldSet.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MutableFieldSet.java index df2b850..da874c8 100644 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MutableFieldSet.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MutableFieldSet.java @@ -31,6 +31,8 @@ public class MutableFieldSet implements FieldSet { private final Map<Field<?>, MvccEntity> entities; + private boolean readRepairExecuted; + public MutableFieldSet( final int expectedSize ) { this.entities = new HashMap<>( expectedSize ); @@ -60,4 +62,14 @@ public class MutableFieldSet implements FieldSet { public boolean isEmpty() { return entities.size() == 0; } + + @Override + public void setEntityRepairExecuted( final boolean readRepairExecuted ){ + this.readRepairExecuted = readRepairExecuted; + } + + @Override + public boolean getEntityRepairExecuted(){ + return readRepairExecuted; + } }
