Merge branch 'hotfix-20160819'
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/aa0353d9 Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/aa0353d9 Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/aa0353d9 Branch: refs/heads/master Commit: aa0353d96716642746a8bc185414a11948138da0 Parents: ae39eca 9fae803 Author: Michael Russo <[email protected]> Authored: Thu Sep 1 17:51:09 2016 -0700 Committer: Michael Russo <[email protected]> Committed: Thu Sep 1 17:51:09 2016 -0700 ---------------------------------------------------------------------- .../corepersistence/CpEntityManager.java | 21 +- .../pipeline/read/ResultsPage.java | 14 +- .../read/traverse/AbstractReadGraphFilter.java | 7 + .../results/IdQueryExecutor.java | 10 + .../results/ObservableQueryExecutor.java | 3 + .../actorsystem/ActorSystemManagerImpl.java | 12 +- .../mvcc/stage/CollectionIoEvent.java | 10 +- .../mvcc/stage/delete/MarkCommit.java | 2 +- .../mvcc/stage/write/WriteCommit.java | 13 +- .../mvcc/stage/write/WriteUniqueVerify.java | 16 +- .../uniquevalues/UniqueValuesServiceTest.java | 4 +- .../astyanax/MultiRowShardColumnIterator.java | 30 ++ .../persistence/core/shard/SmartShard.java | 9 +- .../serialization/impl/shard/DirectedEdge.java | 7 + .../impl/shard/NodeShardAllocation.java | 5 +- .../graph/serialization/impl/shard/Shard.java | 17 +- .../impl/shard/impl/EdgeSearcher.java | 17 +- .../shard/impl/NodeShardAllocationImpl.java | 13 +- .../impl/shard/impl/NodeShardCacheImpl.java | 24 +- .../shard/impl/ShardGroupCompactionImpl.java | 2 +- .../impl/shard/impl/ShardGroupDeletionImpl.java | 24 +- .../impl/ShardedEdgeSerializationImpl.java | 116 ++++-- .../shard/impl/serialize/ShardSerializer.java | 11 +- .../graph/GraphManagerShardConsistencyIT.java | 396 +++++++++++++++++-- .../impl/shard/NodeShardAllocationTest.java | 4 +- .../impl/shard/NodeShardCacheTest.java | 4 +- .../persistence/index/query/Identifier.java | 2 +- .../apache/usergrid/rest/ShutdownListener.java | 2 +- .../rest/applications/users/UsersResource.java | 14 +- .../rest/management/ManagementResource.java | 33 +- .../collection/CollectionsResourceIT.java | 18 +- .../rest/management/ManagementResourceIT.java | 4 +- .../usergrid/tools/CollectionIterator.java | 273 +++++++++++++ .../org/apache/usergrid/tools/ExportApp.java | 118 +----- .../org/apache/usergrid/tools/ShardManager.java | 242 ++++++++++++ .../org/apache/usergrid/tools/ToolBase.java | 2 + .../usergrid/tools/UniqueValueManager.java | 311 +++++++++++++++ .../usergrid/tools/export/ExportConnection.java | 74 ++++ .../usergrid/tools/export/ExportEntity.java | 66 ++++ 39 files changed, 1678 insertions(+), 272 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/aa0353d9/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java ---------------------------------------------------------------------- diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java index fc1382f,9d376f9..edd44e8 --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java @@@ -22,10 -22,7 +22,11 @@@ import java.util.HashMap import java.util.Map; import java.util.UUID; + +import com.datastax.driver.core.BatchStatement; +import com.datastax.driver.core.Session; + + import org.apache.commons.lang3.StringUtils; import org.apache.usergrid.persistence.actorsystem.ActorSystemFig; import org.apache.usergrid.persistence.collection.exception.WriteUniqueVerifyException; import org.apache.usergrid.persistence.collection.uniquevalues.UniqueValueException; http://git-wip-us.apache.org/repos/asf/usergrid/blob/aa0353d9/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java ---------------------------------------------------------------------- diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java index e7dbb10,c51c742..7ef573d --- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java +++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java @@@ -30,9 -22,14 +30,10 @@@ import com.google.common.base.Precondit import com.google.inject.Inject; import com.google.inject.Singleton; import com.netflix.astyanax.Keyspace; -import com.netflix.astyanax.MutationBatch; -import com.netflix.astyanax.connectionpool.exceptions.ConnectionException; -import com.netflix.astyanax.model.ConsistencyLevel; import com.netflix.hystrix.HystrixCommand; import com.netflix.hystrix.HystrixCommandGroupKey; -import com.netflix.hystrix.HystrixCommandProperties; import com.netflix.hystrix.HystrixThreadPoolProperties; + import org.apache.commons.lang3.StringUtils; import org.apache.usergrid.persistence.actorsystem.ActorSystemFig; import org.apache.usergrid.persistence.collection.MvccEntity; import org.apache.usergrid.persistence.collection.exception.WriteUniqueVerifyException; @@@ -176,35 -169,40 +177,34 @@@ public class WriteUniqueVerify implemen // use write-first then read strategy final UniqueValue written = new UniqueValueImpl( field, mvccEntity.getId(), mvccEntity.getVersion() ); - try { - // don't use read repair on this pre-write check - // stronger consistency is extremely important here, more so than performance - UniqueValueSet set = uniqueValueStrat.load(scope, cassandraFig.getReadCL(), - written.getEntityId().getType(), Collections.singletonList(written.getField()), false); + // don't use read repair on this pre-write check - // stronger consistency is extremely important here, more so than performance - UniqueValueSet set = uniqueValueStrat.load(scope, cassandraFig.getDataStaxReadConsistentCl(), ++ UniqueValueSet set = uniqueValueStrat.load(scope, cassandraFig.getDataStaxReadCl(), + written.getEntityId().getType(), Collections.singletonList(written.getField()), false); - set.forEach(uniqueValue -> { + set.forEach(uniqueValue -> { - if(!uniqueValue.getEntityId().getUuid().equals(written.getEntityId().getUuid())){ - - if(logger.isTraceEnabled()){ - logger.trace("Pre-write violation detected. Attempted write for unique value [{}={}] and " + - "entity id [{}], entity version [{}] conflicts with already existing entity id [{}], " + - "entity version [{}]", - written.getField().getName(), - written.getField().getValue().toString(), - written.getEntityId().getUuid(), - written.getEntityVersion(), - uniqueValue.getEntityId().getUuid(), - uniqueValue.getEntityVersion()); - } - - preWriteUniquenessViolations.put(field.getName(), field); + if(!uniqueValue.getEntityId().getUuid().equals(written.getEntityId().getUuid())){ + if(logger.isTraceEnabled()){ + logger.trace("Pre-write violation detected. Attempted write for unique value [{}={}] and " + + "entity id [{}], entity version [{}] conflicts with already existing entity id [{}], " + + "entity version [{}]", + written.getField().getName(), + written.getField().getValue().toString(), + written.getEntityId().getUuid(), + written.getEntityVersion(), + uniqueValue.getEntityId().getUuid(), + uniqueValue.getEntityVersion()); } - }); + preWriteUniquenessViolations.put(field.getName(), field); + } + + }); - } catch (ConnectionException e) { - throw new RuntimeException("Error connecting to cassandra", e); - } // only build the batch statement if we don't have a violation for the field if( preWriteUniquenessViolations.get(field.getName()) == null) { http://git-wip-us.apache.org/repos/asf/usergrid/blob/aa0353d9/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardCacheImpl.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/aa0353d9/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardedEdgeSerializationImpl.java ----------------------------------------------------------------------
