Repository: usergrid Updated Branches: refs/heads/readRepairForIndexValues [created] a67057909
Added tests and read repair to the guts of 1.0 Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/a6705790 Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/a6705790 Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/a6705790 Branch: refs/heads/readRepairForIndexValues Commit: a67057909e480c97273c9f6c366643069f24e114 Parents: 7db5769 Author: George Reyes <[email protected]> Authored: Tue Nov 10 09:06:15 2015 -0800 Committer: George Reyes <[email protected]> Committed: Tue Nov 10 09:06:15 2015 -0800 ---------------------------------------------------------------------- .../cassandra/EntityManagerImpl.java | 32 ++++ .../usergrid/persistence/EntityManagerIT.java | 2 + .../usergrid/services/ServiceRequestIT.java | 169 ++++++++++++++++++- 3 files changed, 202 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/a6705790/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java index 4a6e2ea..ecd0f0a 100644 --- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java +++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java @@ -109,6 +109,7 @@ import static java.lang.String.CASE_INSENSITIVE_ORDER; import static java.util.Arrays.asList; import static me.prettyprint.hector.api.factory.HFactory.createCounterSliceQuery; +import static me.prettyprint.hector.api.factory.HFactory.createMutator; import static org.apache.commons.lang.StringUtils.capitalize; import static org.apache.commons.lang.StringUtils.isBlank; import static org.apache.usergrid.locking.LockHelper.getUniqueUpdateLock; @@ -156,6 +157,7 @@ import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtil import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.key; import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.toStorableBinaryValue; import static org.apache.usergrid.persistence.cassandra.CassandraService.ALL_COUNT; +import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID; import static org.apache.usergrid.persistence.cassandra.Serializers.be; import static org.apache.usergrid.persistence.cassandra.Serializers.le; import static org.apache.usergrid.persistence.cassandra.Serializers.se; @@ -538,6 +540,8 @@ public class EntityManagerImpl implements EntityManager { Object key = createUniqueIndexKey( ownerEntityId, collectionNameInternal, propertyName, propertyValue ); + //need to fix by asking todd as to why 2. + //why is this set to 2? List<HColumn<ByteBuffer, ByteBuffer>> cols = cass.getColumns( cass.getApplicationKeyspace( applicationId ), ENTITY_UNIQUE, key, null, null, 2, false ); @@ -549,12 +553,40 @@ public class EntityManagerImpl implements EntityManager { } //shouldn't happen, but it's an error case + if ( cols.size() > 1 ) { logger.error( "INDEX CORRUPTION: More than 1 unique value exists for entities in ownerId {} of type {} on " + "property {} with value {}", new Object[] { ownerEntityId, collectionNameInternal, propertyName, propertyValue } ); + + //retreive up to 100 columns + List<HColumn<ByteBuffer, ByteBuffer>> indexingColumns = cass.getColumns( cass.getApplicationKeyspace( applicationId ), ENTITY_UNIQUE, key, null, null, 100, + false ); + + + + //go through it column + for ( HColumn<ByteBuffer, ByteBuffer> col : indexingColumns ) { + UUID indexCorruptionUuid = ue.fromByteBuffer( col.getName()); + if (get( indexCorruptionUuid ) == null ) { + UUID timestampUuid = newTimeUUID(); + long timestamp = getTimestampInMicros( timestampUuid ); + Keyspace ko = cass.getApplicationKeyspace( ownerEntityId ); + Mutator<ByteBuffer> mutator = createMutator( ko, be ); + + addDeleteToMutator( mutator, ENTITY_UNIQUE, key, indexCorruptionUuid, timestamp ); + mutator.execute(); + cols.remove( col ); + } + else{ + + } + } } + cols = cass.getColumns( cass.getApplicationKeyspace( applicationId ), ENTITY_UNIQUE, key, null, null, 2, + false ); + /** * Doing this in a loop sucks, but we need to account for possibly having more than 1 entry in the index due * to corruption. We need to allow them to update, otherwise http://git-wip-us.apache.org/repos/asf/usergrid/blob/a6705790/stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java ---------------------------------------------------------------------- diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java index 5cfc013..b911b3b 100644 --- a/stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java +++ b/stack/core/src/test/java/org/apache/usergrid/persistence/EntityManagerIT.java @@ -561,4 +561,6 @@ public class EntityManagerIT extends AbstractCoreIT { //Not an owner assertFalse( em.isCollectionMember( createdUser2, "devices", createdDevice ) ); } + + } http://git-wip-us.apache.org/repos/asf/usergrid/blob/a6705790/stack/services/src/test/java/org/apache/usergrid/services/ServiceRequestIT.java ---------------------------------------------------------------------- diff --git a/stack/services/src/test/java/org/apache/usergrid/services/ServiceRequestIT.java b/stack/services/src/test/java/org/apache/usergrid/services/ServiceRequestIT.java index 5b5428a..d89536f 100644 --- a/stack/services/src/test/java/org/apache/usergrid/services/ServiceRequestIT.java +++ b/stack/services/src/test/java/org/apache/usergrid/services/ServiceRequestIT.java @@ -17,7 +17,9 @@ package org.apache.usergrid.services; +import java.nio.ByteBuffer; import java.util.Arrays; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -27,15 +29,39 @@ import org.junit.Rule; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + +import org.apache.commons.lang.RandomStringUtils; + import org.apache.usergrid.ServiceITSetup; import org.apache.usergrid.ServiceITSetupImpl; import org.apache.usergrid.ServiceITSuite; import org.apache.usergrid.cassandra.ClearShiroSubject; import org.apache.usergrid.cassandra.Concurrent; - +import org.apache.usergrid.management.ApplicationInfo; +import org.apache.usergrid.management.OrganizationOwnerInfo; +import org.apache.usergrid.persistence.Entity; +import org.apache.usergrid.persistence.EntityManager; +import org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils; +import org.apache.usergrid.persistence.cassandra.CassandraService; +import org.apache.usergrid.persistence.entities.User; +import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsException; +import org.apache.usergrid.utils.UUIDUtils; + +import me.prettyprint.hector.api.Keyspace; +import me.prettyprint.hector.api.mutation.Mutator; + +import static me.prettyprint.hector.api.factory.HFactory.createMutator; +import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_UNIQUE; +import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addInsertToMutator; +import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.createTimestamp; import static org.apache.usergrid.persistence.cassandra.CassandraService.DEFAULT_APPLICATION_ID; +import static org.apache.usergrid.persistence.cassandra.Serializers.be; import static org.apache.usergrid.services.ServiceParameter.filter; import static org.apache.usergrid.services.ServiceParameter.parameters; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.fail; @Concurrent() @@ -76,4 +102,145 @@ public class ServiceRequestIT { p = filter( path.getParameters(), replaceParameters ); logger.info( "" + p ); } + + //Verify that entity read repair is functioning as intended. + @Test + public void testRepairOfSingleEntity() throws Exception{ + String rand = RandomStringUtils.randomAlphanumeric( 10 ); + + String orgName = "org_" + rand; + String appName = "app_" +rand; + String username = "username_" + rand; + String adminUsername = "admin_"+rand; + String email = username+"@derp.com"; + String password = username; + + String collectionName = "users"; + + + OrganizationOwnerInfo organizationOwnerInfo = setup.getMgmtSvc().createOwnerAndOrganization( orgName,adminUsername,adminUsername,email,password ); + + ApplicationInfo applicationInfo = setup.getMgmtSvc().createApplication( organizationOwnerInfo.getOrganization().getUuid(),appName ); + + EntityManager entityManager = setup.getEmf().getEntityManager( applicationInfo.getId() ); + + Map<String,Object> userInfo = new HashMap<String, Object>( ); + userInfo.put( "username",username ); + + //Entity entityToBeCorrupted = entityManager.create( collectionName,userInfo ); + + CassandraService cass = setup.getCassSvc(); + + Object key = CassandraPersistenceUtils.key( applicationInfo.getId(), collectionName, "username", username ); + + Keyspace ko = cass.getApplicationKeyspace( applicationInfo.getId() ); + Mutator<ByteBuffer> m = createMutator( ko, be ); + + UUID testEntityUUID = UUIDUtils.newTimeUUID(); + //this below calll should make the column family AND the column name + addInsertToMutator( m,ENTITY_UNIQUE,key, testEntityUUID,0,createTimestamp()); + + m.execute(); + + //verify that there is no corresponding entity with the uuid or alias provided + //verify it returns null. + assertNull(entityManager.get( testEntityUUID )); + + //the below works but not needed for this test. + //assertNull( entityManager.getAlias("user",username)); + + //verify that we cannot recreate the entity due to duplicate unique property exception + Entity entityToBeCorrupted = null; + try { + entityToBeCorrupted = entityManager.create( collectionName, userInfo ); + fail(); + }catch(DuplicateUniquePropertyExistsException dup){ + + } + catch(Exception e){ + fail("shouldn't throw something else i think"); + } + + + entityToBeCorrupted = entityManager.create( collectionName,userInfo ); + + assertNotNull( entityToBeCorrupted ); + assertEquals( username,( ( User ) entityToBeCorrupted ).getUsername() ); + + } + + + @Test + public void testRepairOfOnlyOneOfTwoColumns() throws Exception{ + String rand = RandomStringUtils.randomAlphanumeric( 10 ); + + String orgName = "org_" + rand; + String appName = "app_" +rand; + String username = "username_" + rand; + String adminUsername = "admin_"+rand; + String email = username+"@derp.com"; + String password = username; + + String collectionName = "users"; + + + OrganizationOwnerInfo organizationOwnerInfo = setup.getMgmtSvc().createOwnerAndOrganization( orgName,adminUsername,adminUsername,email,password ); + + ApplicationInfo applicationInfo = setup.getMgmtSvc().createApplication( organizationOwnerInfo.getOrganization().getUuid(),appName ); + + EntityManager entityManager = setup.getEmf().getEntityManager( applicationInfo.getId() ); + + Map<String,Object> userInfo = new HashMap<String, Object>( ); + userInfo.put( "username",username ); + + //Entity entityToBeCorrupted = entityManager.create( collectionName,userInfo ); + + CassandraService cass = setup.getCassSvc(); + + Object key = CassandraPersistenceUtils.key( applicationInfo.getId(), collectionName, "username", username ); + + Keyspace ko = cass.getApplicationKeyspace( applicationInfo.getId() ); + Mutator<ByteBuffer> m = createMutator( ko, be ); + + //create a new column + Entity validCoexistingEntity = entityManager.create( collectionName, userInfo ); + + UUID testEntityUUID = UUIDUtils.newTimeUUID(); + //this below calll should make the column family AND the column name for an already existing column thus adding one legit and one dummy value. + addInsertToMutator( m,ENTITY_UNIQUE,key, testEntityUUID,0,createTimestamp()); + + m.execute(); + + //verify that there is no corresponding entity with the uuid or alias provided + //verify it returns null. + assertNull(entityManager.get( testEntityUUID )); + + //the below works but not needed for this test. + //assertNull( entityManager.getAlias("user",username)); + + //verify that we cannot recreate the entity due to duplicate unique property exception + Entity entityToBeCorrupted = null; + try { + entityToBeCorrupted = entityManager.create( collectionName, userInfo ); + fail(); + }catch(DuplicateUniquePropertyExistsException dup){ + + } + catch(Exception e){ + throw e; + } + + //should return null since we have duplicate alias. Will Cause index corruptions to be thrown. + //TODO: fix the below so that it fails everytime. + //50/50 chance to succeed or fail + // assertNull( entityManager + // .get( entityManager.getAlias( applicationInfo.getId(), collectionName, username ).getUuid() ) ); + + + + //verifies it works now. + assertNotNull( entityManager + .get( entityManager.getAlias( applicationInfo.getId(), collectionName, username ).getUuid() ) ); + + } }
