Repository: usergrid Updated Branches: refs/heads/USERGRID-1076 03a123dc8 -> 6e8b2b621
Removing code that has been refactored out. Edited flow to lead up to range query with more readability. Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/6e8b2b62 Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/6e8b2b62 Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/6e8b2b62 Branch: refs/heads/USERGRID-1076 Commit: 6e8b2b621cc45636bb322b03d0670d5d01b0a97d Parents: 03a123d Author: George Reyes <[email protected]> Authored: Wed Nov 4 16:34:48 2015 -0800 Committer: George Reyes <[email protected]> Committed: Wed Nov 4 16:34:48 2015 -0800 ---------------------------------------------------------------------- .../usergrid/tools/UniqueIndexCleanup.java | 206 +++++-------------- 1 file changed, 54 insertions(+), 152 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/6e8b2b62/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java ---------------------------------------------------------------------- diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java index 52288e4..9b1c202 100644 --- a/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java +++ b/stack/tools/src/main/java/org/apache/usergrid/tools/UniqueIndexCleanup.java @@ -98,12 +98,7 @@ import static org.apache.usergrid.utils.UUIDUtils.newTimeUUID; /** * This is a utility to audit all available entity ids in the secondary index. It then checks to see if any index value * is not present in the Entity_Index_Entries. If it is not, the value from the index is removed, and a forced re-index - * is triggered - * <p/> - * USERGRID-323 - * <p/> - * <p/> - * UniqueIndexCleanup -app [appid] -col [collectionname] + * is triggered <p/> USERGRID-323 <p/> <p/> UniqueIndexCleanup -app [appid] -col [collectionname] * * @author tnine */ @@ -115,7 +110,6 @@ public class UniqueIndexCleanup extends ToolBase { private static final int PAGE_SIZE = 100; - private static final Logger logger = LoggerFactory.getLogger( UniqueIndexCleanup.class ); /** @@ -130,7 +124,7 @@ public class UniqueIndexCleanup extends ToolBase { @Override - @SuppressWarnings("static-access") + @SuppressWarnings( "static-access" ) public Options createOptions() { @@ -169,97 +163,66 @@ public class UniqueIndexCleanup extends ToolBase { public void runTool( CommandLine line ) throws Exception { startSpring(); - logger.info( "Starting entity cleanup" ); - - Map<String, UUID> apps = getApplications( emf, line ); - - - for ( Entry<String, UUID> app : apps.entrySet() ) { - - logger.info( "Starting cleanup for app {}", app.getKey() ); - - UUID applicationId = app.getValue(); - EntityManagerImpl em = ( EntityManagerImpl ) emf.getEntityManager( applicationId ); - - //sanity check for corrupt apps - Application appEntity = em.getApplication(); - - if ( appEntity == null ) { - logger.warn( "Application does not exist in data. {}", app.getKey() ); - continue; - } - - CassandraService cass = em.getCass(); + logger.info( "Starting entity unique index cleanup" ); - Keyspace ko = cass.getUsergridApplicationKeyspace(); - Mutator<ByteBuffer> m = createMutator( ko, be ); - UUID timestampUuid = newTimeUUID(); - long timestamp = getTimestampInMicros( timestampUuid ); + // go through each collection and audit the values + Keyspace ko = cass.getUsergridApplicationKeyspace(); + Mutator<ByteBuffer> m = createMutator( ko, be ); - - // go through each collection and audit the values - for ( String collectionName : getCollectionNames( em, line ) ) { - - RangeSlicesQuery<ByteBuffer, ByteBuffer, ByteBuffer> rangeSlicesQuery = HFactory - .createRangeSlicesQuery( ko, be, be, be ) - .setColumnFamily( ENTITY_UNIQUE.getColumnFamily() ) - //not sure if I trust the lower two settings as it might iterfere with paging or set arbitrary limits and what I want to retrieve. + RangeSlicesQuery<ByteBuffer, ByteBuffer, ByteBuffer> rangeSlicesQuery = + HFactory.createRangeSlicesQuery( ko, be, be, be ).setColumnFamily( ENTITY_UNIQUE.getColumnFamily() ) + //not sure if I trust the lower two settings as it might iterfere with paging or set + // arbitrary limits and what I want to retrieve. //That needs to be verified. - .setKeys( null, null ) - .setRange( null, null, false, 100 ); - + .setKeys( null, null ).setRange( null, null, false, 100 ); - RangeSlicesIterator rangeSlicesIterator = new RangeSlicesIterator( rangeSlicesQuery,null,null ); - QueryResult<OrderedRows<ByteBuffer, ByteBuffer, ByteBuffer>> result = rangeSlicesQuery.execute(); - OrderedRows<ByteBuffer, ByteBuffer, ByteBuffer> rows = result.get(); - result.get().getList().get( 0 ).getColumnSlice(); + RangeSlicesIterator rangeSlicesIterator = new RangeSlicesIterator( rangeSlicesQuery, null, null ); - while(rangeSlicesIterator.hasNext()) { - Row rangeSliceValue = rangeSlicesIterator.next(); + while ( rangeSlicesIterator.hasNext() ) { + Row rangeSliceValue = rangeSlicesIterator.next(); - String returnedRowKey = - new String( ( ( ByteBuffer ) rangeSliceValue.getKey() ).array(), Charsets.UTF_8 ).trim(); + String returnedRowKey = + new String( ( ( ByteBuffer ) rangeSliceValue.getKey() ).array(), Charsets.UTF_8 ).trim(); - String[] parsedRowKey = returnedRowKey.split( ":" ); - if ( parsedRowKey[1].equals( "users" ) ) { + String[] parsedRowKey = returnedRowKey.split( ":" ); + UUID applicationId = UUID.fromString( parsedRowKey[0] ); + String collectionName = parsedRowKey[1]; + String uniqueValueKey = parsedRowKey[2]; + String uniqueValue = parsedRowKey[3]; - ColumnSlice<ByteBuffer, ByteBuffer> columnSlice = rangeSliceValue.getColumnSlice(); - if ( columnSlice.getColumns().size() != 0 ) { - System.out.println( returnedRowKey ); - List<HColumn<ByteBuffer, ByteBuffer>> cols = columnSlice.getColumns(); - - for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) { - UUID entityId = ue.fromByteBuffer( col.getName() ); + EntityManagerImpl em = ( EntityManagerImpl ) emf.getEntityManager( applicationId ); + Boolean cleanup = false; - if(parsedRowKey[0].equals( MANAGEMENT_APPLICATION_ID.toString() )){ - if(managementService.getAdminUserByUuid( entityId )==null ){ - logger.warn( "Entity with id {} did not exist in app {}", entityId, applicationId ); - System.out.println( "Deleting column uuid: " + entityId.toString() ); + //TODO: make parsed row key more human friendly. Anybody looking at it doesn't know what value means what. + if ( parsedRowKey[1].equals( "users" ) ) { + ColumnSlice<ByteBuffer, ByteBuffer> columnSlice = rangeSliceValue.getColumnSlice(); + if ( columnSlice.getColumns().size() != 0 ) { + System.out.println( returnedRowKey ); + List<HColumn<ByteBuffer, ByteBuffer>> cols = columnSlice.getColumns(); - Object key = key( applicationId, collectionName, parsedRowKey[2], parsedRowKey[3]); - addDeleteToMutator( m, ENTITY_UNIQUE, key, entityId, timestamp ); - m.execute(); - continue; - } - } - else if ( em.get( entityId ) == null ) { - logger.warn( "Entity with id {} did not exist in app {}", entityId, applicationId ); - System.out.println( "Deleting column uuid: " + entityId.toString() ); + for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) { + UUID entityId = ue.fromByteBuffer( col.getName() ); - Object key = key( applicationId, collectionName, parsedRowKey[2], parsedRowKey[3]); - addDeleteToMutator( m, ENTITY_UNIQUE, key, entityId, timestamp ); - m.execute(); - continue; - } + if ( applicationId.equals( MANAGEMENT_APPLICATION_ID ) ) { + if ( managementService.getAdminUserByUuid( entityId ) == null ) { + cleanup = true; } } + else if ( em.get( entityId ) == null ) { + cleanup =true; + } + + if(cleanup == true){ + DeleteUniqueValue( m, applicationId, collectionName, uniqueValueKey, uniqueValue, + entityId ); + cleanup = false; + } } } - } } @@ -267,78 +230,17 @@ public class UniqueIndexCleanup extends ToolBase { } - private Map<String, UUID> getApplications( EntityManagerFactory emf, CommandLine line ) throws Exception { - String appName = line.getOptionValue( APPLICATION_ARG ); - - if ( appName == null ) { - return emf.getApplications(); - } - - ApplicationInfo app = managementService.getApplicationInfo( Identifier.from( appName ) ); - - if ( app == null ) { - logger.error( "Could not find application with id or name {}", appName ); - System.exit( 3 ); - } - - - Map<String, UUID> apps = new HashMap<String, UUID>(); - - apps.put( app.getName(), app.getId() ); - - return apps; - } - - - private Set<String> getCollectionNames( EntityManager em, CommandLine line ) throws Exception { - - String collectionName = line.getOptionValue( COLLECTION_ARG ); - - if ( collectionName == null ) { - return em.getApplicationCollections(); - } - - - Set<String> names = new HashSet<String>(); - names.add( collectionName ); - - return names; - } - - - private List<HColumn<ByteBuffer, ByteBuffer>> scanIndexForAllTypes( Keyspace ko, - IndexBucketLocator indexBucketLocator, - UUID applicationId, Object rowKey, - UUID entityId, String prop ) throws Exception { - - //TODO Determine the index bucket. Scan the entire index for properties with this entityId. - - - DynamicComposite start = null; - - List<HColumn<ByteBuffer, ByteBuffer>> cols; - - List<HColumn<ByteBuffer, ByteBuffer>> results = new ArrayList<HColumn<ByteBuffer, ByteBuffer>>(); - - - do { - cols = cass.getColumns( ko, ENTITY_INDEX, rowKey, start, null, 100, false ); - - for ( HColumn<ByteBuffer, ByteBuffer> col : cols ) { - DynamicComposite secondaryIndexValue = DynamicComposite.fromByteBuffer( col.getName().duplicate() ); - - UUID storedId = ( UUID ) secondaryIndexValue.get( 2 ); - - //add it to the set. We can't short circuit due to property ordering - if ( entityId.equals( storedId ) ) { - results.add( col ); - } - - start = secondaryIndexValue; - } - } - while ( cols.size() == 100 ); + private void DeleteUniqueValue( final Mutator<ByteBuffer> m, final UUID applicationId, + final String collectionName, final String uniqueValueKey, final String uniqueValue, + final UUID entityId ) throws Exception { + logger.warn( "Entity with id {} did not exist in app {}", entityId, applicationId ); + System.out.println( "Deleting column uuid: " + entityId.toString() ); + UUID timestampUuid = newTimeUUID(); + long timestamp = getTimestampInMicros( timestampUuid ); - return results; + Object key = key( applicationId, collectionName, uniqueValueKey, uniqueValue ); + addDeleteToMutator( m, ENTITY_UNIQUE, key, entityId, timestamp ); + m.execute(); + return; } }
