1. Add missing start for delete queue workers. 2. Add dropping of application local keyspace for testing. 3. Testing changes.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/3f7afcd8 Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/3f7afcd8 Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/3f7afcd8 Branch: refs/heads/master Commit: 3f7afcd8836ed9b589b69523d8bcc507a739d912 Parents: c201f1f Author: Mike Dunker <[email protected]> Authored: Mon Aug 21 17:01:48 2017 -0700 Committer: Mike Dunker <[email protected]> Committed: Mon Aug 21 17:01:48 2017 -0700 ---------------------------------------------------------------------- .../corepersistence/CpEntityManager.java | 3 +-- .../asyncevents/AsyncEventServiceImpl.java | 4 +++ .../index/CollectionSettingsImpl.java | 1 - .../usergrid/persistence/EntityManager.java | 2 +- .../persistence/entities/Application.java | 26 ++++++++++++++++++++ .../usergrid/persistence/CoreSchemaManager.java | 16 ++++++++---- .../usergrid/persistence/RebuildIndexTest.java | 24 +++++++++++++----- .../resources/usergrid-custom-test.properties | 2 +- .../resources/usergrid-custom-test.properties | 2 +- .../AbstractServiceNotificationIT.java | 4 +-- .../gcm/NotificationsServiceIT.java | 2 +- .../resources/usergrid-custom-test.properties | 2 +- 12 files changed, 67 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java index cdb4fc7..68c4ef0 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java @@ -2471,10 +2471,9 @@ public class CpEntityManager implements EntityManager { final Entity entity; - //this is the fall back, why isn't this writt if ( entityType == null ) { return null; -// throw new EntityNotFoundException( String.format( "Counld not find type for uuid {}", uuid ) ); +// throw new EntityNotFoundException( String.format( "Could not find type for uuid {}", uuid ) ); } entity = get( new SimpleEntityRef( entityType, uuid ) ); http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java index 79a80c0..428772f 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java @@ -875,6 +875,10 @@ public class AsyncEventServiceImpl implements AsyncEventService { startWorker(AsyncEventQueueType.UTILITY); } + for (int i = 0; i < deleteCount; i++) { + startWorker(AsyncEventQueueType.DELETE); + } + if( indexQueue instanceof SNSQueueManagerImpl) { logger.info("Queue manager implementation supports dead letters, start dead letter queue workers."); for (int i = 0; i < indexDeadCount; i++) { http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionSettingsImpl.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionSettingsImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionSettingsImpl.java index 921777a..76f0b4b 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionSettingsImpl.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionSettingsImpl.java @@ -20,7 +20,6 @@ package org.apache.usergrid.corepersistence.index; import com.google.common.base.Optional; import com.google.inject.Inject; -import com.google.inject.Singleton; import org.apache.usergrid.persistence.map.MapManager; import org.apache.usergrid.utils.JsonUtils; import org.slf4j.Logger; http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java index ae4623d..c9752c3 100644 --- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java +++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java @@ -743,7 +743,7 @@ public interface EntityManager { /** * Add a new index to the application for scale - * @param suffix unique indentifier for additional index + * @param newIndexName unique identifier for additional index * @param shards number of shards * @param replicas number of replicas * @param writeConsistency only "one, quorum, or all" http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Application.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Application.java b/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Application.java index 790b4d9..0a4360f 100644 --- a/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Application.java +++ b/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Application.java @@ -48,6 +48,32 @@ public class Application extends TypedEntity implements Serializable { public static final String COLLECTION_ACTIVITIES = "activities"; + public static final String COLLECTION_EVENTS = "events"; + + public static final String COLLECTION_FOLDERS = "folders"; + + public static final String COLLECTION_DEVICES = "devices"; + + public static final String COLLECTION_NOTIFICATIONS = "notifications"; + + public static final String COLLECTION_ROLES = "roles"; + + public static boolean isCustomCollectionName(String collectionName) { + switch (collectionName.toLowerCase()) { + case COLLECTION_USERS: + case COLLECTION_GROUPS: + case COLLECTION_ASSETS: + case COLLECTION_ACTIVITIES: + case COLLECTION_EVENTS: + case COLLECTION_FOLDERS: + case COLLECTION_DEVICES: + case COLLECTION_NOTIFICATIONS: + case COLLECTION_ROLES: + return false; + } + return true; + } + @EntityProperty(indexed = true, fulltextIndexed = false, required = true, mutable = false, aliasProperty = true, basic = true) protected String name; http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/core/src/test/java/org/apache/usergrid/persistence/CoreSchemaManager.java ---------------------------------------------------------------------- diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/CoreSchemaManager.java b/stack/core/src/test/java/org/apache/usergrid/persistence/CoreSchemaManager.java index c064751..1d51c25 100644 --- a/stack/core/src/test/java/org/apache/usergrid/persistence/CoreSchemaManager.java +++ b/stack/core/src/test/java/org/apache/usergrid/persistence/CoreSchemaManager.java @@ -18,6 +18,7 @@ package org.apache.usergrid.persistence; import org.apache.usergrid.locking.LockManager; +import org.apache.usergrid.persistence.core.CassandraConfig; import org.apache.usergrid.persistence.core.CassandraFig; import org.apache.usergrid.persistence.core.datastax.CQLUtils; import org.apache.usergrid.persistence.core.datastax.DataStaxCluster; @@ -37,14 +38,14 @@ public class CoreSchemaManager implements SchemaManager { private static final Logger logger = LoggerFactory.getLogger( CoreSchemaManager.class ); private final Setup setup; - private final CassandraFig cassandraFig; + private final CassandraConfig cassandraConfig; private final LockManager lockManager; private final DataStaxCluster dataStaxCluster; public CoreSchemaManager( final Setup setup, Injector injector ) { this.setup = setup; - this.cassandraFig = injector.getInstance( CassandraFig.class ); + this.cassandraConfig = injector.getInstance( CassandraConfig.class ); this.lockManager = injector.getInstance( LockManager.class ); this.dataStaxCluster = injector.getInstance( DataStaxCluster.class ); } @@ -80,16 +81,21 @@ public class CoreSchemaManager implements SchemaManager { @Override public void destroy() { - logger.info( "dropping keyspaces" ); try { + logger.info( "dropping application keyspace" ); dataStaxCluster.getClusterSession() - .execute("DROP KEYSPACE "+ CQLUtils.quote(cassandraFig.getApplicationKeyspace())); + .execute("DROP KEYSPACE "+ CQLUtils.quote(cassandraConfig.getApplicationKeyspace())); + dataStaxCluster.waitForSchemaAgreement(); + + logger.info( "dropping application local keyspace" ); + dataStaxCluster.getClusterSession() + .execute("DROP KEYSPACE "+ CQLUtils.quote(cassandraConfig.getApplicationLocalKeyspace())); dataStaxCluster.waitForSchemaAgreement(); dataStaxCluster.getClusterSession().close(); // close session so it's meta will get refreshed } catch ( Exception e ) { - logger.error("Error dropping application keyspace: {} error: {}", cassandraFig.getApplicationKeyspace(), e); + logger.error("Error dropping application keyspaces: {} error: {}", cassandraConfig.getApplicationKeyspace(), e); } logger.info( "keyspaces dropped" ); logger.info( "dropping indices" ); http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java ---------------------------------------------------------------------- diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java index d2bff37..a1ff84b 100644 --- a/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java +++ b/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java @@ -74,7 +74,7 @@ public class RebuildIndexTest extends AbstractCoreIT { } - @Test( timeout = 120000 ) + @Test( timeout = 240000 ) public void rebuildOneCollectionIndex() throws Exception { logger.info( "Started rebuildOneCollectionIndex()" ); @@ -163,7 +163,7 @@ public class RebuildIndexTest extends AbstractCoreIT { waitForRebuild( status, reIndexService ); - app.waitForQueueDrainAndRefreshIndex(5000); + app.waitForQueueDrainAndRefreshIndex(15000); // ----------------- test that we can read the catherder collection and not the catshepard @@ -278,7 +278,7 @@ public class RebuildIndexTest extends AbstractCoreIT { assertNotNull( status.getJobId(), "JobId is present" ); - logger.info( "Rebuilt index" ); + logger.info( "Rebuilt index, jobID={}", status.getJobId()); waitForRebuild( status, reIndexService ); @@ -511,14 +511,26 @@ public class RebuildIndexTest extends AbstractCoreIT { * Wait for the rebuild to occur */ private void waitForRebuild( final ReIndexService.ReIndexStatus status, final ReIndexService reIndexService ) - throws InterruptedException { + throws InterruptedException, IllegalArgumentException { + if (status != null) { + logger.info("waitForRebuild: jobID={}", status.getJobId()); + } else { + logger.info("waitForRebuild: error, status = null"); + throw new IllegalArgumentException("reindexStatus = null"); + } while ( true ) { try { final ReIndexService.ReIndexStatus updatedStatus = reIndexService.getStatus( status.getJobId() ); - if ( updatedStatus.getStatus() == ReIndexService.Status.COMPLETE ) { - break; + if (updatedStatus == null) { + logger.info("waitForRebuild: updated status is null"); + } else { + logger.info("waitForRebuild: status={} numberProcessed={}", updatedStatus.getStatus().toString(), updatedStatus.getNumberProcessed()); + + if ( updatedStatus.getStatus() == ReIndexService.Status.COMPLETE ) { + break; + } } } catch ( IllegalArgumentException iae ) { http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/core/src/test/resources/usergrid-custom-test.properties ---------------------------------------------------------------------- diff --git a/stack/core/src/test/resources/usergrid-custom-test.properties b/stack/core/src/test/resources/usergrid-custom-test.properties index df253f0..24f3046 100644 --- a/stack/core/src/test/resources/usergrid-custom-test.properties +++ b/stack/core/src/test/resources/usergrid-custom-test.properties @@ -29,7 +29,7 @@ elasticsearch.management_number_replicas=0 elasticsearch.managment_index=usergrid_core_management #cassandra.keyspace.application=core_tests_schema elasticsearch.queue_impl.resolution=true -elasticsearch.queue_impl=DISTRIBUTED +elasticsearch.queue_impl=LOCAL elasticsearch.buffer_timeout=1 http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/rest/src/test/resources/usergrid-custom-test.properties ---------------------------------------------------------------------- diff --git a/stack/rest/src/test/resources/usergrid-custom-test.properties b/stack/rest/src/test/resources/usergrid-custom-test.properties index 4b76ca4..ee7b039 100644 --- a/stack/rest/src/test/resources/usergrid-custom-test.properties +++ b/stack/rest/src/test/resources/usergrid-custom-test.properties @@ -68,7 +68,7 @@ usergrid.cluster.port=2555 collection.uniquevalues.actors=300 collection.uniquevalues.authoritative.region=us-east -elasticsearch.queue_impl=DISTRIBUTED +elasticsearch.queue_impl=LOCAL # Queueing Test Settings # Reduce the long polling time for the tests http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/services/src/test/java/org/apache/usergrid/services/notifications/AbstractServiceNotificationIT.java ---------------------------------------------------------------------- diff --git a/stack/services/src/test/java/org/apache/usergrid/services/notifications/AbstractServiceNotificationIT.java b/stack/services/src/test/java/org/apache/usergrid/services/notifications/AbstractServiceNotificationIT.java index c035192..9b3dfcd 100644 --- a/stack/services/src/test/java/org/apache/usergrid/services/notifications/AbstractServiceNotificationIT.java +++ b/stack/services/src/test/java/org/apache/usergrid/services/notifications/AbstractServiceNotificationIT.java @@ -49,9 +49,9 @@ public abstract class AbstractServiceNotificationIT extends AbstractServiceIT { protected Notification notificationWaitForComplete(Notification notification) throws Exception { - long timeout = System.currentTimeMillis() + 60000; + long timeout = System.currentTimeMillis() + 120000; while (System.currentTimeMillis() < timeout) { - app.waitForQueueDrainAndRefreshIndex(200); + app.waitForQueueDrainAndRefreshIndex(1000); notification = app.getEntityManager().get(notification.getUuid(), Notification.class); if (notification.getFinished() != null) { return notification; http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java ---------------------------------------------------------------------- diff --git a/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java index 8360009..ecc5443 100644 --- a/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java +++ b/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java @@ -632,7 +632,7 @@ public class NotificationsServiceIT extends AbstractServiceNotificationIT { // receipts are created and queried, wait a bit longer for this to happen as indexing - app.waitForQueueDrainAndRefreshIndex(500); + app.waitForQueueDrainAndRefreshIndex(5000); // get the receipts entity IDs List<EntityRef> receipts = getNotificationReceipts(notification); http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/services/src/test/resources/usergrid-custom-test.properties ---------------------------------------------------------------------- diff --git a/stack/services/src/test/resources/usergrid-custom-test.properties b/stack/services/src/test/resources/usergrid-custom-test.properties index bcc8b8e..4178a07 100644 --- a/stack/services/src/test/resources/usergrid-custom-test.properties +++ b/stack/services/src/test/resources/usergrid-custom-test.properties @@ -35,7 +35,7 @@ elasticsearch.managment_index=usergrid_services_management elasticsearch.buffer_timeout=1 elasticsearch.queue_impl.resolution=true -elasticsearch.queue_impl=DISTRIBUTED +elasticsearch.queue_impl=LOCAL # Queueing Test Settings queue.long.polling.time.millis=50
