Repository: usergrid Updated Branches: refs/heads/master 8913e43d2 -> faafed885
Fixed failing tests and bugs -Fixed tests for Core module that were broken due to changes in 8913e43 -Fixed failing tests for Services module related to threading issues -Fixed failing tests for Rest module related to Index rebuild -Fixed re-index code to ensure rebuild status is correctly maintained -Updated Shiro to 1.3.2 (latest) Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/faafed88 Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/faafed88 Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/faafed88 Branch: refs/heads/master Commit: faafed8858d9999195337f1776a94d57f84ac7be Parents: 8913e43 Author: Keyur Karnik <[email protected]> Authored: Wed Aug 29 05:00:13 2018 -0700 Committer: Keyur Karnik <[email protected]> Committed: Tue Sep 4 21:25:31 2018 -0700 ---------------------------------------------------------------------- .../corepersistence/index/ReIndexService.java | 2 +- .../index/ReIndexServiceImpl.java | 22 ++++---- .../usergrid/persistence/RebuildIndexTest.java | 57 ++++++++++++++++---- stack/pom.xml | 2 +- .../apache/usergrid/rest/IndexResourceIT.java | 45 +++++++++++----- .../org/apache/usergrid/NewOrgAppAdminRule.java | 1 + .../management/OrganizationConfigIT.java | 4 +- .../management/export/ExportServiceIT.java | 3 ++ 8 files changed, 98 insertions(+), 38 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/faafed88/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java index d37f117..48c3908 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java @@ -97,7 +97,7 @@ public interface ReIndexService { } /** - * Get the jobId used to resume this operation + * Get the collectionName used to resume this operation */ public String getCollectionName() { return collectionName; http://git-wip-us.apache.org/repos/asf/usergrid/blob/faafed88/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java index d4fb249..036f89c 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java @@ -202,7 +202,7 @@ public class ReIndexServiceImpl implements ReIndexService { .subscribeOn( Schedulers.io() ).subscribe(); if(isForCollection){ - return new ReIndexStatus( "", Status.STARTED, 0, 0, reIndexRequestBuilder.getCollectionName().get() ); + return new ReIndexStatus( "", Status.STARTED, 0, 0, CpNamingUtils.getNameFromEdgeType(reIndexRequestBuilder.getCollectionName().get()) ); } @@ -365,21 +365,23 @@ public class ReIndexServiceImpl implements ReIndexService { private void writeStateMetaForCollection(final String appIdString, final String collectionName, final Status status, final long processedCount, final long lastUpdated ) { - if(logger.isDebugEnabled()) { + String prefixedColName = CpNamingUtils.getEdgeTypeFromCollectionName( collectionName.toLowerCase() ); + if(logger.isDebugEnabled()) { logger.debug( "Flushing state for collection {}, status {}, processedCount {}, lastUpdated {}", - collectionName, status, processedCount, lastUpdated); + collectionName, status, processedCount, lastUpdated); } - mapManager.putString( appIdString + MAP_SEPARATOR + collectionName + MAP_STATUS_KEY, status.name() ); - mapManager.putLong( appIdString + MAP_SEPARATOR + collectionName + MAP_COUNT_KEY, processedCount ); - mapManager.putLong( appIdString + MAP_SEPARATOR + collectionName + MAP_UPDATED_KEY, lastUpdated ); + mapManager.putString( appIdString + MAP_SEPARATOR + prefixedColName + MAP_STATUS_KEY, status.name() ); + mapManager.putLong( appIdString + MAP_SEPARATOR + prefixedColName + MAP_COUNT_KEY, processedCount ); + mapManager.putLong( appIdString + MAP_SEPARATOR + prefixedColName + MAP_UPDATED_KEY, lastUpdated ); } private ReIndexStatus getIndexResponseForCollection( final String appIdString, final String collectionName ) { - final String stringStatus = - mapManager.getString( appIdString + MAP_SEPARATOR + collectionName + MAP_STATUS_KEY ); + String prefixedColName = CpNamingUtils.getEdgeTypeFromCollectionName( collectionName.toLowerCase() ); + final String stringStatus = + mapManager.getString( appIdString + MAP_SEPARATOR + prefixedColName + MAP_STATUS_KEY ); if(stringStatus == null){ return new ReIndexStatus( "", Status.UNKNOWN, 0, 0, collectionName ); @@ -387,8 +389,8 @@ public class ReIndexServiceImpl implements ReIndexService { final Status status = Status.valueOf( stringStatus ); - final long processedCount = mapManager.getLong( appIdString + MAP_SEPARATOR + collectionName + MAP_COUNT_KEY ); - final long lastUpdated = mapManager.getLong( appIdString + MAP_SEPARATOR + collectionName + MAP_UPDATED_KEY ); + final long processedCount = mapManager.getLong( appIdString + MAP_SEPARATOR + prefixedColName + MAP_COUNT_KEY ); + final long lastUpdated = mapManager.getLong( appIdString + MAP_SEPARATOR + prefixedColName + MAP_UPDATED_KEY ); return new ReIndexStatus( "", status, processedCount, lastUpdated, collectionName ); } http://git-wip-us.apache.org/repos/asf/usergrid/blob/faafed88/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java ---------------------------------------------------------------------- diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java index 57962c0..90af5ba 100644 --- a/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java +++ b/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java @@ -154,12 +154,12 @@ public class RebuildIndexTest extends AbstractCoreIT { ReIndexService.ReIndexStatus status = reIndexService.rebuildIndex(builder); - assertNotNull(status.getJobId(), "JobId is present"); + assertNotNull(status.getCollectionName(), "Collection name is present"); logger.info("Rebuilt index"); - waitForRebuild(status, reIndexService); + waitForRebuild(em.getApplicationId().toString(), status.getCollectionName(), reIndexService); //app.waitForQueueDrainAndRefreshIndex(15000); @@ -279,7 +279,7 @@ public class RebuildIndexTest extends AbstractCoreIT { logger.info("Rebuilt index, jobID={}", status.getJobId()); - waitForRebuild(status, reIndexService); + waitForRebuild(status.getJobId(), reIndexService); logger.info("Rebuilt index"); @@ -387,7 +387,7 @@ public class RebuildIndexTest extends AbstractCoreIT { logger.info("Rebuilt index"); - waitForRebuild(status, reIndexService); + waitForRebuild(status.getJobId(), reIndexService); logger.info("Rebuilt index"); @@ -486,7 +486,7 @@ public class RebuildIndexTest extends AbstractCoreIT { logger.info("Rebuilt index"); - waitForRebuild(status, reIndexService); + waitForRebuild(status.getJobId(), reIndexService); logger.info("Rebuilt index"); @@ -505,18 +505,53 @@ public class RebuildIndexTest extends AbstractCoreIT { /** * Wait for the rebuild to occur */ - private void waitForRebuild(final ReIndexService.ReIndexStatus status, final ReIndexService reIndexService) + private void waitForRebuild(final String jobId, final ReIndexService reIndexService) throws InterruptedException, IllegalArgumentException { - if (status != null) { - logger.info("waitForRebuild: jobID={}", status.getJobId()); + if (jobId != null && !jobId.trim().equals("")) { + logger.info("waitForRebuild: jobID={}", jobId); } else { - logger.info("waitForRebuild: error, status = null"); - throw new IllegalArgumentException("reindexStatus = null"); + logger.info("waitForRebuild: error, jobId = null or empty"); + throw new IllegalArgumentException("jobId = null or empty"); } while (true) { try { - final ReIndexService.ReIndexStatus updatedStatus = reIndexService.getStatus(status.getJobId()); + final ReIndexService.ReIndexStatus updatedStatus = reIndexService.getStatus(jobId); + + if (updatedStatus == null) { + logger.info("waitForRebuild: updated status is null"); + } else { + logger.info("waitForRebuild: status={} numberProcessed={}", updatedStatus.getStatus().toString(), updatedStatus.getNumberProcessed()); + + if (updatedStatus.getStatus() == ReIndexService.Status.COMPLETE) { + break; + } + } + } catch (IllegalArgumentException iae) { + //swallow. Thrown if our job can't be found. I.E hasn't updated yet + } + + + Thread.sleep(1000); + } + } + + + /** + * Wait for the rebuild to occur + */ + private void waitForRebuild(final String appId, final String collectionName, final ReIndexService reIndexService) + throws InterruptedException, IllegalArgumentException { + if (appId != null && !appId.trim().equals("") && collectionName != null && !collectionName.trim().equals("")) { + logger.info("waitForRebuild: appId={} collName={}", appId, collectionName); + } else { + logger.info("waitForRebuild: error, appId or collName = null or empty"); + throw new IllegalArgumentException("appId or collName = null or empty"); + } + while (true) { + + try { + final ReIndexService.ReIndexStatus updatedStatus = reIndexService.getStatusForCollection(appId, collectionName); if (updatedStatus == null) { logger.info("waitForRebuild: updated status is null"); http://git-wip-us.apache.org/repos/asf/usergrid/blob/faafed88/stack/pom.xml ---------------------------------------------------------------------- diff --git a/stack/pom.xml b/stack/pom.xml index c98c72d..aa289d0 100644 --- a/stack/pom.xml +++ b/stack/pom.xml @@ -114,7 +114,7 @@ <junit-version>4.12</junit-version> <log4j-version>1.2.16</log4j-version> <org.springframework.version>3.2.13.RELEASE</org.springframework.version> - <shiro-version>1.2.4</shiro-version> + <shiro-version>1.3.2</shiro-version> <slf4j-version>1.7.2</slf4j-version> <snakeyaml-version>1.9</snakeyaml-version> <tomcat-version>7.0.64</tomcat-version> http://git-wip-us.apache.org/repos/asf/usergrid/blob/faafed88/stack/rest/src/test/java/org/apache/usergrid/rest/IndexResourceIT.java ---------------------------------------------------------------------- diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/IndexResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/IndexResourceIT.java index 28d6501..be71881 100644 --- a/stack/rest/src/test/java/org/apache/usergrid/rest/IndexResourceIT.java +++ b/stack/rest/src/test/java/org/apache/usergrid/rest/IndexResourceIT.java @@ -25,6 +25,7 @@ import org.apache.usergrid.rest.test.resource.AbstractRestIT; import org.apache.usergrid.rest.test.resource.model.ApiResponse; import org.apache.usergrid.rest.test.resource.model.QueryParameters; import org.apache.usergrid.rest.test.resource.model.Token; +import org.apache.usergrid.corepersistence.index.ReIndexService.Status; import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature; import org.junit.Ignore; import org.junit.Test; @@ -68,35 +69,51 @@ public class IndexResourceIT extends AbstractRestIT { .get(clientSetup.getSuperuserName(),clientSetup.getSuperuserPassword()); QueryParameters queryParameters = new QueryParameters(); - queryParameters.addParam( "access_token",superUserToken.getAccessToken()); + queryParameters.addParam("access_token", superUserToken.getAccessToken()); ApiResponse result = clientSetup.getRestClient() - .pathResource( "system/index/rebuild/" + clientSetup.getAppUuid() + "/StOrElaTloNs" ) - .post( false, ApiResponse.class, null, queryParameters, true ); + .pathResource("system/index/rebuild/" + clientSetup.getAppUuid() + "/StOrElaTloNs") + .post(false, ApiResponse.class, null, queryParameters, true); assertNotNull(result); + assertEquals(Status.STARTED.name(), result.getStatus()); //try the reindex endpoint with all lowercase Characters queryParameters = new QueryParameters(); - queryParameters.addParam( "access_token",clientSetup.getSuperuserToken().getAccessToken() ); + queryParameters.addParam("access_token", superUserToken.getAccessToken()); result = clientSetup.getRestClient() - .pathResource( "system/index/rebuild/"+clientSetup.getAppUuid()+"/storelatlons" ) - .post( false, ApiResponse.class,null,queryParameters,true); - String status = result.getProperties().get("jobId").toString(); - - assertNotNull( result ); + .pathResource("system/index/rebuild/" + clientSetup.getAppUuid() + "/storelatlons") + .post(false, ApiResponse.class, null, queryParameters, true); + + assertNotNull(result); + //at this point, this could return a result of the previous reindex, or if it has completed, it will create a new job + assertNotEquals(Status.UNKNOWN.name(), result.getStatus()); WebTarget res = clientSetup.getRestClient() - .pathResource( "system/index/rebuild/" + result.getProperties() - .get( "jobId" ).toString() ) + .pathResource("system/index/rebuild/" + clientSetup.getAppUuid() + "/storelatlons") .getTarget(); HttpAuthenticationFeature feature = HttpAuthenticationFeature.basicBuilder() .credentials( "superuser", "superpassword" ).build(); result = res.register( feature ).request().get( ApiResponse.class ); - - assertNotNull( result ); - assertEquals(status,result.getProperties().get("jobId").toString()); + assertNotNull(result); + + int retry = 0; + while(retry < 5 && !result.getStatus().equals(Status.COMPLETE.name())) { + try { + //hope reindex completes, if not, that's still ok + Thread.sleep(1000); + } catch (InterruptedException e) { + + } + result = res.register( feature ).request().get( ApiResponse.class ); + retry++; + assertNotNull(result); + } + + Map<String, Object> resultMap = result.getProperties(); + assertNotNull( resultMap ); + assertEquals(1,resultMap.get("numberQueued")); } http://git-wip-us.apache.org/repos/asf/usergrid/blob/faafed88/stack/services/src/test/java/org/apache/usergrid/NewOrgAppAdminRule.java ---------------------------------------------------------------------- diff --git a/stack/services/src/test/java/org/apache/usergrid/NewOrgAppAdminRule.java b/stack/services/src/test/java/org/apache/usergrid/NewOrgAppAdminRule.java index 659520e..892a440 100644 --- a/stack/services/src/test/java/org/apache/usergrid/NewOrgAppAdminRule.java +++ b/stack/services/src/test/java/org/apache/usergrid/NewOrgAppAdminRule.java @@ -120,6 +120,7 @@ public class NewOrgAppAdminRule implements TestRule { * Create the org admin and application */ protected void before( Description description ) throws Exception { + logger.info( "Test {}: Starting with application", description.getDisplayName() ); final String className = description.getClassName(); final String methodName = description.getMethodName(); final String uuidString = newUUIDString(); http://git-wip-us.apache.org/repos/asf/usergrid/blob/faafed88/stack/services/src/test/java/org/apache/usergrid/management/OrganizationConfigIT.java ---------------------------------------------------------------------- diff --git a/stack/services/src/test/java/org/apache/usergrid/management/OrganizationConfigIT.java b/stack/services/src/test/java/org/apache/usergrid/management/OrganizationConfigIT.java index 6424dbd..d2932c2 100644 --- a/stack/services/src/test/java/org/apache/usergrid/management/OrganizationConfigIT.java +++ b/stack/services/src/test/java/org/apache/usergrid/management/OrganizationConfigIT.java @@ -32,12 +32,14 @@ import org.junit.ClassRule; import org.junit.Rule; import org.junit.Test; +import net.jcip.annotations.NotThreadSafe; + import java.util.*; import static org.apache.usergrid.TestHelper.*; import static org.junit.Assert.*; - +@NotThreadSafe public class OrganizationConfigIT { @Rule http://git-wip-us.apache.org/repos/asf/usergrid/blob/faafed88/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java ---------------------------------------------------------------------- diff --git a/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java index 870b678..2268ea6 100644 --- a/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java +++ b/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java @@ -59,6 +59,8 @@ import com.amazonaws.SDKGlobalConfiguration; import com.google.common.collect.ImmutableSet; import com.google.inject.Module; +import net.jcip.annotations.NotThreadSafe; + import static org.apache.usergrid.TestHelper.newUUIDString; import static org.apache.usergrid.TestHelper.uniqueApp; import static org.apache.usergrid.TestHelper.uniqueOrg; @@ -74,6 +76,7 @@ import static org.mockito.Mockito.when; * * */ +@NotThreadSafe public class ExportServiceIT { private static final Logger logger = LoggerFactory.getLogger( ExportServiceIT.class );
