Repository: usergrid Updated Branches: refs/heads/release-2.1.1 8014b4f91 -> f5daca59a
Add concurrency to re-index and fix de-serialization problems with Shiro cache, plus comments. Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/f5daca59 Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/f5daca59 Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/f5daca59 Branch: refs/heads/release-2.1.1 Commit: f5daca59a94175a83cb03a7b413eacaa90e78589 Parents: 8014b4f Author: Michael Russo <[email protected]> Authored: Wed Mar 30 16:14:59 2016 -0700 Committer: Michael Russo <[email protected]> Committed: Wed Mar 30 16:14:59 2016 -0700 ---------------------------------------------------------------------- .../corepersistence/index/IndexProcessorFig.java | 11 ++++++++++- .../corepersistence/index/ReIndexServiceImpl.java | 10 ++++++---- .../security/shiro/credentials/AdminUserAccessToken.java | 3 +++ .../shiro/credentials/ApplicationClientCredentials.java | 5 +++++ .../security/shiro/principals/AdminUserPrincipal.java | 3 +++ .../shiro/principals/ApplicationGuestPrincipal.java | 4 +++- .../security/shiro/principals/ApplicationPrincipal.java | 4 +++- .../shiro/principals/ApplicationUserPrincipal.java | 3 +++ .../security/shiro/principals/OrganizationPrincipal.java | 4 +++- 9 files changed, 39 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/f5daca59/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java index 15f9f9d..b94da65 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexProcessorFig.java @@ -44,6 +44,8 @@ public interface IndexProcessorFig extends GuicyFig { String REINDEX_BUFFER_SIZE = "elasticsearch.reindex.buffer_size"; + String REINDEX_CONCURRENCY_FACTOR = "elasticsearch.reindex.concurrency.factor"; + /** * Set the amount of time to wait when Elasticsearch rejects a requests before @@ -88,11 +90,18 @@ public interface IndexProcessorFig extends GuicyFig { @Key(ELASTICSEARCH_QUEUE_IMPL) String getQueueImplementation(); - @Default("1000") + @Default("100") @Key(REINDEX_BUFFER_SIZE) int getReindexBufferSize(); /** + * The number of parallel buffers during re-index that can be processed + */ + @Default("10") + @Key(REINDEX_CONCURRENCY_FACTOR) + int getReindexConcurrencyFactor(); + + /** * Flag to resolve the LOCAL queue implementation service synchronously. */ @Default("false") http://git-wip-us.apache.org/repos/asf/usergrid/blob/f5daca59/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java index da2ef1d..b5f7d77 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java @@ -125,12 +125,14 @@ public class ReIndexServiceImpl implements ReIndexService { final Observable<List<EdgeScope>> runningReIndex = allEntityIdsObservable.getEdgesToEntities( applicationScopes, reIndexRequestBuilder.getCollectionName(), cursorSeek.getSeekValue() ) .buffer( indexProcessorFig.getReindexBufferSize()) - .doOnNext(edges -> { + .flatMap( edgeScopes -> Observable.just(edgeScopes) + .doOnNext(edges -> { - logger.info("Sending batch of {} to be indexed.", edges.size()); - indexService.indexBatch(edges, modifiedSince); + logger.info("Sending batch of {} to be indexed.", edges.size()); + indexService.indexBatch(edges, modifiedSince); + }) + .subscribeOn( Schedulers.io() ), indexProcessorFig.getReindexConcurrencyFactor()); - }); //start our sampler and state persistence http://git-wip-us.apache.org/repos/asf/usergrid/blob/f5daca59/stack/services/src/main/java/org/apache/usergrid/security/shiro/credentials/AdminUserAccessToken.java ---------------------------------------------------------------------- diff --git a/stack/services/src/main/java/org/apache/usergrid/security/shiro/credentials/AdminUserAccessToken.java b/stack/services/src/main/java/org/apache/usergrid/security/shiro/credentials/AdminUserAccessToken.java index a2f3743..4c3ffce 100644 --- a/stack/services/src/main/java/org/apache/usergrid/security/shiro/credentials/AdminUserAccessToken.java +++ b/stack/services/src/main/java/org/apache/usergrid/security/shiro/credentials/AdminUserAccessToken.java @@ -19,6 +19,9 @@ package org.apache.usergrid.security.shiro.credentials; public class AdminUserAccessToken extends AbstractAccessTokenCredentials implements AdminUserCredentials { + /** + * Needed for Jackson. Do not remove + */ public AdminUserAccessToken() {} public AdminUserAccessToken( String token ) { http://git-wip-us.apache.org/repos/asf/usergrid/blob/f5daca59/stack/services/src/main/java/org/apache/usergrid/security/shiro/credentials/ApplicationClientCredentials.java ---------------------------------------------------------------------- diff --git a/stack/services/src/main/java/org/apache/usergrid/security/shiro/credentials/ApplicationClientCredentials.java b/stack/services/src/main/java/org/apache/usergrid/security/shiro/credentials/ApplicationClientCredentials.java index 8c20bb6..0972c37 100644 --- a/stack/services/src/main/java/org/apache/usergrid/security/shiro/credentials/ApplicationClientCredentials.java +++ b/stack/services/src/main/java/org/apache/usergrid/security/shiro/credentials/ApplicationClientCredentials.java @@ -19,6 +19,11 @@ package org.apache.usergrid.security.shiro.credentials; public class ApplicationClientCredentials extends AbstractClientCredentials implements ApplicationCredentials { + /** + * Needed for Jackson, do not remove + */ + public ApplicationClientCredentials(){} + public ApplicationClientCredentials( String key, String secret ) { super( key, secret ); } http://git-wip-us.apache.org/repos/asf/usergrid/blob/f5daca59/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/AdminUserPrincipal.java ---------------------------------------------------------------------- diff --git a/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/AdminUserPrincipal.java b/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/AdminUserPrincipal.java index 0e18f35..a594d1e 100644 --- a/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/AdminUserPrincipal.java +++ b/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/AdminUserPrincipal.java @@ -37,6 +37,9 @@ import static org.apache.usergrid.security.shiro.utils.SubjectUtils.getPermissio public class AdminUserPrincipal extends UserPrincipal { private static final Logger logger = LoggerFactory.getLogger(AdminUserPrincipal.class); + /** + * Needed for Jackson, do not remove + */ public AdminUserPrincipal() { } http://git-wip-us.apache.org/repos/asf/usergrid/blob/f5daca59/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/ApplicationGuestPrincipal.java ---------------------------------------------------------------------- diff --git a/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/ApplicationGuestPrincipal.java b/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/ApplicationGuestPrincipal.java index 38da94d..9e9d821 100644 --- a/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/ApplicationGuestPrincipal.java +++ b/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/ApplicationGuestPrincipal.java @@ -43,7 +43,9 @@ public class ApplicationGuestPrincipal extends PrincipalIdentifier { ApplicationInfo application; - + /** + * Needed for Jackson, do not remove + */ public ApplicationGuestPrincipal( ) {} public ApplicationGuestPrincipal( ApplicationInfo application ) { http://git-wip-us.apache.org/repos/asf/usergrid/blob/f5daca59/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/ApplicationPrincipal.java ---------------------------------------------------------------------- diff --git a/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/ApplicationPrincipal.java b/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/ApplicationPrincipal.java index 5252052..cff9daa 100644 --- a/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/ApplicationPrincipal.java +++ b/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/ApplicationPrincipal.java @@ -34,7 +34,9 @@ public class ApplicationPrincipal extends PrincipalIdentifier { ApplicationInfo application; - + /** + * Needed for Jackson, do not remove + */ public ApplicationPrincipal() {} public ApplicationPrincipal( ApplicationInfo application ) { http://git-wip-us.apache.org/repos/asf/usergrid/blob/f5daca59/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/ApplicationUserPrincipal.java ---------------------------------------------------------------------- diff --git a/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/ApplicationUserPrincipal.java b/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/ApplicationUserPrincipal.java index 962fcf5..65503b4 100644 --- a/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/ApplicationUserPrincipal.java +++ b/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/ApplicationUserPrincipal.java @@ -45,6 +45,9 @@ public class ApplicationUserPrincipal extends UserPrincipal { private static final Logger logger = LoggerFactory.getLogger(AdminUserPrincipal.class); + /** + * Needed for Jackson, do not remove + */ public ApplicationUserPrincipal() {} public ApplicationUserPrincipal( UUID applicationId, UserInfo user ) { http://git-wip-us.apache.org/repos/asf/usergrid/blob/f5daca59/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/OrganizationPrincipal.java ---------------------------------------------------------------------- diff --git a/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/OrganizationPrincipal.java b/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/OrganizationPrincipal.java index e885365..0ebbaa6 100644 --- a/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/OrganizationPrincipal.java +++ b/stack/services/src/main/java/org/apache/usergrid/security/shiro/principals/OrganizationPrincipal.java @@ -40,7 +40,9 @@ public class OrganizationPrincipal extends PrincipalIdentifier { OrganizationInfo organization; - + /** + * Needed for Jackson, do not remove + */ public OrganizationPrincipal() { }
