add delete back
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/a87ca3a1 Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/a87ca3a1 Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/a87ca3a1 Branch: refs/heads/USERGRID-347 Commit: a87ca3a1968d29170314dc8a67ae108862fb04e6 Parents: 65c20a1 Author: Shawn Feldman <[email protected]> Authored: Wed Mar 18 17:45:35 2015 -0600 Committer: Shawn Feldman <[email protected]> Committed: Wed Mar 18 17:45:35 2015 -0600 ---------------------------------------------------------------------- .../corepersistence/CpEntityManager.java | 5 +- .../usergrid/persistence/EntityManager.java | 3 +- .../cassandra/EntityManagerImpl.java | 4 +- .../org/apache/usergrid/CoreApplication.java | 7 ++- .../usergrid/persistence/index/EntityIndex.java | 2 +- .../index/impl/EsEntityIndexBatchImpl.java | 5 +- .../index/impl/EsEntityIndexImpl.java | 59 +++++++++++++++----- .../persistence/index/impl/IndexingUtils.java | 25 +++++---- 8 files changed, 80 insertions(+), 30 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a87ca3a1/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java index 789e640..8787410 100644 --- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java +++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java @@ -36,6 +36,7 @@ import java.util.UUID; import com.codahale.metrics.Meter; import org.apache.usergrid.persistence.collection.FieldSet; import org.apache.usergrid.persistence.core.future.BetterFuture; +import org.elasticsearch.action.ListenableActionFuture; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.util.Assert; @@ -2899,9 +2900,9 @@ public class CpEntityManager implements EntityManager { ei.initializeIndex(); } - public void deleteIndex(){ + public ListenableActionFuture deleteIndex(){ EntityIndex ei = managerCache.getEntityIndex( applicationScope ); - ei.deleteIndex(); + return ei.deleteIndex(); } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a87ca3a1/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java index 65fac8d..2a951aa 100644 --- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java +++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java @@ -34,6 +34,7 @@ import org.apache.usergrid.persistence.entities.Role; import org.apache.usergrid.persistence.index.query.CounterResolution; import org.apache.usergrid.persistence.index.query.Identifier; import org.apache.usergrid.persistence.index.query.Query.Level; +import org.elasticsearch.action.ListenableActionFuture; /** @@ -696,7 +697,7 @@ public interface EntityManager { /** * Create the index, should ONLY ever be called the first time an application is created */ - void deleteIndex(); + ListenableActionFuture deleteIndex(); public void init( EntityManagerFactory emf, UUID applicationId); http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a87ca3a1/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java ---------------------------------------------------------------------- diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java index ce45ebf..966a601 100644 --- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java +++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java @@ -36,6 +36,7 @@ import java.util.UUID; import javax.annotation.Resource; +import org.elasticsearch.action.ListenableActionFuture; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationContext; @@ -2895,8 +2896,9 @@ public class EntityManagerImpl implements EntityManager { @Override - public void deleteIndex() { + public ListenableActionFuture deleteIndex() { //no op + return null; } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a87ca3a1/stack/core/src/test/java/org/apache/usergrid/CoreApplication.java ---------------------------------------------------------------------- diff --git a/stack/core/src/test/java/org/apache/usergrid/CoreApplication.java b/stack/core/src/test/java/org/apache/usergrid/CoreApplication.java index 99cfcf5..a48ce4e 100644 --- a/stack/core/src/test/java/org/apache/usergrid/CoreApplication.java +++ b/stack/core/src/test/java/org/apache/usergrid/CoreApplication.java @@ -20,6 +20,7 @@ package org.apache.usergrid; import java.util.LinkedHashMap; import java.util.Map; import java.util.UUID; +import java.util.concurrent.ExecutionException; import org.junit.rules.TestRule; import org.junit.runner.Description; @@ -144,7 +145,11 @@ public class CoreApplication implements Application, TestRule { protected void after( Description description ) { LOG.info( "Test {}: finish with application", description.getDisplayName() ); - setup.getEmf().getEntityManager( id ).deleteIndex(); + try { + setup.getEmf().getEntityManager(id).deleteIndex().get(); + }catch (Exception ee){ + throw new RuntimeException(ee); + } } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a87ca3a1/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java index c8840fa..db4b50d 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java @@ -45,7 +45,7 @@ public interface EntityIndex { /** * Delete the index from ES */ - public void deleteIndex(); + public ListenableActionFuture deleteIndex(); /** * Create an index and add to alias, will create alias and remove any old index from write alias if alias already exists http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a87ca3a1/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java index c0d0d1e..d7b1f9c 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java @@ -94,7 +94,10 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch { IndexValidationUtils.validateIndexScope( indexScope ); ValidationUtils.verifyEntityWrite( entity ); ValidationUtils.verifyVersion( entity.getVersion() ); - + //add app id for indexing + entity.setField( + new StringField(APPLICATION_ID_FIELDNAME,IndexingUtils.idString(applicationScope.getApplication())) + ); final String context = createContextName(indexScope); if ( log.isDebugEnabled() ) { http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a87ca3a1/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java index 5e23d67..0ecc31b 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java @@ -265,7 +265,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex { String indexName = indexIdentifier.getIndex(indexSuffix); final AdminClient adminClient = esProvider.getClient().admin(); - String[] indexNames = getIndexesFromEs( AliasType.Write ); + String[] indexNames = getIndexesFromEs(AliasType.Write); final IndicesAliasesRequestBuilder aliasesRequestBuilder = adminClient.indices().prepareAliases(); @@ -280,7 +280,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex { // add read alias aliasesRequestBuilder.addAlias( indexName, alias.getReadAlias()); - logger.info( "Created new read Alias Name [{}] on Index [{}]", alias.getReadAlias(), indexName); + logger.info("Created new read Alias Name [{}] on Index [{}]", alias.getReadAlias(), indexName); //add write alias @@ -623,23 +623,56 @@ public class EsEntityIndexImpl implements AliasedEntityIndex { /** * Completely delete an index. */ - public void deleteIndex() { - AdminClient adminClient = esProvider.getClient().admin(); + public ListenableActionFuture deleteIndex() { + String idString = IndexingUtils.idString(applicationScope.getApplication()); - DeleteIndexResponse response = adminClient.indices() - .prepareDelete( indexIdentifier.getIndex(null) ).get(); + final TermQueryBuilder tqb = QueryBuilders.termQuery(APPLICATION_ID_FIELDNAME, idString); + + //Added For Graphite Metrics + + final ListenableActionFuture<DeleteByQueryResponse> response = esProvider.getClient() + .prepareDeleteByQuery( alias.getWriteAlias() ).setQuery( tqb ).execute(); + + response.addListener(new ActionListener<DeleteByQueryResponse>() { + + @Override + public void onResponse(DeleteByQueryResponse response) { + checkDeleteByQueryResponse(tqb,response); + } + + + @Override + public void onFailure(Throwable e) { + logger.error("failed on delete index",e); + } + }); + return response; + } + /** + * Validate the response doesn't contain errors, if it does, fail fast at the first error we encounter + */ + private void checkDeleteByQueryResponse( + final QueryBuilder query, final DeleteByQueryResponse response ) { + + for ( IndexDeleteByQueryResponse indexDeleteByQueryResponse : response ) { + final ShardOperationFailedException[] failures = indexDeleteByQueryResponse.getFailures(); + + for ( ShardOperationFailedException failedException : failures ) { + logger.error( String.format("Unable to delete by query %s. " + + "Failed with code %d and reason %s on shard %s in index %s", + query.toString(), + failedException.status().getStatus(), + failedException.reason(), + failedException.shardId(), + failedException.index() ) + ); + } - if ( response.isAcknowledged() ) { - logger.info( "Deleted index: read {} write {}", alias.getReadAlias(), alias.getWriteAlias()); - //invlaidate the alias - aliasCache.invalidate(alias); - } - else { - logger.info( "Failed to delete index: read {} write {}", alias.getReadAlias(), alias.getWriteAlias()); } } + /** * Do the retry operation */ http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a87ca3a1/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java index aa6d0f3..d49d8cc 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java @@ -50,6 +50,7 @@ public class IndexingUtils { // // Reserved UG fields. // + public static final String APPLICATION_ID_FIELDNAME = "ug_applicationId"; public static final String ENTITY_CONTEXT_FIELDNAME = "ug_context"; @@ -162,23 +163,27 @@ public class IndexingUtils { .endObject() .startArray("dynamic_templates") // we need most specific mappings first since it's a stop on match algorithm - .startObject() - .startObject("entity_id_template") - .field("match", IndexingUtils.ENTITYID_ID_FIELDNAME) + .startObject() + .startObject("application_id_template") + .field("match", APPLICATION_ID_FIELDNAME) .field("match_mapping_type", "string") - .startObject("mapping").field("type", "string") - .field("index", "not_analyzed") - .endObject() - .endObject() + .startObject("mapping").field("type", "string").field("index", "not_analyzed").field(DOC_VALUES_KEY, true).endObject() + .endObject() + .endObject() + .startObject() + .startObject("entity_id_template") + .field("match", IndexingUtils.ENTITYID_ID_FIELDNAME) + .field("match_mapping_type", "string") + .startObject("mapping").field("type", "string").field("index", "not_analyzed").field(DOC_VALUES_KEY, true).endObject() .endObject() + .endObject() .startObject() .startObject("entity_context_template") .field("match", IndexingUtils.ENTITY_CONTEXT_FIELDNAME) .field("match_mapping_type", "string") .startObject("mapping") - .field("type", "string") - .field("index", "not_analyzed") + .field("type", "string").field("index", "not_analyzed").field(DOC_VALUES_KEY, true) .endObject() .endObject() .endObject() @@ -187,7 +192,7 @@ public class IndexingUtils { .startObject("entity_version_template") .field("match", IndexingUtils.ENTITY_VERSION_FIELDNAME) .field("match_mapping_type", "string") - .startObject("mapping").field("type", "long").field(DOC_VALUES_KEY, true) + .startObject("mapping").field("type", "long").field("index", "not_analyzed").field(DOC_VALUES_KEY, true) .endObject() .endObject() .endObject()
