add data migration
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/f695724b Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/f695724b Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/f695724b Branch: refs/heads/two-dot-o-dev Commit: f695724b0417131eb9592a642ae8fa7a29afd8cd Parents: 8a62d11 Author: Shawn Feldman <[email protected]> Authored: Thu Mar 26 10:14:12 2015 -0600 Committer: Shawn Feldman <[email protected]> Committed: Thu Mar 26 10:14:12 2015 -0600 ---------------------------------------------------------------------- .../usergrid/persistence/index/IndexAlias.java | 38 ++++++ .../persistence/index/IndexIdentifier.java | 18 --- .../persistence/index/guice/IndexModule.java | 19 +++ .../impl/EsApplicationEntityIndexImpl.java | 2 +- .../index/impl/EsEntityIndexImpl.java | 2 +- .../persistence/index/impl/EsIndexCache.java | 7 +- .../impl/SearchRequestBuilderStrategy.java | 5 +- .../migration/EsIndexDataMigrationImpl.java | 134 +++++++++++++++++++ .../index/migration/EsIndexMigrationPlugin.java | 46 +++++++ .../index/migration/IndexMigration.java | 35 +++++ 10 files changed, 281 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f695724b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexAlias.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexAlias.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexAlias.java new file mode 100644 index 0000000..a04f80e --- /dev/null +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexAlias.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.usergrid.persistence.index; + +/** + * Abstraction for Index alias names + */ +public class IndexAlias{ + private final String readAlias; + private final String writeAlias; + + public IndexAlias(IndexFig indexFig,String indexBase) { + this.writeAlias = indexBase + "_write_" + indexFig.getAliasPostfix(); + this.readAlias = indexBase + "_read_" + indexFig.getAliasPostfix(); + } + + public String getReadAlias() { + return readAlias; + } + + public String getWriteAlias() { + return writeAlias; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f695724b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java index c659ed6..48c48f3 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java @@ -55,24 +55,6 @@ public class IndexIdentifier{ } - public class IndexAlias{ - private final String readAlias; - private final String writeAlias; - - public IndexAlias(IndexFig indexFig,String indexBase) { - this.writeAlias = indexBase + "_write_" + indexFig.getAliasPostfix(); - this.readAlias = indexBase + "_read_" + indexFig.getAliasPostfix(); - } - - public String getReadAlias() { - return readAlias; - } - - public String getWriteAlias() { - return writeAlias; - } - } - public String toString() { return "index id"+config.getIndexPrefix(); } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f695724b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java index c9125c5..a42dea8 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java @@ -19,6 +19,11 @@ package org.apache.usergrid.persistence.index.guice; +import com.google.inject.TypeLiteral; +import com.google.inject.multibindings.Multibinder; +import org.apache.usergrid.persistence.core.migration.data.DataMigration; +import org.apache.usergrid.persistence.core.migration.data.MigrationPlugin; +import org.apache.usergrid.persistence.core.scope.ApplicationScope; import org.apache.usergrid.persistence.index.*; import com.google.inject.AbstractModule; import com.google.inject.assistedinject.FactoryModuleBuilder; @@ -28,6 +33,9 @@ import org.apache.usergrid.persistence.index.impl.EsEntityIndexFactoryImpl; import org.apache.usergrid.persistence.index.impl.EsEntityIndexImpl; import org.apache.usergrid.persistence.index.impl.EsIndexBufferConsumerImpl; import org.apache.usergrid.persistence.index.impl.EsIndexBufferProducerImpl; +import org.apache.usergrid.persistence.index.migration.EsIndexDataMigrationImpl; +import org.apache.usergrid.persistence.index.migration.EsIndexMigrationPlugin; +import org.apache.usergrid.persistence.index.migration.IndexMigration; import org.apache.usergrid.persistence.map.guice.MapModule; import org.apache.usergrid.persistence.queue.guice.QueueModule; @@ -56,6 +64,17 @@ public class IndexModule extends AbstractModule { bind( BufferQueue.class).toProvider( QueueProvider.class ); + + //wire up the edg migration + Multibinder<DataMigration<ApplicationScope>> dataMigrationMultibinder = + Multibinder.newSetBinder( binder(), new TypeLiteral<DataMigration<ApplicationScope>>() {}, IndexMigration.class ); + + + dataMigrationMultibinder.addBinding().to(EsIndexDataMigrationImpl.class); + + + //wire up the collection migration plugin + Multibinder.newSetBinder( binder(), MigrationPlugin.class ).addBinding().to(EsIndexMigrationPlugin.class); } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f695724b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java index ca071e2..27791ae 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java @@ -76,7 +76,7 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex{ private final EsIndexCache indexCache; private final IndexFig indexFig; private final EsProvider esProvider; - private final IndexIdentifier.IndexAlias alias; + private final IndexAlias alias; private final Timer deleteApplicationTimer; private final Meter deleteApplicationMeter; private final SearchRequestBuilderStrategy searchRequest; http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f695724b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java index 259fa55..cab8ded 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java @@ -77,7 +77,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex { public static final String DEFAULT_TYPE = "_default_"; - private final IndexIdentifier.IndexAlias alias; + private final IndexAlias alias; private final IndexBufferProducer indexBatchBufferProducer; private final IndexFig indexFig; private final Timer addTimer; http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f695724b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java index ef518dd..9ba89ca 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexCache.java @@ -27,6 +27,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import org.apache.usergrid.persistence.index.IndexAlias; import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; import org.elasticsearch.client.AdminClient; import org.elasticsearch.cluster.metadata.AliasMetaData; @@ -98,7 +99,7 @@ public class EsIndexCache { /** * Get indexes for an alias */ - public String[] getIndexes( IndexIdentifier.IndexAlias alias, AliasedEntityIndex.AliasType aliasType ) { + public String[] getIndexes( IndexAlias alias, AliasedEntityIndex.AliasType aliasType ) { String[] indexes; try { indexes = aliasIndexCache.get( getAliasName( alias, aliasType ) ); @@ -127,7 +128,7 @@ public class EsIndexCache { * @param aliasType * @return */ - private String getAliasName( IndexIdentifier.IndexAlias alias, AliasedEntityIndex.AliasType aliasType ) { + private String getAliasName( IndexAlias alias, AliasedEntityIndex.AliasType aliasType ) { return aliasType == AliasedEntityIndex.AliasType.Read ? alias.getReadAlias() : alias.getWriteAlias(); } @@ -135,7 +136,7 @@ public class EsIndexCache { /** * clean up cache */ - public void invalidate( IndexIdentifier.IndexAlias alias ) { + public void invalidate( IndexAlias alias ) { aliasIndexCache.invalidate( alias.getWriteAlias() ); aliasIndexCache.invalidate( alias.getReadAlias() ); } http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f695724b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java index d146e4c..15796f9 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java @@ -22,6 +22,7 @@ package org.apache.usergrid.persistence.index.impl; import com.google.common.base.Preconditions; import org.apache.usergrid.persistence.core.scope.ApplicationScope; import org.apache.usergrid.persistence.core.util.ValidationUtils; +import org.apache.usergrid.persistence.index.IndexAlias; import org.apache.usergrid.persistence.index.IndexIdentifier; import org.apache.usergrid.persistence.index.IndexScope; import org.apache.usergrid.persistence.index.SearchTypes; @@ -53,11 +54,11 @@ public class SearchRequestBuilderStrategy { private final EsProvider esProvider; private final ApplicationScope applicationScope; - private final IndexIdentifier.IndexAlias alias; + private final IndexAlias alias; private final int cursorTimeout; public static final int MAX_LIMIT = 1000; - public SearchRequestBuilderStrategy(final EsProvider esProvider, final ApplicationScope applicationScope, final IndexIdentifier.IndexAlias alias, int cursorTimeout){ + public SearchRequestBuilderStrategy(final EsProvider esProvider, final ApplicationScope applicationScope, final IndexAlias alias, int cursorTimeout){ this.esProvider = esProvider; this.applicationScope = applicationScope; http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f695724b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java new file mode 100644 index 0000000..b5dab53 --- /dev/null +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.usergrid.persistence.index.migration; + +import com.google.inject.Inject; +import org.apache.usergrid.persistence.core.migration.data.DataMigration; +import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider; +import org.apache.usergrid.persistence.core.migration.data.ProgressObserver; +import org.apache.usergrid.persistence.core.scope.ApplicationScope; +import org.apache.usergrid.persistence.index.AliasedEntityIndex; +import org.apache.usergrid.persistence.index.IndexAlias; +import org.apache.usergrid.persistence.index.IndexFig; +import org.apache.usergrid.persistence.index.IndexIdentifier; +import org.apache.usergrid.persistence.index.impl.EsIndexCache; +import org.apache.usergrid.persistence.index.impl.EsProvider; +import org.apache.usergrid.persistence.index.impl.IndexingUtils; +import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder; +import org.elasticsearch.client.AdminClient; + +import java.util.ArrayList; +import java.util.List; + +/** + * Classy class class. + */ +public class EsIndexDataMigrationImpl implements DataMigration<ApplicationScope> { + + private final AliasedEntityIndex entityIndex; + private final EsProvider provider; + private final IndexFig indexFig; + private final IndexIdentifier indexIdentifier; + private final EsIndexCache indexCache; + + @Inject + public EsIndexDataMigrationImpl(AliasedEntityIndex entityIndex, EsProvider provider, IndexFig indexFig, IndexIdentifier indexIdentifier, EsIndexCache indexCache){ + this.entityIndex = entityIndex; + this.provider = provider; + this.indexFig = indexFig; + this.indexIdentifier = indexIdentifier; + this.indexCache = indexCache; + } + + @Override + public int migrate(int currentVersion, MigrationDataProvider<ApplicationScope> migrationDataProvider, ProgressObserver observer) { + migrationDataProvider.getData().doOnNext(applicationScope -> { + LegacyIndexIdentifier legacyIndexIdentifier = new LegacyIndexIdentifier(indexFig,applicationScope); + String[] indexes = indexCache.getIndexes(legacyIndexIdentifier.getAlias(), AliasedEntityIndex.AliasType.Read); + AdminClient adminClient = provider.getClient().admin(); + + for (String index : indexes) { + IndicesAliasesRequestBuilder aliasesRequestBuilder = adminClient.indices().prepareAliases(); + aliasesRequestBuilder = adminClient.indices().prepareAliases(); + // add read alias + aliasesRequestBuilder.addAlias(index, indexIdentifier.getAlias().getReadAlias()); + } + }); + return 0; + } + + @Override + public boolean supports(int currentVersion) { + return false; + } + + @Override + public int getMaxVersion() { + return 0; + } + /** + * Class is used to generate an index name and alias name the old way via app name + */ + public class LegacyIndexIdentifier{ + private final IndexFig config; + private final ApplicationScope applicationScope; + + public LegacyIndexIdentifier(IndexFig config, ApplicationScope applicationScope) { + this.config = config; + this.applicationScope = applicationScope; + } + + /** + * Get the alias name + * @return + */ + public IndexAlias getAlias() { + return new IndexAlias(config,getIndexBase()); + } + + /** + * Get index name, send in additional parameter to add incremental indexes + * @param suffix + * @return + */ + public String getIndex(String suffix) { + if (suffix != null) { + return getIndexBase() + "_" + suffix; + } else { + return getIndexBase(); + } + } + + /** + * returns the base name for index which will be used to add an alias and index + * @return + */ + private String getIndexBase() { + StringBuilder sb = new StringBuilder(); + sb.append(config.getIndexPrefix()).append(IndexingUtils.SEPARATOR); + IndexingUtils.idString(sb, applicationScope.getApplication()); + return sb.toString(); + } + + + + public String toString() { + return "application: " + applicationScope.getApplication().getUuid(); + } + + } +} http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f695724b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexMigrationPlugin.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexMigrationPlugin.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexMigrationPlugin.java new file mode 100644 index 0000000..a28c701 --- /dev/null +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexMigrationPlugin.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.usergrid.persistence.index.migration; + +import com.google.inject.Inject; +import org.apache.usergrid.persistence.core.migration.data.*; +import org.apache.usergrid.persistence.core.scope.ApplicationScope; + +import java.util.Set; + +/** + * Classy class class. + */ +public class EsIndexMigrationPlugin extends AbstractMigrationPlugin<ApplicationScope>{ + + @Inject + public EsIndexMigrationPlugin(@IndexMigration final Set<DataMigration<ApplicationScope>> entityDataMigrations, + final MigrationDataProvider<ApplicationScope> entityIdScopeDataMigrationProvider, + final MigrationInfoSerialization migrationInfoSerialization ){ + super(entityDataMigrations,entityIdScopeDataMigrationProvider,migrationInfoSerialization); + } + + @Override + public String getName() { + return "index-migration"; + } + + @Override + public PluginPhase getPhase() { + return PluginPhase.MIGRATE; + } +} http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f695724b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/IndexMigration.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/IndexMigration.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/IndexMigration.java new file mode 100644 index 0000000..c398aac --- /dev/null +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/IndexMigration.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.usergrid.persistence.index.migration; + +import com.google.inject.BindingAnnotation; + +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import static java.lang.annotation.ElementType.FIELD; +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.ElementType.PARAMETER; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +/** + * Index migration annotation + */ + +@BindingAnnotation +@Target({ FIELD, PARAMETER, METHOD }) @Retention(RUNTIME) +public @interface IndexMigration {}
