migration
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/c47b02d4 Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/c47b02d4 Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/c47b02d4 Branch: refs/heads/two-dot-o-dev Commit: c47b02d41910c524957149dbf2bf8f4a3dff051d Parents: 81e0fba Author: Shawn Feldman <[email protected]> Authored: Fri Aug 21 19:20:26 2015 -0700 Committer: Shawn Feldman <[email protected]> Committed: Fri Aug 21 19:20:26 2015 -0700 ---------------------------------------------------------------------- .../persistence/index/guice/IndexModule.java | 6 +- .../EsIndexMappingMigrationPlugin.java | 148 +++++++++++++++++++ .../impl/EsIndexMappingMigrationPluginTest.java | 60 ++++++++ 3 files changed, 212 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/usergrid/blob/c47b02d4/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java index b22a461..d03bbf8 100644 --- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java @@ -28,6 +28,7 @@ import org.apache.usergrid.persistence.index.*; import com.google.inject.AbstractModule; import org.apache.usergrid.persistence.index.impl.*; +import org.apache.usergrid.persistence.index.migration.EsIndexMappingMigrationPlugin; import org.apache.usergrid.persistence.index.migration.EsIndexMigrationPlugin; import org.apache.usergrid.persistence.index.migration.IndexMigration; import org.apache.usergrid.persistence.map.guice.MapModule; @@ -61,8 +62,9 @@ public abstract class IndexModule extends AbstractModule { //wire up the collection migration plugin - Multibinder.newSetBinder( binder(), MigrationPlugin.class ).addBinding().to(EsIndexMigrationPlugin.class); - + final Multibinder<MigrationPlugin> plugins = Multibinder.newSetBinder(binder(), MigrationPlugin.class); + plugins.addBinding().to(EsIndexMigrationPlugin.class); + plugins.addBinding().to(EsIndexMappingMigrationPlugin.class); //invoke the migration plugin config configureMigrationProvider(); http://git-wip-us.apache.org/repos/asf/usergrid/blob/c47b02d4/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexMappingMigrationPlugin.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexMappingMigrationPlugin.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexMappingMigrationPlugin.java new file mode 100644 index 0000000..10acc46 --- /dev/null +++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexMappingMigrationPlugin.java @@ -0,0 +1,148 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one or more + * * contributor license agreements. The ASF licenses this file to You + * * under the Apache License, Version 2.0 (the "License"); you may not + * * use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. For additional information regarding + * * copyright in this work, please see the NOTICE file in the top level + * * directory of this distribution. + * + */ +package org.apache.usergrid.persistence.index.migration; + +import com.google.common.base.Charsets; +import com.google.common.io.Resources; +import com.google.inject.Inject; +import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerialization; +import org.apache.usergrid.persistence.core.migration.data.MigrationPlugin; +import org.apache.usergrid.persistence.core.migration.data.PluginPhase; +import org.apache.usergrid.persistence.core.migration.data.ProgressObserver; +import org.apache.usergrid.persistence.index.impl.EsProvider; +import org.elasticsearch.action.ActionFuture; +import org.elasticsearch.action.ListenableActionFuture; +import org.elasticsearch.action.admin.indices.get.GetIndexRequest; +import org.elasticsearch.action.admin.indices.get.GetIndexResponse; +import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import rx.Observable; + +import java.io.IOException; +import java.net.URL; +import java.util.Arrays; +import java.util.List; + +/** + * Classy class class. + */ +public class EsIndexMappingMigrationPlugin implements MigrationPlugin { + + private static final Logger logger = LoggerFactory.getLogger(EsIndexMappingMigrationPlugin.class); + + private final MigrationInfoSerialization migrationInfoSerialization; + private final EsProvider provider; + + @Inject + public EsIndexMappingMigrationPlugin( + final MigrationInfoSerialization migrationInfoSerialization, + final EsProvider provider + ){ + + this.migrationInfoSerialization = migrationInfoSerialization; + this.provider = provider; + } + + @Override + public String getName() { + return "index_mapping_migration"; + } + + @Override + public void run(ProgressObserver observer) { + + final int version = migrationInfoSerialization.getVersion(getName()); + + if (version == getMaxVersion()) { + logger.debug("Skipping Migration Plugin: " + getName()); + return; + } + + try { + ActionFuture<GetIndexResponse> responseFuture = provider.getClient().admin().indices().getIndex(new GetIndexRequest()); + Observable + .from(responseFuture) + .flatMap(response -> { + List<String> indices = Arrays.asList(response.getIndices()); + return Observable.from(indices); + }) + + .doOnNext(index -> { + createMappings(index); + observer.update(getMaxVersion(), "running update for " + index); + }) + .doOnError(t -> { + observer.failed(getMaxVersion(),"failed to update",t); + }) + .doOnCompleted(() -> { + migrationInfoSerialization.setVersion(getName(), getMaxVersion()); + observer.complete(); + }) + .subscribe(); + + + }catch (Exception ee){ + observer.failed(getMaxVersion(),"failed to update",ee); + throw new RuntimeException(ee); + } + + } + + + /** + * Setup ElasticSearch type mappings as a template that applies to all new indexes. + * Applies to all indexes that* start with our prefix. + */ + private void createMappings(final String indexName) { + + //Added For Graphite Metrics + PutMappingResponse pitr = provider.getClient().admin().indices().preparePutMapping( indexName ).setType( "entity" ).setSource( + getMappingsContent() ).execute().actionGet(); + if ( !pitr.isAcknowledged() ) { + throw new RuntimeException( "Unable to create default mappings" ); + } + } + + + /** + * Get the content from our mappings file + * @return + */ + private String getMappingsContent(){ + URL url = Resources.getResource("org/apache/usergrid/persistence/index/usergrid-mappings.json"); + try { + return Resources.toString(url, Charsets.UTF_8); + } + catch ( IOException e ) { + throw new RuntimeException( "Unable to read mappings file", e ); + } + } + + @Override + public int getMaxVersion() { + return 1; + } + + @Override + public PluginPhase getPhase() { + return PluginPhase.BOOTSTRAP; + } +} http://git-wip-us.apache.org/repos/asf/usergrid/blob/c47b02d4/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EsIndexMappingMigrationPluginTest.java ---------------------------------------------------------------------- diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EsIndexMappingMigrationPluginTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EsIndexMappingMigrationPluginTest.java new file mode 100644 index 0000000..e5d9380 --- /dev/null +++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EsIndexMappingMigrationPluginTest.java @@ -0,0 +1,60 @@ +/* + * + * * Licensed to the Apache Software Foundation (ASF) under one or more + * * contributor license agreements. The ASF licenses this file to You + * * under the Apache License, Version 2.0 (the "License"); you may not + * * use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. For additional information regarding + * * copyright in this work, please see the NOTICE file in the top level + * * directory of this distribution. + * + */ +package org.apache.usergrid.persistence.index.impl; + +import com.google.inject.Inject; +import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerialization; +import org.apache.usergrid.persistence.core.migration.data.ProgressObserver; +import org.apache.usergrid.persistence.core.migration.data.TestProgressObserver; +import org.apache.usergrid.persistence.core.test.UseModules; +import org.apache.usergrid.persistence.index.EntityIndexFactory; +import org.apache.usergrid.persistence.index.guice.TestIndexModule; +import org.apache.usergrid.persistence.index.migration.EsIndexMappingMigrationPlugin; +import org.junit.Test; +import org.junit.runner.RunWith; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Classy class class. + */ + +@RunWith( EsRunner.class ) +@UseModules( { TestIndexModule.class } ) +public class EsIndexMappingMigrationPluginTest extends BaseIT { + @Inject + public EntityIndexFactory eif; + @Inject + public MigrationInfoSerialization serialization; + @Inject + EsProvider provider; + @Test + public void runMigration(){ + EsIndexMappingMigrationPlugin plugin = new EsIndexMappingMigrationPlugin(serialization,provider); + TestProgressObserver progressObserver = new TestProgressObserver(); + plugin.run(progressObserver); + assertFalse( "Progress observer should not have failed", progressObserver.isFailed() ); + assertTrue("Progress observer should have update messages", progressObserver.getUpdates().size() > 0); + + + } + +}
