This is an automated email from the ASF dual-hosted git repository. jsinovassinnaik pushed a commit to branch UNOMI-728-index-migration in repository https://gitbox.apache.org/repos/asf/unomi.git
commit a112841dd37426120f4eef29689d2ddeb6e6aa9a Author: jsinovassin <[email protected]> AuthorDate: Thu Feb 9 18:04:49 2023 +0100 feedbacks --- .../unomi/itests/migration/Migrate16xTo220IT.java | 4 +- ...migrate-2.2.0-00-rolloverAndMigrateEvent.groovy | 74 ---------------------- ...2.2.0-00-rolloverAndMigrateEventSession.groovy} | 54 ++++++++++++---- ...g.json => base_index_withRollover_request.json} | 0 4 files changed, 43 insertions(+), 89 deletions(-) diff --git a/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo220IT.java b/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo220IT.java index 63e84ff58..fba7bbda0 100644 --- a/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo220IT.java +++ b/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo220IT.java @@ -84,13 +84,13 @@ public class Migrate16xTo220IT extends BaseIT { checkViewEventRestructured(); checkEventTypesNotPersistedAnymore(); checkForMappingUpdates(); - checkNewIndexesExists(); + checkEventSessionRollover2_2_0(); } /** * Checks if at least the new index for events and sessions exists. */ - private void checkNewIndexesExists() throws IOException { + private void checkEventSessionRollover2_2_0() throws IOException { Assert.assertTrue(MigrationUtils.indexExists(httpClient, "http://localhost:9400";, "context-event-000001")); Assert.assertTrue(MigrationUtils.indexExists(httpClient, "http://localhost:9400";, "context-session-000001")); } diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-00-rolloverAndMigrateEvent.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-00-rolloverAndMigrateEvent.groovy deleted file mode 100644 index 99fcf9e02..000000000 --- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-00-rolloverAndMigrateEvent.groovy +++ /dev/null @@ -1,74 +0,0 @@ -import org.apache.unomi.shell.migration.service.MigrationContext -import org.apache.unomi.shell.migration.utils.HttpUtils -import org.apache.unomi.shell.migration.utils.MigrationUtils - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -MigrationContext context = migrationContext -String esAddress = context.getConfigString("esAddress") -String indexPrefix = context.getConfigString("indexPrefix") -String newEventIndex = indexPrefix + "-event-000001" -String rolloverPolicyName = indexPrefix + "-unomi-rollover-policy" -String rolloverEventAlias = indexPrefix + "-event" - - -context.performMigrationStep("2.2.0-00-update-lifecyle-poll-interval", () -> { - String updatePollIntervalBody = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.2.0/update_settings_poll_interval.json") - .replace("#pollIntervalValue", "\"2s\"") - HttpUtils.executePutRequest(context.getHttpClient(), esAddress + "/_cluster/settings", updatePollIntervalBody, null) -}) - -context.performMigrationStep("2.2.0-00-create-rollover-policy", () -> { - String createRolloverPolicyQuery = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.2.0/create_rollover_policy_query.json") - String rolloverQueryBody = MigrationUtils.buildRolloverPolicyCreationRequest(createRolloverPolicyQuery, context) - - HttpUtils.executePutRequest(context.getHttpClient(), esAddress + "/_ilm/policy/" + rolloverPolicyName, rolloverQueryBody, null) -}) - -context.performMigrationStep("2.2.0-00-create-event-index", () -> { - if (!MigrationUtils.indexExists(context.getHttpClient(), esAddress, newEventIndex)) { - String baseRequest = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.2.0/base_index_mapping.json") - String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "event.json") - - String newIndexSettings = MigrationUtils.buildIndexCreationRequestWithRollover(baseRequest, mapping, context, rolloverPolicyName, rolloverEventAlias) - HttpUtils.executePutRequest(context.getHttpClient(), esAddress + "/" + newEventIndex, newIndexSettings, null) - } -}) - -Set<String> eventIndices = MigrationUtils.getIndexesPrefixedBy(context.getHttpClient(), esAddress, indexPrefix + "-event-date-") -List<String> sortedIndices = new ArrayList<>(eventIndices) -Collections.sort(sortedIndices) - -context.performMigrationStep("2.2.0-00-migrate-existing-events", () -> { - sortedIndices.each { eventIndex -> - MigrationUtils.moveToIndex(context.getHttpClient(), bundleContext, esAddress, eventIndex, indexPrefix + "-event") - sleep(3000) - } -}) - -context.performMigrationStep("2.2.0-00-remove-old-events-indices", () -> { - sortedIndices.each { eventIndex -> - MigrationUtils.deleteIndex(context.getHttpClient(), esAddress, eventIndex) - } -}) - -context.performMigrationStep("2.2.0-00-reset-poll-interval", () -> { - String updatePollIntervalBody = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.2.0/update_settings_poll_interval.json") - .replace("#pollIntervalValue", "null") - HttpUtils.executePutRequest(context.getHttpClient(), esAddress + "/_cluster/settings", updatePollIntervalBody, null) -}) diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-10-rolloverAndMigrateSession.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-00-rolloverAndMigrateEventSession.groovy similarity index 60% rename from tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-10-rolloverAndMigrateSession.groovy rename to tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-00-rolloverAndMigrateEventSession.groovy index a553ae808..d3a6202ad 100644 --- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-10-rolloverAndMigrateSession.groovy +++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-00-rolloverAndMigrateEventSession.groovy @@ -22,27 +22,55 @@ import org.apache.unomi.shell.migration.utils.MigrationUtils MigrationContext context = migrationContext String esAddress = context.getConfigString("esAddress") String indexPrefix = context.getConfigString("indexPrefix") -String newSessionIndex = indexPrefix + "-session-000001" +String newEventIndex = indexPrefix + "-event-000001" String rolloverPolicyName = indexPrefix + "-unomi-rollover-policy" +String rolloverEventAlias = indexPrefix + "-event" +String newSessionIndex = indexPrefix + "-session-000001" String rolloverSessionAlias = indexPrefix + "-session" - -context.performMigrationStep("2.2.0-10-update-lifecyle-poll-interval", () -> { +context.performMigrationStep("2.2.0-update-lifecyle-poll-interval", () -> { String updatePollIntervalBody = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.2.0/update_settings_poll_interval.json") .replace("#pollIntervalValue", "\"2s\"") HttpUtils.executePutRequest(context.getHttpClient(), esAddress + "/_cluster/settings", updatePollIntervalBody, null) }) -context.performMigrationStep("2.2.0-10-create-rollover-policy", () -> { +context.performMigrationStep("2.2.0-create-rollover-policy", () -> { String createRolloverPolicyQuery = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.2.0/create_rollover_policy_query.json") String rolloverQueryBody = MigrationUtils.buildRolloverPolicyCreationRequest(createRolloverPolicyQuery, context) HttpUtils.executePutRequest(context.getHttpClient(), esAddress + "/_ilm/policy/" + rolloverPolicyName, rolloverQueryBody, null) }) -context.performMigrationStep("2.2.0-10-create-session-index", () -> { +context.performMigrationStep("2.2.0-create-event-index", () -> { + if (!MigrationUtils.indexExists(context.getHttpClient(), esAddress, newEventIndex)) { + String baseRequest = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.2.0/base_index_withRollover_request.json") + String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "event.json") + + String newIndexSettings = MigrationUtils.buildIndexCreationRequestWithRollover(baseRequest, mapping, context, rolloverPolicyName, rolloverEventAlias) + HttpUtils.executePutRequest(context.getHttpClient(), esAddress + "/" + newEventIndex, newIndexSettings, null) + } +}) + +Set<String> eventIndices = MigrationUtils.getIndexesPrefixedBy(context.getHttpClient(), esAddress, indexPrefix + "-event-date-") +List<String> eventSortedIndices = new ArrayList<>(eventIndices) +Collections.sort(eventSortedIndices) + +context.performMigrationStep("2.2.0-migrate-existing-events", () -> { + eventSortedIndices.each { eventIndex -> + MigrationUtils.moveToIndex(context.getHttpClient(), bundleContext, esAddress, eventIndex, indexPrefix + "-event") + sleep(3000) + } +}) + +context.performMigrationStep("2.2.0-remove-old-events-indices", () -> { + eventSortedIndices.each { eventIndex -> + MigrationUtils.deleteIndex(context.getHttpClient(), esAddress, eventIndex) + } +}) + +context.performMigrationStep("2.2.0-create-session-index", () -> { if (!MigrationUtils.indexExists(context.getHttpClient(), esAddress, newSessionIndex)) { - String baseRequest = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.2.0/base_index_mapping.json") + String baseRequest = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.2.0/base_index_withRollover_request.json") String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "session.json") String newIndexSettings = MigrationUtils.buildIndexCreationRequestWithRollover(baseRequest, mapping, context, rolloverPolicyName, rolloverSessionAlias) @@ -51,23 +79,23 @@ context.performMigrationStep("2.2.0-10-create-session-index", () -> { }) Set<String> sessionIndices = MigrationUtils.getIndexesPrefixedBy(context.getHttpClient(), esAddress, indexPrefix + "-session-date-") -List<String> sortedIndices = new ArrayList<>(sessionIndices) -Collections.sort(sortedIndices) +List<String> sessionSortedIndices = new ArrayList<>(sessionIndices) +Collections.sort(sessionSortedIndices) -context.performMigrationStep("2.2.0-10-migrate-existing-sessions", () -> { - sortedIndices.each { sessionIndex -> +context.performMigrationStep("2.2.0-migrate-existing-sessions", () -> { + sessionSortedIndices.each { sessionIndex -> MigrationUtils.moveToIndex(context.getHttpClient(), bundleContext, esAddress, sessionIndex, indexPrefix + "-session") sleep(3000) } }) -context.performMigrationStep("2.2.0-10-remove-old-sessions-indices", () -> { - sortedIndices.each { sessionIndex -> +context.performMigrationStep("2.2.0-remove-old-sessions-indices", () -> { + sessionSortedIndices.each { sessionIndex -> MigrationUtils.deleteIndex(context.getHttpClient(), esAddress, sessionIndex) } }) -context.performMigrationStep("2.2.0-10-reset-poll-interval", () -> { +context.performMigrationStep("2.2.0-reset-poll-interval", () -> { String updatePollIntervalBody = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.2.0/update_settings_poll_interval.json") .replace("#pollIntervalValue", "null") HttpUtils.executePutRequest(context.getHttpClient(), esAddress + "/_cluster/settings", updatePollIntervalBody, null) diff --git a/tools/shell-commands/src/main/resources/requestBody/2.2.0/base_index_mapping.json b/tools/shell-commands/src/main/resources/requestBody/2.2.0/base_index_withRollover_request.json similarity index 100% rename from tools/shell-commands/src/main/resources/requestBody/2.2.0/base_index_mapping.json rename to tools/shell-commands/src/main/resources/requestBody/2.2.0/base_index_withRollover_request.json
