This is an automated email from the ASF dual-hosted git repository. jsinovassinnaik pushed a commit to branch UNOMI-728-index-migration in repository https://gitbox.apache.org/repos/asf/unomi.git
commit 01bc8d8ada069e7be5151f01852f2e241558bea2 Author: jsinovassin <[email protected]> AuthorDate: Thu Feb 9 18:04:49 2023 +0100 feedbacks --- .../unomi/itests/migration/Migrate16xTo220IT.java | 45 ++++++++++++- .../migration/match_all_body_request.json | 5 ++ .../must_not_match_some_eventype_body.json | 18 ++++++ .../shell/migration/utils/MigrationUtils.java | 25 ++++++-- ...migrate-2.2.0-00-rolloverAndMigrateEvent.groovy | 74 ---------------------- ...2.2.0-00-rolloverAndMigrateEventSession.groovy} | 56 ++++++++++++---- ...g.json => base_index_withRollover_request.json} | 0 .../requestBody/2.2.0/match_all_body_request.json | 5 ++ 8 files changed, 134 insertions(+), 94 deletions(-) diff --git a/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo220IT.java b/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo220IT.java index 63e84ff58..f2d9ff8a6 100644 --- a/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo220IT.java +++ b/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo220IT.java @@ -16,6 +16,7 @@ */ package org.apache.unomi.itests.migration; +import com.fasterxml.jackson.databind.JsonNode; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.unomi.api.*; import org.apache.unomi.itests.BaseIT; @@ -35,6 +36,9 @@ import java.util.Objects; public class Migrate16xTo220IT extends BaseIT { + private int eventCount = 0; + private int sessionCount = 0; + @Override @Before public void waitForStartup() throws InterruptedException { @@ -49,7 +53,9 @@ public class Migrate16xTo220IT extends BaseIT { throw new RuntimeException("Unable to retrieve 1.6.x snapshot for ES restore"); } // Restore the snapshot - HttpUtils.executePostRequest(httpClient, "http://localhost:9400/_snapshot/snapshots_repository/snapshot_1.6.x/_restore?wait_for_completion=true";, "{}", null); + HttpUtils.executePostRequest(httpClient, "http://localhost:9400/_snapshot/snapshots_repository/snapshot_1.6.x/_restore?wait_for_completion=true";, "{}", null); + fillNumberEventAndSessionBeforeMigration(httpClient); + } catch (IOException e) { throw new RuntimeException(e); } @@ -84,16 +90,31 @@ public class Migrate16xTo220IT extends BaseIT { checkViewEventRestructured(); checkEventTypesNotPersistedAnymore(); checkForMappingUpdates(); - checkNewIndexesExists(); + checkEventSessionRollover2_2_0(); } /** * Checks if at least the new index for events and sessions exists. */ - private void checkNewIndexesExists() throws IOException { + private void checkEventSessionRollover2_2_0() throws IOException { Assert.assertTrue(MigrationUtils.indexExists(httpClient, "http://localhost:9400";, "context-event-000001")); Assert.assertTrue(MigrationUtils.indexExists(httpClient, "http://localhost:9400";, "context-session-000001")); + + int newEventcount = 0; + for (String eventIndex : MigrationUtils.getIndexesPrefixedBy(httpClient, "http://localhost:9400";, "context-event-0")) { + JsonNode jsonNode = objectMapper.readTree(HttpUtils.executePostRequest(httpClient, "http://localhost:9400"; + "/" + eventIndex + "/_count", resourceAsString("migration/match_all_body_request.json"), null)); + newEventcount += jsonNode.get("count").asInt(); + } + + int newSessioncount = 0; + for (String eventIndex : MigrationUtils.getIndexesPrefixedBy(httpClient, "http://localhost:9400";, "context-session-0")) { + JsonNode jsonNode = objectMapper.readTree(HttpUtils.executePostRequest(httpClient, "http://localhost:9400"; + "/" + eventIndex + "/_count", resourceAsString("migration/match_all_body_request.json"), null)); + newSessioncount += jsonNode.get("count").asInt(); + } + Assert.assertEquals(eventCount, newEventcount); + Assert.assertEquals(sessionCount, newSessioncount); } + /** * Multiple index mappings have been update, check a simple check that after migration those mappings contains the latest modifications. */ @@ -274,4 +295,22 @@ public class Migrate16xTo220IT extends BaseIT { Assert.assertNotNull(persistenceService.load(masterProfile, Profile.class)); Assert.assertNull(persistenceService.load(masterProfile, ProfileAlias.class)); } + + private void fillNumberEventAndSessionBeforeMigration(CloseableHttpClient httpClient) { + try { + for (String eventIndex : MigrationUtils.getIndexesPrefixedBy(httpClient, "http://localhost:9400";, "context-event-date")) { + JsonNode jsonNode = objectMapper.readTree(HttpUtils.executePostRequest(httpClient, "http://localhost:9400"; + "/" + eventIndex + "/_count", resourceAsString("migration/must_not_match_some_eventype_body.json"), null)); + eventCount += jsonNode.get("count").asInt(); + } + + for (String eventIndex : MigrationUtils.getIndexesPrefixedBy(httpClient, "http://localhost:9400";, "context-session-date")) { + JsonNode jsonNode = objectMapper.readTree(HttpUtils.executePostRequest(httpClient, "http://localhost:9400"; + "/" + eventIndex + "/_count", resourceAsString("migration/match_all_body_request.json"), null)); + sessionCount += jsonNode.get("count").asInt(); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + + + } } diff --git a/itests/src/test/resources/migration/match_all_body_request.json b/itests/src/test/resources/migration/match_all_body_request.json new file mode 100644 index 000000000..487927ea1 --- /dev/null +++ b/itests/src/test/resources/migration/match_all_body_request.json @@ -0,0 +1,5 @@ +{ + "query": { + "match_all": {} + } +} \ No newline at end of file diff --git a/itests/src/test/resources/migration/must_not_match_some_eventype_body.json b/itests/src/test/resources/migration/must_not_match_some_eventype_body.json new file mode 100644 index 000000000..7fc953198 --- /dev/null +++ b/itests/src/test/resources/migration/must_not_match_some_eventype_body.json @@ -0,0 +1,18 @@ +{ + "query": { + "bool": { + "must_not": [ + { + "match": { + "eventType": "sessionCreated" + } + }, + { + "match": { + "eventType": "updateProperties" + } + } + ] + } + } +} \ No newline at end of file diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java index 19bb6af2e..a12fd6ec7 100644 --- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java +++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java @@ -32,10 +32,7 @@ import org.osgi.framework.BundleContext; import java.io.*; import java.net.URL; import java.nio.charset.StandardCharsets; -import java.util.Collections; -import java.util.Enumeration; -import java.util.Set; -import java.util.StringJoiner; +import java.util.*; import java.util.stream.Collectors; import static org.apache.unomi.shell.migration.service.MigrationConfig.*; @@ -106,6 +103,26 @@ public class MigrationUtils { return Collections.emptySet(); } + public static void cleanAllIndexWithRollover(CloseableHttpClient httpClient, BundleContext bundleContext, String esAddress, String prefix, String indexName) throws IOException { + Set<String> indexes = getIndexesPrefixedBy(httpClient, esAddress, prefix + "-" + indexName + "-000"); + List<String> sortedIndexes = new ArrayList<>(indexes); + Collections.sort(sortedIndexes); + + if (!sortedIndexes.isEmpty()) { + String lastIndexName = sortedIndexes.remove(sortedIndexes.size() - 1); + sortedIndexes.forEach(index -> { + try { + deleteIndex(httpClient, esAddress, index); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + String matchAllBodyRequest = resourceAsString(bundleContext, "requestBody/2.2.0/match_all_body_request.json"); + + HttpUtils.executePostRequest(httpClient, esAddress + "/" + lastIndexName + "/_delete_by_query", matchAllBodyRequest, null); + } + } + public static String extractMappingFromBundles(BundleContext bundleContext, String fileName) throws IOException { for (Bundle bundle : bundleContext.getBundles()) { Enumeration<URL> predefinedMappings = bundle.findEntries("META-INF/cxs/mappings", fileName, true); diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-00-rolloverAndMigrateEvent.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-00-rolloverAndMigrateEvent.groovy deleted file mode 100644 index 99fcf9e02..000000000 --- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-00-rolloverAndMigrateEvent.groovy +++ /dev/null @@ -1,74 +0,0 @@ -import org.apache.unomi.shell.migration.service.MigrationContext -import org.apache.unomi.shell.migration.utils.HttpUtils -import org.apache.unomi.shell.migration.utils.MigrationUtils - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -MigrationContext context = migrationContext -String esAddress = context.getConfigString("esAddress") -String indexPrefix = context.getConfigString("indexPrefix") -String newEventIndex = indexPrefix + "-event-000001" -String rolloverPolicyName = indexPrefix + "-unomi-rollover-policy" -String rolloverEventAlias = indexPrefix + "-event" - - -context.performMigrationStep("2.2.0-00-update-lifecyle-poll-interval", () -> { - String updatePollIntervalBody = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.2.0/update_settings_poll_interval.json") - .replace("#pollIntervalValue", "\"2s\"") - HttpUtils.executePutRequest(context.getHttpClient(), esAddress + "/_cluster/settings", updatePollIntervalBody, null) -}) - -context.performMigrationStep("2.2.0-00-create-rollover-policy", () -> { - String createRolloverPolicyQuery = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.2.0/create_rollover_policy_query.json") - String rolloverQueryBody = MigrationUtils.buildRolloverPolicyCreationRequest(createRolloverPolicyQuery, context) - - HttpUtils.executePutRequest(context.getHttpClient(), esAddress + "/_ilm/policy/" + rolloverPolicyName, rolloverQueryBody, null) -}) - -context.performMigrationStep("2.2.0-00-create-event-index", () -> { - if (!MigrationUtils.indexExists(context.getHttpClient(), esAddress, newEventIndex)) { - String baseRequest = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.2.0/base_index_mapping.json") - String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "event.json") - - String newIndexSettings = MigrationUtils.buildIndexCreationRequestWithRollover(baseRequest, mapping, context, rolloverPolicyName, rolloverEventAlias) - HttpUtils.executePutRequest(context.getHttpClient(), esAddress + "/" + newEventIndex, newIndexSettings, null) - } -}) - -Set<String> eventIndices = MigrationUtils.getIndexesPrefixedBy(context.getHttpClient(), esAddress, indexPrefix + "-event-date-") -List<String> sortedIndices = new ArrayList<>(eventIndices) -Collections.sort(sortedIndices) - -context.performMigrationStep("2.2.0-00-migrate-existing-events", () -> { - sortedIndices.each { eventIndex -> - MigrationUtils.moveToIndex(context.getHttpClient(), bundleContext, esAddress, eventIndex, indexPrefix + "-event") - sleep(3000) - } -}) - -context.performMigrationStep("2.2.0-00-remove-old-events-indices", () -> { - sortedIndices.each { eventIndex -> - MigrationUtils.deleteIndex(context.getHttpClient(), esAddress, eventIndex) - } -}) - -context.performMigrationStep("2.2.0-00-reset-poll-interval", () -> { - String updatePollIntervalBody = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.2.0/update_settings_poll_interval.json") - .replace("#pollIntervalValue", "null") - HttpUtils.executePutRequest(context.getHttpClient(), esAddress + "/_cluster/settings", updatePollIntervalBody, null) -}) diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-10-rolloverAndMigrateSession.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-00-rolloverAndMigrateEventSession.groovy similarity index 57% rename from tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-10-rolloverAndMigrateSession.groovy rename to tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-00-rolloverAndMigrateEventSession.groovy index a553ae808..a04a4b097 100644 --- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-10-rolloverAndMigrateSession.groovy +++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-00-rolloverAndMigrateEventSession.groovy @@ -22,27 +22,56 @@ import org.apache.unomi.shell.migration.utils.MigrationUtils MigrationContext context = migrationContext String esAddress = context.getConfigString("esAddress") String indexPrefix = context.getConfigString("indexPrefix") -String newSessionIndex = indexPrefix + "-session-000001" +String newEventIndex = indexPrefix + "-event-000001" String rolloverPolicyName = indexPrefix + "-unomi-rollover-policy" +String rolloverEventAlias = indexPrefix + "-event" +String newSessionIndex = indexPrefix + "-session-000001" String rolloverSessionAlias = indexPrefix + "-session" - -context.performMigrationStep("2.2.0-10-update-lifecyle-poll-interval", () -> { +context.performMigrationStep("2.2.0-update-lifecyle-poll-interval", () -> { String updatePollIntervalBody = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.2.0/update_settings_poll_interval.json") .replace("#pollIntervalValue", "\"2s\"") HttpUtils.executePutRequest(context.getHttpClient(), esAddress + "/_cluster/settings", updatePollIntervalBody, null) }) -context.performMigrationStep("2.2.0-10-create-rollover-policy", () -> { +context.performMigrationStep("2.2.0-create-rollover-policy", () -> { String createRolloverPolicyQuery = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.2.0/create_rollover_policy_query.json") String rolloverQueryBody = MigrationUtils.buildRolloverPolicyCreationRequest(createRolloverPolicyQuery, context) HttpUtils.executePutRequest(context.getHttpClient(), esAddress + "/_ilm/policy/" + rolloverPolicyName, rolloverQueryBody, null) }) -context.performMigrationStep("2.2.0-10-create-session-index", () -> { +context.performMigrationStep("2.2.0-create-event-index", () -> { + if (!MigrationUtils.indexExists(context.getHttpClient(), esAddress, newEventIndex)) { + String baseRequest = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.2.0/base_index_withRollover_request.json") + String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "event.json") + + String newIndexSettings = MigrationUtils.buildIndexCreationRequestWithRollover(baseRequest, mapping, context, rolloverPolicyName, rolloverEventAlias) + HttpUtils.executePutRequest(context.getHttpClient(), esAddress + "/" + newEventIndex, newIndexSettings, null) + } +}) + +Set<String> eventIndices = MigrationUtils.getIndexesPrefixedBy(context.getHttpClient(), esAddress, indexPrefix + "-event-date-") +List<String> eventSortedIndices = new ArrayList<>(eventIndices) +Collections.sort(eventSortedIndices) + +context.performMigrationStep("2.2.0-migrate-existing-events", () -> { + MigrationUtils.cleanAllIndexWithRollover(context.getHttpClient(), bundleContext, esAddress, indexPrefix, "event") + eventSortedIndices.each { eventIndex -> + MigrationUtils.moveToIndex(context.getHttpClient(), bundleContext, esAddress, eventIndex, indexPrefix + "-event") + sleep(3000) + } +}) + +context.performMigrationStep("2.2.0-remove-old-events-indices", () -> { + eventSortedIndices.each { eventIndex -> + MigrationUtils.deleteIndex(context.getHttpClient(), esAddress, eventIndex) + } +}) + +context.performMigrationStep("2.2.0-create-session-index", () -> { if (!MigrationUtils.indexExists(context.getHttpClient(), esAddress, newSessionIndex)) { - String baseRequest = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.2.0/base_index_mapping.json") + String baseRequest = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.2.0/base_index_withRollover_request.json") String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "session.json") String newIndexSettings = MigrationUtils.buildIndexCreationRequestWithRollover(baseRequest, mapping, context, rolloverPolicyName, rolloverSessionAlias) @@ -51,23 +80,24 @@ context.performMigrationStep("2.2.0-10-create-session-index", () -> { }) Set<String> sessionIndices = MigrationUtils.getIndexesPrefixedBy(context.getHttpClient(), esAddress, indexPrefix + "-session-date-") -List<String> sortedIndices = new ArrayList<>(sessionIndices) -Collections.sort(sortedIndices) +List<String> sessionSortedIndices = new ArrayList<>(sessionIndices) +Collections.sort(sessionSortedIndices) -context.performMigrationStep("2.2.0-10-migrate-existing-sessions", () -> { - sortedIndices.each { sessionIndex -> +context.performMigrationStep("2.2.0-migrate-existing-sessions", () -> { + MigrationUtils.cleanAllIndexWithRollover(context.getHttpClient(), bundleContext, esAddress, indexPrefix, "session") + sessionSortedIndices.each { sessionIndex -> MigrationUtils.moveToIndex(context.getHttpClient(), bundleContext, esAddress, sessionIndex, indexPrefix + "-session") sleep(3000) } }) -context.performMigrationStep("2.2.0-10-remove-old-sessions-indices", () -> { - sortedIndices.each { sessionIndex -> +context.performMigrationStep("2.2.0-remove-old-sessions-indices", () -> { + sessionSortedIndices.each { sessionIndex -> MigrationUtils.deleteIndex(context.getHttpClient(), esAddress, sessionIndex) } }) -context.performMigrationStep("2.2.0-10-reset-poll-interval", () -> { +context.performMigrationStep("2.2.0-reset-poll-interval", () -> { String updatePollIntervalBody = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.2.0/update_settings_poll_interval.json") .replace("#pollIntervalValue", "null") HttpUtils.executePutRequest(context.getHttpClient(), esAddress + "/_cluster/settings", updatePollIntervalBody, null) diff --git a/tools/shell-commands/src/main/resources/requestBody/2.2.0/base_index_mapping.json b/tools/shell-commands/src/main/resources/requestBody/2.2.0/base_index_withRollover_request.json similarity index 100% rename from tools/shell-commands/src/main/resources/requestBody/2.2.0/base_index_mapping.json rename to tools/shell-commands/src/main/resources/requestBody/2.2.0/base_index_withRollover_request.json diff --git a/tools/shell-commands/src/main/resources/requestBody/2.2.0/match_all_body_request.json b/tools/shell-commands/src/main/resources/requestBody/2.2.0/match_all_body_request.json new file mode 100644 index 000000000..487927ea1 --- /dev/null +++ b/tools/shell-commands/src/main/resources/requestBody/2.2.0/match_all_body_request.json @@ -0,0 +1,5 @@ +{ + "query": { + "match_all": {} + } +} \ No newline at end of file
