This is an automated email from the ASF dual-hosted git repository. jsinovassinnaik pushed a commit to branch UNOMI-885 in repository https://gitbox.apache.org/repos/asf/unomi.git
commit 56714bce890f5984680f8032cb6f8b8d313bd501 Author: jsinovassin <[email protected]> AuthorDate: Tue Mar 4 16:11:59 2025 +0100 UNOMI-885: fix migration error on rollover alias --- .../shell/migration/utils/MigrationUtils.java | 66 +++++++++++++++++++++- ...-2.2.0-10-rolloverAndMigrateEventSession.groovy | 4 ++ ...te-2.5.0-00-cleanPastEventProfileSession.groovy | 16 ++++-- .../migrate-2.5.0-10-loginEventScope.groovy | 1 - .../requestBody/2.0.0/base_reindex_request.json | 3 +- .../2.2.0/base_index_withRollover_request.json | 4 +- .../requestBody/2.2.0/configure_alias_body.json | 11 ++++ 7 files changed, 92 insertions(+), 13 deletions(-) diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java index 06d05f535..e1109e650 100644 --- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java +++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java @@ -97,6 +97,23 @@ public class MigrationUtils { } } + public static void configureAlias(CloseableHttpClient httpClient, String esAddress, String alias, String writeIndex, Set<String> readIndices, String configureAliasBody, MigrationContext context) throws IOException { + String readIndicesToAdd = ""; + if (!readIndices.isEmpty()) { + readIndicesToAdd = "," + readIndices.stream().map(index -> "{\"add\": {\"index\": \"" + index + "\", \"alias\": \"" + alias + "\", \"is_write_index\": false}}").collect(Collectors.joining(",")); + } + if (context != null) { + context.printMessage("Will set " + writeIndex + " as write index for alias " + alias); + context.printMessage("Will set " + readIndices.toString() + " as read indices"); + } else { + LOGGER.info("Will set {} as write index for alias {}", writeIndex, alias); + LOGGER.info("Will set {} as read indices", readIndices.toString()); + } + String requestBody = configureAliasBody.replace("#writeIndexName", writeIndex).replace("#aliasName", alias).replace("#readIndicesToAdd", readIndicesToAdd); + + HttpUtils.executePostRequest(httpClient, esAddress + "/_aliases", requestBody, null); + } + public static Set<String> getIndexesPrefixedBy(CloseableHttpClient httpClient, String esAddress, String prefix) throws IOException { try (CloseableHttpResponse response = httpClient.execute(new HttpGet(esAddress + "/_aliases"))) { if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { @@ -300,9 +317,9 @@ public class MigrationUtils { * <p>This method sends a request to update documents that match the provided query in the specified index. The update operation is * performed asynchronously, and the method waits for the task to complete before returning.</p> * - * @param httpClient the CloseableHttpClient used to send the request to the Elasticsearch server - * @param esAddress the address of the Elasticsearch server - * @param indexName the name of the index where documents should be updated + * @param httpClient the CloseableHttpClient used to send the request to the Elasticsearch server + * @param esAddress the address of the Elasticsearch server + * @param indexName the name of the index where documents should be updated * @param requestBody the JSON body containing the query and update instructions for the documents * @throws Exception if there is an error during the HTTP request or while waiting for the task to finish */ @@ -332,6 +349,30 @@ public class MigrationUtils { waitForTaskToFinish(httpClient, esAddress, task.getString("task"), null); } + private static void printResponseDetail(JSONObject response, MigrationContext migrationContext){ + StringBuilder sb = new StringBuilder(); + if (response.has("total")) { + sb.append("Total: ").append(response.getInt("total")).append(" "); + } + if (response.has("updated")) { + sb.append("Updated: ").append(response.getInt("updated")).append(" "); + } + if (response.has("created")) { + sb.append("Created: ").append(response.getInt("created")).append(" "); + } + if (response.has("deleted")) { + sb.append("Deleted: ").append(response.getInt("deleted")).append(" "); + } + if (response.has("batches")) { + sb.append("Batches: ").append(response.getInt("batches")).append(" "); + } + if (migrationContext != null) { + migrationContext.printMessage(sb.toString()); + } else { + LOGGER.info(sb.toString()); + } + } + public static void waitForTaskToFinish(CloseableHttpClient httpClient, String esAddress, String taskId, MigrationContext migrationContext) throws IOException { while (true) { final JSONObject status = new JSONObject( @@ -343,6 +384,25 @@ public class MigrationUtils { } else { LOGGER.info("Task is completed"); } + if (status.has("response")) { + final JSONObject response = status.getJSONObject("response"); + printResponseDetail(response, migrationContext); + if (response.has("failures")) { + final JSONArray failures = response.getJSONArray("failures"); + if (!failures.isEmpty()) { + for (int i = 0; i < failures.length(); i++) { + JSONObject failure = failures.getJSONObject(i); + JSONObject cause = failure.getJSONObject("cause"); + if (migrationContext != null) { + migrationContext.printMessage("Cause of failure: " + cause.toString()); + } else { + LOGGER.error("Cause of failure: {}", cause.toString()); + } + } + throw new IOException("Task completed with failures, check previous log for details"); + } + } + } break; } if (status.has("error")) { diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-10-rolloverAndMigrateEventSession.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-10-rolloverAndMigrateEventSession.groovy index c692fa7a2..7da6f86c3 100644 --- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-10-rolloverAndMigrateEventSession.groovy +++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-10-rolloverAndMigrateEventSession.groovy @@ -45,9 +45,11 @@ context.performMigrationStep("2.2.0-create-event-index", () -> { if (!MigrationUtils.indexExists(context.getHttpClient(), esAddress, newEventIndex)) { String baseRequest = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.2.0/base_index_withRollover_request.json") String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "event.json") + String configureAliasBody = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.2.0/configure_alias_body.json") String newIndexSettings = MigrationUtils.buildIndexCreationRequestWithRollover(baseRequest, mapping, context, rolloverPolicyName, rolloverEventAlias) HttpUtils.executePutRequest(context.getHttpClient(), esAddress + "/" + newEventIndex, newIndexSettings, null) + MigrationUtils.configureAlias(context.getHttpClient(), esAddress, rolloverEventAlias, newEventIndex, Collections.emptySet(), configureAliasBody, context) } }) @@ -73,9 +75,11 @@ context.performMigrationStep("2.2.0-create-session-index", () -> { if (!MigrationUtils.indexExists(context.getHttpClient(), esAddress, newSessionIndex)) { String baseRequest = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.2.0/base_index_withRollover_request.json") String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "session.json") + String configureAliasBody = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.2.0/configure_alias_body.json") String newIndexSettings = MigrationUtils.buildIndexCreationRequestWithRollover(baseRequest, mapping, context, rolloverPolicyName, rolloverSessionAlias) HttpUtils.executePutRequest(context.getHttpClient(), esAddress + "/" + newSessionIndex, newIndexSettings, null) + MigrationUtils.configureAlias(context.getHttpClient(), esAddress, rolloverSessionAlias, newSessionIndex, Collections.emptySet(), configureAliasBody, context) } }) diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.5.0-00-cleanPastEventProfileSession.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.5.0-00-cleanPastEventProfileSession.groovy index 2e45b437d..01d69aec8 100644 --- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.5.0-00-cleanPastEventProfileSession.groovy +++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.5.0-00-cleanPastEventProfileSession.groovy @@ -1,8 +1,5 @@ import org.apache.unomi.shell.migration.service.MigrationContext -import org.apache.unomi.shell.migration.utils.HttpUtils import org.apache.unomi.shell.migration.utils.MigrationUtils -import org.osgi.framework.BundleContext -import org.osgi.framework.Bundle /* * Licensed to the Apache Software Foundation (ASF) under one or more @@ -25,7 +22,7 @@ MigrationContext context = migrationContext String esAddress = context.getConfigString("esAddress") String indexPrefix = context.getConfigString("indexPrefix") String rolloverPolicyName = indexPrefix + "-unomi-rollover-policy" -String rolloverEventAlias = indexPrefix + "-session" +String rolloverSessionAlias = indexPrefix + "-session" context.performMigrationStep("2.5.0-clean-profile-mapping", () -> { String baseSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/base_index_mapping.json") @@ -39,10 +36,17 @@ context.performMigrationStep("2.5.0-clean-session-mapping", () -> { String baseSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.2.0/base_index_withRollover_request.json") String cleanPastEventScript = MigrationUtils.getFileWithoutComments(bundleContext, "requestBody/2.5.0/remove_pastEvents_session.painless") String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "session.json") - String newIndexSettings = MigrationUtils.buildIndexCreationRequestWithRollover(baseSettings, mapping, context, rolloverPolicyName, rolloverEventAlias) + String newIndexSettings = MigrationUtils.buildIndexCreationRequestWithRollover(baseSettings, mapping, context, rolloverPolicyName, rolloverSessionAlias) Set<String> sessionIndices = MigrationUtils.getIndexesPrefixedBy(context.getHttpClient(), esAddress, "${indexPrefix}-session-") + String configureAliasBody = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.2.0/configure_alias_body.json") - sessionIndices.each { sessionIndex -> + Set<String> sortedSet = new TreeSet<>(sessionIndices) + sortedSet.each { sessionIndex -> MigrationUtils.reIndex(context.getHttpClient(), bundleContext, esAddress, sessionIndex, newIndexSettings, cleanPastEventScript, context, "2.5.0-clean-session-mapping") } + SortedSet<String> allExceptLast = Collections.emptySortedSet(); + if (sortedSet.size() > 1){ + allExceptLast = sortedSet.headSet(sortedSet.last()); + } + MigrationUtils.configureAlias(context.getHttpClient(), esAddress, rolloverSessionAlias, sortedSet.last(), allExceptLast, configureAliasBody, context) }) diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.5.0-10-loginEventScope.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.5.0-10-loginEventScope.groovy index 967bc5714..890e93981 100644 --- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.5.0-10-loginEventScope.groovy +++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.5.0-10-loginEventScope.groovy @@ -1,5 +1,4 @@ import org.apache.unomi.shell.migration.service.MigrationContext -import org.apache.unomi.shell.migration.utils.HttpUtils import org.apache.unomi.shell.migration.utils.MigrationUtils /* diff --git a/tools/shell-commands/src/main/resources/requestBody/2.0.0/base_reindex_request.json b/tools/shell-commands/src/main/resources/requestBody/2.0.0/base_reindex_request.json index 589e71084..a44b69b35 100644 --- a/tools/shell-commands/src/main/resources/requestBody/2.0.0/base_reindex_request.json +++ b/tools/shell-commands/src/main/resources/requestBody/2.0.0/base_reindex_request.json @@ -4,5 +4,6 @@ }, "dest": { "index": "#dest" - }#painless + }, + "size": 20000#painless } diff --git a/tools/shell-commands/src/main/resources/requestBody/2.2.0/base_index_withRollover_request.json b/tools/shell-commands/src/main/resources/requestBody/2.2.0/base_index_withRollover_request.json index c59422642..442d2d1b5 100644 --- a/tools/shell-commands/src/main/resources/requestBody/2.2.0/base_index_withRollover_request.json +++ b/tools/shell-commands/src/main/resources/requestBody/2.2.0/base_index_withRollover_request.json @@ -23,8 +23,8 @@ }, "aliases": { "#lifecycleRolloverAlias": { - "is_write_index": true + "is_write_index": false } }, "mappings": #mappings -} \ No newline at end of file +} diff --git a/tools/shell-commands/src/main/resources/requestBody/2.2.0/configure_alias_body.json b/tools/shell-commands/src/main/resources/requestBody/2.2.0/configure_alias_body.json new file mode 100644 index 000000000..4016c0f9b --- /dev/null +++ b/tools/shell-commands/src/main/resources/requestBody/2.2.0/configure_alias_body.json @@ -0,0 +1,11 @@ +{ + "actions": [ + { + "add": { + "index": "#writeIndexName", + "alias": "#aliasName", + "is_write_index": true + } + }#readIndicesToAdd + ] +}
