This is an automated email from the ASF dual-hosted git repository. jsinovassinnaik pushed a commit to branch UNOMI-861 in repository https://gitbox.apache.org/repos/asf/unomi.git
commit ee94556b209937c53f9bb1a9d4daa9a230f55c77 Author: jsinovassin <[email protected]> AuthorDate: Wed Oct 23 17:08:51 2024 +0100 UNOMI-861: use asynchronous delete_by_query in migration scripts to avoid timeout --- .../apache/unomi/shell/migration/utils/MigrationUtils.java | 12 +++++++++++- .../META-INF/cxs/migration/migrate-2.0.0-01-aliases.groovy | 4 ++-- .../cxs/migration/migrate-2.0.0-15-eventsReindex.groovy | 4 ++-- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java index 2059f1dc8..720afce9a 100644 --- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java +++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java @@ -123,7 +123,11 @@ public class MigrationUtils { }); String matchAllBodyRequest = resourceAsString(bundleContext, "requestBody/2.2.0/match_all_body_request.json"); - HttpUtils.executePostRequest(httpClient, esAddress + "/" + lastIndexName + "/_delete_by_query", matchAllBodyRequest, null); + try { + deleteByQuery(httpClient, esAddress, lastIndexName, matchAllBodyRequest); + } catch (Exception e) { + throw new RuntimeException(e); + } } } @@ -290,6 +294,12 @@ public class MigrationUtils { } + public static void deleteByQuery(CloseableHttpClient httpClient, String esAddress, String indexName, String requestBody) throws Exception { + JSONObject task = new JSONObject(HttpUtils.executePostRequest(httpClient ,esAddress + "/" +indexName + "/_delete_by_query?wait_for_completion=false", requestBody, null)); + //Wait for the deletion task to finish + waitForTaskToFinish(httpClient, esAddress, task.getString("task"), null); + } + public static void waitForTaskToFinish(CloseableHttpClient httpClient, String esAddress, String taskId, MigrationContext migrationContext) throws IOException { while (true) { final JSONObject status = new JSONObject( diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-aliases.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-aliases.groovy index 685d44288..ad6863682 100644 --- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-aliases.groovy +++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-aliases.groovy @@ -91,5 +91,5 @@ context.performMigrationStep("2.0.0-create-aliases-for-existing-merged-profiles" context.performMigrationStep("2.0.0-delete-existing-merged-profiles", () -> { String profileMergedDeleteRequest = MigrationUtils.resourceAsString(bundleContext,"requestBody/2.0.0/profile_merged_delete.json") - HttpUtils.executePostRequest(context.getHttpClient(), esAddress + "/" + profileIndex + "/_delete_by_query", profileMergedDeleteRequest, null) -}) \ No newline at end of file + MigrationUtils.deleteByQuery(context.getHttpClient(), esAddress, profileIndex, profileMergedDeleteRequest) +}) diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-15-eventsReindex.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-15-eventsReindex.groovy index cb9f5ba1a..85a25df87 100644 --- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-15-eventsReindex.groovy +++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-15-eventsReindex.groovy @@ -25,7 +25,7 @@ String indexPrefix = context.getConfigString("indexPrefix") context.performMigrationStep("2.0.0-remove-events-not-persisted-anymore", () -> { String removeInternalEventsRequest = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/event_delete_by_query.json") - HttpUtils.executePostRequest(context.getHttpClient(), "${esAddress}/${indexPrefix}-event-*/_delete_by_query", removeInternalEventsRequest, null) + MigrationUtils.deleteByQuery(context.getHttpClient(), esAddress, "${indexPrefix}-event-*", removeInternalEventsRequest) }) // Reindex the rest of the events @@ -36,4 +36,4 @@ String newIndexSettings = MigrationUtils.buildIndexCreationRequest(baseSettings, Set<String> eventIndices = MigrationUtils.getIndexesPrefixedBy(context.getHttpClient(), esAddress, "${indexPrefix}-event-date-") eventIndices.each { eventIndex -> MigrationUtils.reIndex(context.getHttpClient(), bundleContext, esAddress, eventIndex, newIndexSettings, reIndexScript, context) -} \ No newline at end of file +}
