This is an automated email from the ASF dual-hosted git repository.
jsinovassinnaik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/unomi.git
The following commit(s) were added to refs/heads/master by this push:
new 3b2da028d UNOMI-861: use asynchronous delete_by_query in migration
scripts to a… (#704)
3b2da028d is described below
commit 3b2da028d6ac50b2b276c1150491bf0fc9cd689a
Author: jsinovassin <[email protected]>
AuthorDate: Thu Oct 24 16:09:03 2024 +0100
UNOMI-861: use asynchronous delete_by_query in migration scripts to a…
(#704)
* UNOMI-861: use asynchronous delete_by_query in migration scripts to avoid
timeout
* UNOMI-861: use asynchronous update_by_query in migration scripts to avoid
timeout
* UNOMI-861: surround code by try catch to avoid unwanted rerun
---
.../shell/migration/utils/MigrationUtils.java | 44 +++++++++++++++++++++-
.../internal/UnomiManagementServiceImpl.java | 28 +++++++++-----
.../cxs/migration/migrate-2.0.0-01-aliases.groovy | 4 +-
.../migrate-2.0.0-15-eventsReindex.groovy | 4 +-
.../migrate-2.4.0-15-viewEventPagePath.groovy | 2 +-
.../migrate-2.5.0-10-loginEventScope.groovy | 2 +-
6 files changed, 67 insertions(+), 17 deletions(-)
diff --git
a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
index 2059f1dc8..69521c729 100644
---
a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
+++
b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
@@ -123,7 +123,11 @@ public class MigrationUtils {
});
String matchAllBodyRequest = resourceAsString(bundleContext,
"requestBody/2.2.0/match_all_body_request.json");
- HttpUtils.executePostRequest(httpClient, esAddress + "/" +
lastIndexName + "/_delete_by_query", matchAllBodyRequest, null);
+ try {
+ deleteByQuery(httpClient, esAddress, lastIndexName,
matchAllBodyRequest);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
}
}
@@ -290,6 +294,44 @@ public class MigrationUtils {
}
+ /**
+ * Updates documents in an index based on a specified query.
+ *
+ * <p>This method sends a request to update documents that match the
provided query in the specified index. The update operation is
+ * performed asynchronously, and the method waits for the task to complete
before returning.</p>
+ *
+ * @param httpClient the CloseableHttpClient used to send the request to
the Elasticsearch server
+ * @param esAddress the address of the Elasticsearch server
+ * @param indexName the name of the index where documents should be updated
+ * @param requestBody the JSON body containing the query and update
instructions for the documents
+ * @throws Exception if there is an error during the HTTP request or while
waiting for the task to finish
+ */
+ public static void updateByQuery(CloseableHttpClient httpClient, String
esAddress, String indexName, String requestBody) throws Exception {
+ JSONObject task = new
JSONObject(HttpUtils.executePostRequest(httpClient, esAddress + "/" + indexName
+ "/_update_by_query?wait_for_completion=false", requestBody, null));
+
+ //Wait for the deletion task to finish
+ waitForTaskToFinish(httpClient, esAddress, task.getString("task"),
null);
+ }
+
+ /**
+ * Deletes documents from an index based on a specified query.
+ *
+ * <p>This method sends a request to the Elasticsearch cluster to delete
documents
+ * that match the provided query in the specified index. The deletion
operation is
+ * performed asynchronously, and the method waits for the task to complete
before returning.</p>
+ *
+ * @param httpClient the CloseableHttpClient used to send the request to
the Elasticsearch server
+ * @param esAddress the address of the Elasticsearch server
+ * @param indexName the name of the index from which documents should be
deleted
+ * @param requestBody the JSON body containing the query that defines
which documents to delete
+ * @throws Exception if there is an error during the HTTP request or while
waiting for the task to finish
+ */
+ public static void deleteByQuery(CloseableHttpClient httpClient, String
esAddress, String indexName, String requestBody) throws Exception {
+ JSONObject task = new
JSONObject(HttpUtils.executePostRequest(httpClient, esAddress + "/" + indexName
+ "/_delete_by_query?wait_for_completion=false", requestBody, null));
+ //Wait for the deletion task to finish
+ waitForTaskToFinish(httpClient, esAddress, task.getString("task"),
null);
+ }
+
public static void waitForTaskToFinish(CloseableHttpClient httpClient,
String esAddress, String taskId, MigrationContext migrationContext) throws
IOException {
while (true) {
final JSONObject status = new JSONObject(
diff --git
a/tools/shell-commands/src/main/java/org/apache/unomi/shell/services/internal/UnomiManagementServiceImpl.java
b/tools/shell-commands/src/main/java/org/apache/unomi/shell/services/internal/UnomiManagementServiceImpl.java
index defe72091..fbab16cd3 100644
---
a/tools/shell-commands/src/main/java/org/apache/unomi/shell/services/internal/UnomiManagementServiceImpl.java
+++
b/tools/shell-commands/src/main/java/org/apache/unomi/shell/services/internal/UnomiManagementServiceImpl.java
@@ -27,6 +27,8 @@ import org.osgi.service.component.annotations.Activate;
import org.osgi.service.component.annotations.Component;
import org.osgi.service.component.annotations.Reference;
import org.osgi.service.component.annotations.ReferenceCardinality;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Collections;
@@ -38,26 +40,32 @@ import java.util.List;
@Component(service = UnomiManagementService.class, immediate = true)
public class UnomiManagementServiceImpl implements UnomiManagementService {
+ private static final Logger LOGGER =
LoggerFactory.getLogger(UnomiManagementServiceImpl.class.getName());
+
private BundleContext bundleContext;
-
+
@Reference(cardinality = ReferenceCardinality.MANDATORY)
private MigrationService migrationService;
-
+
private final List<String> bundleSymbolicNames = new ArrayList<>();
private List<String> reversedBundleSymbolicNames;
@Activate
public void init(ComponentContext componentContext) throws Exception {
- this.bundleContext = componentContext.getBundleContext();
- initReversedBundleSymbolicNames();
+ try {
+ this.bundleContext = componentContext.getBundleContext();
+ initReversedBundleSymbolicNames();
- if
(StringUtils.isNotBlank(bundleContext.getProperty("unomi.autoMigrate"))) {
-
migrationService.migrateUnomi(bundleContext.getProperty("unomi.autoMigrate"),
true, null);
- }
+ if
(StringUtils.isNotBlank(bundleContext.getProperty("unomi.autoMigrate"))) {
+
migrationService.migrateUnomi(bundleContext.getProperty("unomi.autoMigrate"),
true, null);
+ }
- if
(StringUtils.isNotBlank(bundleContext.getProperty("unomi.autoStart")) &&
- bundleContext.getProperty("unomi.autoStart").equals("true")) {
- startUnomi();
+ if
(StringUtils.isNotBlank(bundleContext.getProperty("unomi.autoStart")) &&
+
bundleContext.getProperty("unomi.autoStart").equals("true")) {
+ startUnomi();
+ }
+ } catch (Exception e) {
+ LOGGER.error("Error during Unomi startup when processing
'unomi.autoMigrate' or 'unomi.autoStart' properties:", e);
}
}
diff --git
a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-aliases.groovy
b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-aliases.groovy
index 685d44288..ad6863682 100644
---
a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-aliases.groovy
+++
b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-aliases.groovy
@@ -91,5 +91,5 @@
context.performMigrationStep("2.0.0-create-aliases-for-existing-merged-profiles"
context.performMigrationStep("2.0.0-delete-existing-merged-profiles", () -> {
String profileMergedDeleteRequest =
MigrationUtils.resourceAsString(bundleContext,"requestBody/2.0.0/profile_merged_delete.json")
- HttpUtils.executePostRequest(context.getHttpClient(), esAddress + "/" +
profileIndex + "/_delete_by_query", profileMergedDeleteRequest, null)
-})
\ No newline at end of file
+ MigrationUtils.deleteByQuery(context.getHttpClient(), esAddress,
profileIndex, profileMergedDeleteRequest)
+})
diff --git
a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-15-eventsReindex.groovy
b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-15-eventsReindex.groovy
index cb9f5ba1a..85a25df87 100644
---
a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-15-eventsReindex.groovy
+++
b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-15-eventsReindex.groovy
@@ -25,7 +25,7 @@ String indexPrefix = context.getConfigString("indexPrefix")
context.performMigrationStep("2.0.0-remove-events-not-persisted-anymore", ()
-> {
String removeInternalEventsRequest =
MigrationUtils.resourceAsString(bundleContext,
"requestBody/2.0.0/event_delete_by_query.json")
- HttpUtils.executePostRequest(context.getHttpClient(),
"${esAddress}/${indexPrefix}-event-*/_delete_by_query",
removeInternalEventsRequest, null)
+ MigrationUtils.deleteByQuery(context.getHttpClient(), esAddress,
"${indexPrefix}-event-*", removeInternalEventsRequest)
})
// Reindex the rest of the events
@@ -36,4 +36,4 @@ String newIndexSettings =
MigrationUtils.buildIndexCreationRequest(baseSettings,
Set<String> eventIndices =
MigrationUtils.getIndexesPrefixedBy(context.getHttpClient(), esAddress,
"${indexPrefix}-event-date-")
eventIndices.each { eventIndex ->
MigrationUtils.reIndex(context.getHttpClient(), bundleContext, esAddress,
eventIndex, newIndexSettings, reIndexScript, context)
-}
\ No newline at end of file
+}
diff --git
a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.4.0-15-viewEventPagePath.groovy
b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.4.0-15-viewEventPagePath.groovy
index 8eb0de38d..1ac1776b5 100644
---
a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.4.0-15-viewEventPagePath.groovy
+++
b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.4.0-15-viewEventPagePath.groovy
@@ -28,6 +28,6 @@
context.performMigrationStep("2.4.0-migrate-view-event-page-path", () -> {
String updatePathScript =
MigrationUtils.getFileWithoutComments(bundleContext,
"requestBody/2.4.0/view_event_page_path_migrate.painless")
String baseSettings = MigrationUtils.resourceAsString(bundleContext,
"requestBody/2.4.0/base_update_by_query_request.json")
eventIndices.each { eventIndex ->
- HttpUtils.executePostRequest(context.getHttpClient(),
"${esAddress}/${eventIndex}/_update_by_query",
baseSettings.replace('#painless', updatePathScript), null)
+ MigrationUtils.updateByQuery(context.getHttpClient(), esAddress,
eventIndex, baseSettings.replace('#painless', updatePathScript));
}
})
diff --git
a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.5.0-10-loginEventScope.groovy
b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.5.0-10-loginEventScope.groovy
index 24e2042e6..967bc5714 100644
---
a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.5.0-10-loginEventScope.groovy
+++
b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.5.0-10-loginEventScope.groovy
@@ -28,6 +28,6 @@
context.performMigrationStep("2.5.0-migrate-login-event-scope", () -> {
String updatePathScript =
MigrationUtils.getFileWithoutComments(bundleContext,
"requestBody/2.5.0/login_event_scope_migrate.painless")
String baseSettings = MigrationUtils.resourceAsString(bundleContext,
"requestBody/2.5.0/scope_update_by_query_request.json")
eventIndices.each { eventIndex ->
- HttpUtils.executePostRequest(context.getHttpClient(),
"${esAddress}/${eventIndex}/_update_by_query",
baseSettings.replace('#painless', updatePathScript), null)
+ MigrationUtils.updateByQuery(context.getHttpClient(), esAddress,
eventIndex, baseSettings.replace('#painless', updatePathScript));
}
})
