This is an automated email from the ASF dual-hosted git repository. jsinovassinnaik pushed a commit to branch UNOMI-728-index-migration in repository https://gitbox.apache.org/repos/asf/unomi.git
commit 63203afe5f651f677b011e990dfe6e9b4a74f9fa Author: jsinovassin <[email protected]> AuthorDate: Wed Feb 8 11:48:20 2023 +0100 UNOMI-728 : add migration scripts to 2.2.0 version --- .../shell/migration/service/MigrationConfig.java | 8 ++- .../shell/migration/utils/MigrationUtils.java | 59 +++++++++++++---- ...migrate-2.2.0-00-rolloverAndMigrateEvent.groovy | 74 ++++++++++++++++++++++ ...grate-2.2.0-10-rolloverAndMigrateSession.groovy | 74 ++++++++++++++++++++++ .../main/resources/org.apache.unomi.migration.cfg | 3 + .../requestBody/2.2.0/base_index_mapping.json | 30 +++++++++ .../requestBody/2.2.0/base_reindex_request.json | 8 +++ .../2.2.0/create_rollover_policy_query.json | 19 ++++++ .../2.2.0/update_settings_poll_interval.json | 5 ++ 9 files changed, 265 insertions(+), 15 deletions(-) diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationConfig.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationConfig.java index 16fb6728a..3d4ccd4c7 100644 --- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationConfig.java +++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/service/MigrationConfig.java @@ -46,7 +46,9 @@ public class MigrationConfig { public static final String MONTHLY_TOTAL_FIELDS_LIMIT = "monthlyIndex." + TOTAL_FIELDS_LIMIT; public static final String MONTHLY_MAX_DOC_VALUE_FIELDS_SEARCH = "monthlyIndex." + MAX_DOC_VALUE_FIELDS_SEARCH; public static final String MIGRATION_HISTORY_RECOVER = "recoverFromHistory"; - + public static final String ROLLOVER_MAX_AGE = "rolloverMaxAge"; + public static final String ROLLOVER_MAX_SIZE = "rolloverMaxSize"; + public static final String ROLLOVER_MAX_DOCS = "rolloverMaxDocs"; protected static final Map<String, MigrationConfigProperty> configProperties; static { Map<String, MigrationConfigProperty> m = new HashMap<>(); @@ -65,6 +67,10 @@ public class MigrationConfig { m.put(MONTHLY_TOTAL_FIELDS_LIMIT, new MigrationConfigProperty("Enter ElasticSearch monthly index (event, session) mapping configuration: mapping.total_fields.limit (default: 1000): ", "1000")); m.put(MONTHLY_MAX_DOC_VALUE_FIELDS_SEARCH, new MigrationConfigProperty("Enter ElasticSearch monthly index (event, session) mapping configuration: max_docvalue_fields_search (default: 1000): ", "1000")); m.put(MIGRATION_HISTORY_RECOVER, new MigrationConfigProperty("We found an existing migration attempt, should we restart from it ? (this will avoid redoing steps already completed successfully) (yes/no)", null)); + m.put(ROLLOVER_MAX_AGE, new MigrationConfigProperty("Enter ElasticSearch index rollover configuration: max_age (default: 365d): ", "365d")); + m.put(ROLLOVER_MAX_SIZE, new MigrationConfigProperty("Enter ElasticSearch index rollover configuration: max_size (default: null): ", null)); + m.put(ROLLOVER_MAX_DOCS, new MigrationConfigProperty("Enter ElasticSearch index rollover configuration: max_docs (default: null): ", null)); + configProperties = Collections.unmodifiableMap(m); } diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java index 40fd9f549..19bb6af2e 100644 --- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java +++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java @@ -35,6 +35,7 @@ import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.Enumeration; import java.util.Set; +import java.util.StringJoiner; import java.util.stream.Collectors; import static org.apache.unomi.shell.migration.service.MigrationConfig.*; @@ -77,8 +78,9 @@ public class MigrationUtils { String line; StringBuilder value = new StringBuilder(); while ((line = br.readLine()) != null) { - if (!line.startsWith("/*") && !line.startsWith(" *") && !line.startsWith("*/")) + if (!line.startsWith("/*") && !line.startsWith(" *") && !line.startsWith("*/")) { value.append(line); + } } in.close(); return value.toString(); @@ -98,9 +100,7 @@ public class MigrationUtils { try (CloseableHttpResponse response = httpClient.execute(new HttpGet(esAddress + "/_aliases"))) { if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { JSONObject indexesAsJson = new JSONObject(EntityUtils.toString(response.getEntity())); - return indexesAsJson.keySet().stream(). - filter(alias -> alias.startsWith(prefix)). - collect(Collectors.toSet()); + return indexesAsJson.keySet().stream().filter(alias -> alias.startsWith(prefix)).collect(Collectors.toSet()); } } return Collections.emptySet(); @@ -131,8 +131,44 @@ public class MigrationUtils { return settings.replace("#mappings", mapping); } - public static void reIndex(CloseableHttpClient httpClient, BundleContext bundleContext, String esAddress, String indexName, - String newIndexSettings, String painlessScript, MigrationContext migrationContext) throws Exception { + public static String buildIndexCreationRequestWithRollover(String baseIndexSettings, String mapping, MigrationContext context, String lifeCycleName, String rolloverAlias) throws IOException { + return buildIndexCreationRequest(baseIndexSettings, mapping, context, false) + .replace("#lifecycleName", lifeCycleName) + .replace("#lifecycleRolloverAlias", rolloverAlias); + } + + public static String buildRolloverPolicyCreationRequest(String baseRequest, MigrationContext migrationContext) throws IOException { + + StringJoiner rolloverHotActions = new StringJoiner(", "); + + String rolloverMaxAge = migrationContext.getConfigString("rolloverMaxAge"); + String rolloverMaxSize = migrationContext.getConfigString("rolloverMaxSize"); + String rolloverMaxDocs = migrationContext.getConfigString("rolloverMaxDocs"); + if (StringUtils.isNotBlank(rolloverMaxAge)) { + rolloverHotActions.add("\"max_age\": \"" + rolloverMaxAge + "\""); + } + if (StringUtils.isNotBlank(rolloverMaxSize)) { + rolloverHotActions.add("\"max_size\": \"" + rolloverMaxSize + "\""); + } + if (StringUtils.isNotBlank(rolloverMaxDocs)) { + rolloverHotActions.add("\"max_docs\": \"" + rolloverMaxDocs + "\""); + } + return baseRequest.replace("#rolloverHotActions", rolloverHotActions.toString()); + } + + public static void moveToIndex(CloseableHttpClient httpClient, BundleContext bundleContext, String esAddress, String sourceIndexName, String targetIndexName) throws Exception { + String reIndexRequest = resourceAsString(bundleContext, "requestBody/2.2.0/base_reindex_request.json").replace("#source", sourceIndexName).replace("#dest", targetIndexName); + + HttpUtils.executePostRequest(httpClient, esAddress + "/_reindex", reIndexRequest, null); + } + + public static void deleteIndex(CloseableHttpClient httpClient, String esAddress, String indexName) throws Exception { + if (indexExists(httpClient, esAddress, indexName)) { + HttpUtils.executeDeleteRequest(httpClient, esAddress + "/" + indexName, null); + } + } + + public static void reIndex(CloseableHttpClient httpClient, BundleContext bundleContext, String esAddress, String indexName, String newIndexSettings, String painlessScript, MigrationContext migrationContext) throws Exception { if (indexName.endsWith("-cloned")) { // We should never reIndex a clone ... return; @@ -140,9 +176,7 @@ public class MigrationUtils { String indexNameCloned = indexName + "-cloned"; - String reIndexRequest = resourceAsString(bundleContext, "requestBody/2.0.0/base_reindex_request.json") - .replace("#source", indexNameCloned).replace("#dest", indexName) - .replace("#painless", StringUtils.isNotEmpty(painlessScript) ? getScriptPart(painlessScript) : ""); + String reIndexRequest = resourceAsString(bundleContext, "requestBody/2.0.0/base_reindex_request.json").replace("#source", indexNameCloned).replace("#dest", indexName).replace("#painless", StringUtils.isNotEmpty(painlessScript) ? getScriptPart(painlessScript) : ""); String setIndexReadOnlyRequest = resourceAsString(bundleContext, "requestBody/2.0.0/base_set_index_readonly_request.json"); @@ -208,10 +242,7 @@ public class MigrationUtils { } // scroll - response = HttpUtils.executePostRequest(httpClient, esAddress + "/_search/scroll", "{\n" + - " \"scroll_id\": \"" + scrollId + "\",\n" + - " \"scroll\": \"" + scrollDuration + "\"\n" + - "}", null); + response = HttpUtils.executePostRequest(httpClient, esAddress + "/_search/scroll", "{\n" + " \"scroll_id\": \"" + scrollId + "\",\n" + " \"scroll\": \"" + scrollDuration + "\"\n" + "}", null); } } @@ -222,7 +253,7 @@ public class MigrationUtils { while (true) { final JSONObject status = new JSONObject(HttpUtils.executeGetRequest(httpClient, esAddress + "/_cluster/health?wait_for_status=yellow&timeout=60s", null)); if (!status.get("timed_out").equals("true")) { - migrationContext.printMessage("ES Cluster status is " + status.get("status")); + migrationContext.printMessage("ES Cluster status is " + status.get("status")); break; } migrationContext.printMessage("Waiting for ES Cluster status to be Yellow, current status is " + status.get("status")); diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-00-rolloverAndMigrateEvent.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-00-rolloverAndMigrateEvent.groovy new file mode 100644 index 000000000..99fcf9e02 --- /dev/null +++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-00-rolloverAndMigrateEvent.groovy @@ -0,0 +1,74 @@ +import org.apache.unomi.shell.migration.service.MigrationContext +import org.apache.unomi.shell.migration.utils.HttpUtils +import org.apache.unomi.shell.migration.utils.MigrationUtils + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +MigrationContext context = migrationContext +String esAddress = context.getConfigString("esAddress") +String indexPrefix = context.getConfigString("indexPrefix") +String newEventIndex = indexPrefix + "-event-000001" +String rolloverPolicyName = indexPrefix + "-unomi-rollover-policy" +String rolloverEventAlias = indexPrefix + "-event" + + +context.performMigrationStep("2.2.0-00-update-lifecyle-poll-interval", () -> { + String updatePollIntervalBody = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.2.0/update_settings_poll_interval.json") + .replace("#pollIntervalValue", "\"2s\"") + HttpUtils.executePutRequest(context.getHttpClient(), esAddress + "/_cluster/settings", updatePollIntervalBody, null) +}) + +context.performMigrationStep("2.2.0-00-create-rollover-policy", () -> { + String createRolloverPolicyQuery = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.2.0/create_rollover_policy_query.json") + String rolloverQueryBody = MigrationUtils.buildRolloverPolicyCreationRequest(createRolloverPolicyQuery, context) + + HttpUtils.executePutRequest(context.getHttpClient(), esAddress + "/_ilm/policy/" + rolloverPolicyName, rolloverQueryBody, null) +}) + +context.performMigrationStep("2.2.0-00-create-event-index", () -> { + if (!MigrationUtils.indexExists(context.getHttpClient(), esAddress, newEventIndex)) { + String baseRequest = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.2.0/base_index_mapping.json") + String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "event.json") + + String newIndexSettings = MigrationUtils.buildIndexCreationRequestWithRollover(baseRequest, mapping, context, rolloverPolicyName, rolloverEventAlias) + HttpUtils.executePutRequest(context.getHttpClient(), esAddress + "/" + newEventIndex, newIndexSettings, null) + } +}) + +Set<String> eventIndices = MigrationUtils.getIndexesPrefixedBy(context.getHttpClient(), esAddress, indexPrefix + "-event-date-") +List<String> sortedIndices = new ArrayList<>(eventIndices) +Collections.sort(sortedIndices) + +context.performMigrationStep("2.2.0-00-migrate-existing-events", () -> { + sortedIndices.each { eventIndex -> + MigrationUtils.moveToIndex(context.getHttpClient(), bundleContext, esAddress, eventIndex, indexPrefix + "-event") + sleep(3000) + } +}) + +context.performMigrationStep("2.2.0-00-remove-old-events-indices", () -> { + sortedIndices.each { eventIndex -> + MigrationUtils.deleteIndex(context.getHttpClient(), esAddress, eventIndex) + } +}) + +context.performMigrationStep("2.2.0-00-reset-poll-interval", () -> { + String updatePollIntervalBody = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.2.0/update_settings_poll_interval.json") + .replace("#pollIntervalValue", "null") + HttpUtils.executePutRequest(context.getHttpClient(), esAddress + "/_cluster/settings", updatePollIntervalBody, null) +}) diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-10-rolloverAndMigrateSession.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-10-rolloverAndMigrateSession.groovy new file mode 100644 index 000000000..a553ae808 --- /dev/null +++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.2.0-10-rolloverAndMigrateSession.groovy @@ -0,0 +1,74 @@ +import org.apache.unomi.shell.migration.service.MigrationContext +import org.apache.unomi.shell.migration.utils.HttpUtils +import org.apache.unomi.shell.migration.utils.MigrationUtils + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +MigrationContext context = migrationContext +String esAddress = context.getConfigString("esAddress") +String indexPrefix = context.getConfigString("indexPrefix") +String newSessionIndex = indexPrefix + "-session-000001" +String rolloverPolicyName = indexPrefix + "-unomi-rollover-policy" +String rolloverSessionAlias = indexPrefix + "-session" + + +context.performMigrationStep("2.2.0-10-update-lifecyle-poll-interval", () -> { + String updatePollIntervalBody = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.2.0/update_settings_poll_interval.json") + .replace("#pollIntervalValue", "\"2s\"") + HttpUtils.executePutRequest(context.getHttpClient(), esAddress + "/_cluster/settings", updatePollIntervalBody, null) +}) + +context.performMigrationStep("2.2.0-10-create-rollover-policy", () -> { + String createRolloverPolicyQuery = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.2.0/create_rollover_policy_query.json") + String rolloverQueryBody = MigrationUtils.buildRolloverPolicyCreationRequest(createRolloverPolicyQuery, context) + + HttpUtils.executePutRequest(context.getHttpClient(), esAddress + "/_ilm/policy/" + rolloverPolicyName, rolloverQueryBody, null) +}) + +context.performMigrationStep("2.2.0-10-create-session-index", () -> { + if (!MigrationUtils.indexExists(context.getHttpClient(), esAddress, newSessionIndex)) { + String baseRequest = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.2.0/base_index_mapping.json") + String mapping = MigrationUtils.extractMappingFromBundles(bundleContext, "session.json") + + String newIndexSettings = MigrationUtils.buildIndexCreationRequestWithRollover(baseRequest, mapping, context, rolloverPolicyName, rolloverSessionAlias) + HttpUtils.executePutRequest(context.getHttpClient(), esAddress + "/" + newSessionIndex, newIndexSettings, null) + } +}) + +Set<String> sessionIndices = MigrationUtils.getIndexesPrefixedBy(context.getHttpClient(), esAddress, indexPrefix + "-session-date-") +List<String> sortedIndices = new ArrayList<>(sessionIndices) +Collections.sort(sortedIndices) + +context.performMigrationStep("2.2.0-10-migrate-existing-sessions", () -> { + sortedIndices.each { sessionIndex -> + MigrationUtils.moveToIndex(context.getHttpClient(), bundleContext, esAddress, sessionIndex, indexPrefix + "-session") + sleep(3000) + } +}) + +context.performMigrationStep("2.2.0-10-remove-old-sessions-indices", () -> { + sortedIndices.each { sessionIndex -> + MigrationUtils.deleteIndex(context.getHttpClient(), esAddress, sessionIndex) + } +}) + +context.performMigrationStep("2.2.0-10-reset-poll-interval", () -> { + String updatePollIntervalBody = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.2.0/update_settings_poll_interval.json") + .replace("#pollIntervalValue", "null") + HttpUtils.executePutRequest(context.getHttpClient(), esAddress + "/_cluster/settings", updatePollIntervalBody, null) +}) diff --git a/tools/shell-commands/src/main/resources/org.apache.unomi.migration.cfg b/tools/shell-commands/src/main/resources/org.apache.unomi.migration.cfg index f4cd52771..cc907d93f 100644 --- a/tools/shell-commands/src/main/resources/org.apache.unomi.migration.cfg +++ b/tools/shell-commands/src/main/resources/org.apache.unomi.migration.cfg @@ -32,6 +32,9 @@ number_of_shards=${org.apache.unomi.elasticsearch.defaultIndex.nbShards:-5} number_of_replicas=${org.apache.unomi.elasticsearch.defaultIndex.nbReplicas:-0} mapping.total_fields.limit=${org.apache.unomi.elasticsearch.defaultIndex.indexMappingTotalFieldsLimit:-1000} max_docvalue_fields_search=${org.apache.unomi.elasticsearch.defaultIndex.indexMaxDocValueFieldsSearch:-1000} +rolloverMaxSize=${org.apache.unomi.elasticsearch.rollover.maxSize:-} +rolloverMaxAge=${org.apache.unomi.elasticsearch.rollover.maxAge:-365d} +rolloverMaxDocs=${org.apache.unomi.elasticsearch.rollover.maxDocs:-} # Should the migration try to recover from a previous run ? # (This allow to avoid redoing all the steps that would already succeeded on a previous attempt, that was stop or failed in the middle) diff --git a/tools/shell-commands/src/main/resources/requestBody/2.2.0/base_index_mapping.json b/tools/shell-commands/src/main/resources/requestBody/2.2.0/base_index_mapping.json new file mode 100644 index 000000000..c59422642 --- /dev/null +++ b/tools/shell-commands/src/main/resources/requestBody/2.2.0/base_index_mapping.json @@ -0,0 +1,30 @@ +{ + "settings": { + "index": { + "number_of_shards": #numberOfShards, + "number_of_replicas": #numberOfReplicas, + "mapping.total_fields.limit": #mappingTotalFieldsLimit, + "max_docvalue_fields_search": #maxDocValueFieldsSearch, + "lifecycle.name": "#lifecycleName", + "lifecycle.rollover_alias": "#lifecycleRolloverAlias" + }, + "analysis": { + "analyzer": { + "folding": { + "type": "custom", + "tokenizer": "keyword", + "filter": [ + "lowercase", + "asciifolding" + ] + } + } + } + }, + "aliases": { + "#lifecycleRolloverAlias": { + "is_write_index": true + } + }, + "mappings": #mappings +} \ No newline at end of file diff --git a/tools/shell-commands/src/main/resources/requestBody/2.2.0/base_reindex_request.json b/tools/shell-commands/src/main/resources/requestBody/2.2.0/base_reindex_request.json new file mode 100644 index 000000000..ddcb79a5e --- /dev/null +++ b/tools/shell-commands/src/main/resources/requestBody/2.2.0/base_reindex_request.json @@ -0,0 +1,8 @@ +{ + "source": { + "index": "#source" + }, + "dest": { + "index": "#dest" + } +} diff --git a/tools/shell-commands/src/main/resources/requestBody/2.2.0/create_rollover_policy_query.json b/tools/shell-commands/src/main/resources/requestBody/2.2.0/create_rollover_policy_query.json new file mode 100644 index 000000000..c9bc94a29 --- /dev/null +++ b/tools/shell-commands/src/main/resources/requestBody/2.2.0/create_rollover_policy_query.json @@ -0,0 +1,19 @@ +{ + "policy": { + "phases": { + "hot": { + "actions": { + "rollover": { + #rolloverHotActions + } + } + }, + "delete": { + "min_age": "90d", + "actions": { + "delete": {} + } + } + } + } +} \ No newline at end of file diff --git a/tools/shell-commands/src/main/resources/requestBody/2.2.0/update_settings_poll_interval.json b/tools/shell-commands/src/main/resources/requestBody/2.2.0/update_settings_poll_interval.json new file mode 100644 index 000000000..75695e0fe --- /dev/null +++ b/tools/shell-commands/src/main/resources/requestBody/2.2.0/update_settings_poll_interval.json @@ -0,0 +1,5 @@ +{ + "persistent": { + "indices.lifecycle.poll_interval": #pollIntervalValue + } +} \ No newline at end of file
