This is an automated email from the ASF dual-hosted git repository. jsinovassinnaik pushed a commit to branch UNOMI-606-migration-segments-scoring in repository https://gitbox.apache.org/repos/asf/unomi.git
commit 5279bd3c0d86b8f75e8b72198fb05c5ea3feba47 Author: jsinovassin <jsinovassinn...@jahia.com> AuthorDate: Mon Jul 11 13:49:41 2022 +0200 UNOMI-606 : create migration for scoring and fix issues --- .../unomi/shell/migration/MigrationScript.java | 2 +- .../unomi/shell/migration/actions/Migrate.java | 4 +- .../unomi/shell/migration/impl/MigrationTo150.java | 2 +- .../unomi/shell/migration/impl/MigrationTo200.java | 5 +- .../unomi/shell/migration/utils/ConsoleUtils.java | 6 +- .../migrate-2.0.0-01-segmentReindex.groovy | 3 +- ... => migrate-2.0.0-02-scoringPlanReindex.groovy} | 5 +- .../resources/requestBody/2.0.0/scoring_index.json | 68 ++++++++++++++++++++++ 8 files changed, 81 insertions(+), 14 deletions(-) diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationScript.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationScript.java index d2dc304ed..0355781bb 100644 --- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationScript.java +++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/MigrationScript.java @@ -39,7 +39,7 @@ import java.util.regex.Pattern; */ public class MigrationScript implements Comparable<MigrationScript> { - private static final Pattern SCRIPT_FILENAME_PATTERN = Pattern.compile("^migrate-(\\d.\\d.\\d)-(\\d+)-(\\w+).groovy$"); + private static final Pattern SCRIPT_FILENAME_PATTERN = Pattern.compile("^migrate-(\\d.\\d.\\d)-(\\d+)-([\\w|.]+).groovy$"); private final String script; private Script compiledScript; diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java index d325cee56..a3bf07837 100644 --- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java +++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/actions/Migrate.java @@ -46,6 +46,7 @@ import java.util.stream.Stream; public class Migrate implements Action { public static final String CONFIG_ES_ADDRESS = "esAddress"; public static final String CONFIG_TRUST_ALL_CERTIFICATES = "httpClient.trustAllCertificates"; + public static final String INDEX_PREFIX = "indexPrefix"; @Reference Session session; @@ -53,7 +54,7 @@ public class Migrate implements Action { @Reference BundleContext bundleContext; - @Argument(index = 0, name = "originVersion", description = "Origin version without suffix/qualifier (e.g: 1.2.0)", valueToShowInHelp = "1.2.0") + @Argument(name = "originVersion", description = "Origin version without suffix/qualifier (e.g: 1.2.0)", valueToShowInHelp = "1.2.0") private String originVersion; public Object execute() throws Exception { @@ -90,6 +91,7 @@ public class Migrate implements Action { Map<String, Object> migrationConfig = new HashMap<>(); migrationConfig.put(CONFIG_ES_ADDRESS, ConsoleUtils.askUserWithDefaultAnswer(session, "Enter ElasticSearch 7 TARGET address (default = http://localhost:9200): ", "http://localhost:9200")); migrationConfig.put(CONFIG_TRUST_ALL_CERTIFICATES, ConsoleUtils.askUserWithAuthorizedAnswer(session,"We need to initialize a HttpClient, do we need to trust all certificates? (yes/no): ", Arrays.asList("yes", "no")).equalsIgnoreCase("yes")); + migrationConfig.put(INDEX_PREFIX, ConsoleUtils.askUserWithDefaultAnswer(session, "SOURCE index name (default: context) : ", "context")); try (CloseableHttpClient httpClient = HttpUtils.initHttpClient((Boolean) migrationConfig.get(CONFIG_TRUST_ALL_CERTIFICATES))) { diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo150.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo150.java index f0aee214e..4eb84c821 100644 --- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo150.java +++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo150.java @@ -40,7 +40,7 @@ public class MigrationTo150 implements Migration { public void execute(Session session, CloseableHttpClient httpClient, Map<String, Object> migrationConfig, BundleContext bundleContext) throws IOException { String esAddress = (String) migrationConfig.get("esAddress"); String es5Address = ConsoleUtils.askUserWithDefaultAnswer(session, "SOURCE Elasticsearch 5.6 cluster address (default: http://localhost:9210) : ", "http://localhost:9210"); - String sourceIndexPrefix = ConsoleUtils.askUserWithDefaultAnswer(session, "SOURCE index name (default: context) : ", "context"); + String sourceIndexPrefix = (String) migrationConfig.get("indexPrefix"); String destIndexPrefix = ConsoleUtils.askUserWithDefaultAnswer(session, "TARGET index prefix (default: context) : ", "context"); int numberOfShards = Integer.parseInt(ConsoleUtils.askUserWithDefaultAnswer(session, "Number of shards for TARGET (default: 5) : ", "5")); int numberOfReplicas = Integer.parseInt(ConsoleUtils.askUserWithDefaultAnswer(session, "Number of replicas for TARGET (default: 1) : ", "1")); diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java index a859bfb83..6b94e16b7 100644 --- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java +++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/impl/MigrationTo200.java @@ -57,11 +57,10 @@ public class MigrationTo200 implements Migration { this.esAddress = (String) migrationConfig.get("esAddress"); this.bundleContext = bundleContext; - doExecute(); + doExecute((String) migrationConfig.get("indexPrefix")); } - private void doExecute() throws IOException { - String indexPrefix = ConsoleUtils.askUserWithDefaultAnswer(session, "SOURCE index name (default: context) : ", "context"); + private void doExecute(String indexPrefix) throws IOException { Set<String> indexes = getEventIndexes(indexPrefix); for (String index : indexes) { updateMapping(index); diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/ConsoleUtils.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/ConsoleUtils.java index 145e24921..b56a7568c 100644 --- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/ConsoleUtils.java +++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/ConsoleUtils.java @@ -19,8 +19,6 @@ package org.apache.unomi.shell.migration.utils; import org.apache.commons.lang3.StringUtils; import org.apache.karaf.shell.api.console.Session; import org.jline.reader.LineReader; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.PrintStream; @@ -30,7 +28,6 @@ import java.util.List; * @author dgaillard */ public class ConsoleUtils { - private static final Logger logger = LoggerFactory.getLogger(HttpUtils.class); /** * This will ask a question to the user and return the default answer if the user does not answer. @@ -74,9 +71,8 @@ public class ConsoleUtils { * @param session the shell's session * @param msg String message to prompt * @return the user answer - * @throws IOException if there was an error retrieving an answer from the user on the console */ - public static String promptMessageToUser(Session session, String msg) throws IOException { + public static String promptMessageToUser(Session session, String msg) { LineReader reader = (LineReader) session.get(".jline.reader"); return reader.readLine(msg, null); } diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-segmentReindex.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-segmentReindex.groovy index 1ffe88aaa..01a2be0c6 100644 --- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-segmentReindex.groovy +++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-segmentReindex.groovy @@ -18,4 +18,5 @@ import org.apache.unomi.shell.migration.utils.MigrationUtils */ String newIndexSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/segment_index.json"); -MigrationUtils.reIndex(httpClient, bundleContext, migrationConfig.get("esAddress"), "context-segment", newIndexSettings) \ No newline at end of file +MigrationUtils.reIndex(httpClient, bundleContext, migrationConfig.get("esAddress"), migrationConfig.get("indexPrefix") + "-segment", + newIndexSettings) diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-segmentReindex.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scoringPlanReindex.groovy similarity index 87% copy from tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-segmentReindex.groovy copy to tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scoringPlanReindex.groovy index 1ffe88aaa..f30f2ae6f 100644 --- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-01-segmentReindex.groovy +++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-2.0.0-02-scoringPlanReindex.groovy @@ -17,5 +17,6 @@ import org.apache.unomi.shell.migration.utils.MigrationUtils * limitations under the License. */ -String newIndexSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/segment_index.json"); -MigrationUtils.reIndex(httpClient, bundleContext, migrationConfig.get("esAddress"), "context-segment", newIndexSettings) \ No newline at end of file +String newIndexSettings = MigrationUtils.resourceAsString(bundleContext, "requestBody/2.0.0/scoring_index.json"); +MigrationUtils.reIndex(httpClient, bundleContext, migrationConfig.get("esAddress"), migrationConfig.get("indexPrefix") + "-scoring", + newIndexSettings) diff --git a/tools/shell-commands/src/main/resources/requestBody/2.0.0/scoring_index.json b/tools/shell-commands/src/main/resources/requestBody/2.0.0/scoring_index.json new file mode 100644 index 000000000..d2541861d --- /dev/null +++ b/tools/shell-commands/src/main/resources/requestBody/2.0.0/scoring_index.json @@ -0,0 +1,68 @@ +{ + "settings": { + "index": { + "number_of_shards": #numberOfShards, + "number_of_replicas": #numberOfReplicas, + "mapping.total_fields.limit": #mappingTotalFieldsLimit, + "max_docvalue_fields_search": #maxDocValueFieldsSearch + }, + "analysis": { + "analyzer": { + "folding": { + "type": "custom", + "tokenizer": "keyword", + "filter": [ + "lowercase", + "asciifolding" + ] + } + } + } + }, + "mappings": { + "dynamic_templates": [ + { + "all": { + "match": "*", + "match_mapping_type": "string", + "mapping": { + "type": "text", + "analyzer": "folding", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + ], + "properties": { + "metadata": { + "properties": { + "enabled": { + "type": "boolean" + }, + "hidden": { + "type": "boolean" + }, + "missingPlugins": { + "type": "boolean" + }, + "readOnly": { + "type": "boolean" + } + } + }, + "elements": { + "properties": { + "condition": { + "type": "object", + "enabled": false + } + } + } + } + } +}