This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch UNOMI-996-migration-mappings
in repository https://gitbox.apache.org/repos/asf/unomi.git
The following commit(s) were added to refs/heads/UNOMI-996-migration-mappings
by this push:
new 7812c092b UNOMI-996: ship migration mappings in the bundle and fail on
missing ones
7812c092b is described below
commit 7812c092bbcee30e050eb29f5b55047858e3203c
Author: Serge Huber <[email protected]>
AuthorDate: Tue Oct 6 18:07:39 2026 +0200
UNOMI-996: ship migration mappings in the bundle and fail on missing ones
The copied mappings never reached the shell-commands jar, and the new
fallback hid it by silently reusing the 3.0 mapping of known types.
- Package the Elasticsearch and OpenSearch mappings as resources under
META-INF/cxs/migration-mappings/{engine}, outside the folder the
persistence services scan, and pick the engine from the cluster.
- Known types must find their bundled mapping; only unknown types copy
the mapping stored on the index. Print the choice for each index.
- Match item types on the whole index name instead of a prefix and add
a profileAlias configuration.
- Tests: assert the mappings in the migration bundle itself, cover the
system-scope path, and reindex through MigrationUtils.moveToIndex.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
---
.../itests/migration/Migrate40IndexMappingIT.java | 114 +++++++++++--------
tools/shell-commands/pom.xml | 47 ++++----
.../shell/migration/utils/MigrationUtils.java | 126 ++++++++-------------
.../migrate-4.0.0-01-tenantDocumentIds.groovy | 23 +++-
.../migrate-4.0.0-10-tenantInitialization.groovy | 3 +-
.../shell/migration/utils/MigrationUtilsTest.java | 88 +++++++++-----
6 files changed, 216 insertions(+), 185 deletions(-)
diff --git
a/itests/src/test/java/org/apache/unomi/itests/migration/Migrate40IndexMappingIT.java
b/itests/src/test/java/org/apache/unomi/itests/migration/Migrate40IndexMappingIT.java
index c7970ce8e..b66b472f5 100644
---
a/itests/src/test/java/org/apache/unomi/itests/migration/Migrate40IndexMappingIT.java
+++
b/itests/src/test/java/org/apache/unomi/itests/migration/Migrate40IndexMappingIT.java
@@ -18,8 +18,6 @@ package org.apache.unomi.itests.migration;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ArrayNode;
-import com.fasterxml.jackson.databind.node.ObjectNode;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.unomi.itests.BaseIT;
import org.apache.unomi.itests.persistence.SearchBackendIT;
@@ -30,9 +28,14 @@ import org.junit.Assert;
import org.junit.Assume;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import org.osgi.framework.Bundle;
+import org.osgi.framework.BundleContext;
import org.osgi.framework.FrameworkUtil;
import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
/**
* UNOMI-996: migration must find bundled mappings (including cluster nodes)
and
@@ -57,23 +60,66 @@ public class Migrate40IndexMappingIT extends BaseIT {
}
@Test
- public void clusterNodeMappingIsBundledWithTheMigrationCommand() throws
Exception {
- String mapping = MigrationUtils.extractMappingFromBundles(
-
FrameworkUtil.getBundle(MigrationUtils.class).getBundleContext(),
"clusterNode.json");
- Assert.assertTrue("clusterNode mapping should define cpuLoad",
mapping.contains("cpuLoad"));
+ public void mappingsAreBundledWithTheMigrationCommandForBothEngines() {
+ // Look in the migration command bundle only: the started persistence
bundle ships these files too
+ Bundle migrationBundle = FrameworkUtil.getBundle(MigrationUtils.class);
+ for (String engine : Arrays.asList(SEARCH_ENGINE_ELASTICSEARCH,
SEARCH_ENGINE_OPENSEARCH)) {
+ for (String mappingFile : Arrays.asList("clusterNode.json",
"profile.json", "event.json", "geonameEntry.json")) {
+ String path = "META-INF/cxs/migration-mappings/" + engine +
"/" + mappingFile;
+ Assert.assertNotNull(path + " should be shipped with the
migration command", migrationBundle.getEntry(path));
+ }
+ }
+ }
+
+ @Test
+ public void searchEngineIsDetected() throws Exception {
+ assumeHttpAdminApi();
+ try (CloseableHttpClient httpClient = createSearchEngineHttpClient()) {
+ Assert.assertEquals(getPersistenceBackend().providerId(),
+ MigrationUtils.getSearchEngine(httpClient,
getSearchEngineBaseUrl()));
+ }
}
@Test
public void unknownIndexKeepsDocumentsOnTheConfiguredTenant() throws
Exception {
+ JsonNode hits = migrateUnknownIndex(true);
+ Assert.assertEquals(2, hits.size());
+ for (JsonNode hit : hits) {
+ String id = hit.path("_id").asText();
+ Assert.assertTrue("document id should use the configured tenant: "
+ id, id.startsWith(TEST_TENANT_ID + "_"));
+ Assert.assertEquals(TEST_TENANT_ID,
hit.path("_source").path("tenantId").asText());
+ }
+ }
+
+ @Test
+ public void knownIndexKeepsSystemScopeDocumentsOnTheSystemTenant() throws
Exception {
+ JsonNode hits = migrateUnknownIndex(false);
+ Assert.assertEquals(2, hits.size());
+ for (JsonNode hit : hits) {
+ boolean systemScope =
"system".equals(hit.path("_source").path("scope").asText());
+ String expectedTenant = systemScope ? "system" : TEST_TENANT_ID;
+ Assert.assertEquals(expectedTenant + "_" +
hit.path("_source").path("itemId").asText(), hit.path("_id").asText());
+ Assert.assertEquals(expectedTenant,
hit.path("_source").path("tenantId").asText());
+ }
+ }
+
+ private void assumeHttpAdminApi() {
Assume.assumeTrue(
"HTTP admin API required to create migration fixture indices
(provider="
+ getPersistenceBackend().providerId() + ")",
persistenceCapabilities().httpAdminApi());
+ }
+
+ /**
+ * Reindexes a fixture index holding one document without scope and one
with scope "system"
+ * through the 4.0 tenant script, the way the migration does, and returns
the resulting hits.
+ */
+ private JsonNode migrateUnknownIndex(boolean forceDefaultTenant) throws
Exception {
+ assumeHttpAdminApi();
try (CloseableHttpClient httpClient = createSearchEngineHttpClient()) {
String es = getSearchEngineBaseUrl();
- org.osgi.framework.BundleContext migrationBundles =
-
FrameworkUtil.getBundle(MigrationUtils.class).getBundleContext();
+ BundleContext migrationBundles =
FrameworkUtil.getBundle(MigrationUtils.class).getBundleContext();
String sourceSettings = "{"
+
"\"settings\":{\"number_of_shards\":1,\"number_of_replicas\":0},"
@@ -91,55 +137,33 @@ public class Migrate40IndexMappingIT extends BaseIT {
+ "/_doc/unknown2?refresh=true",
"{\"itemId\":\"unknown2\",\"itemType\":\"sfdcConfiguration\",\"scope\":\"system\"}",
null);
- String mapping = MigrationUtils.resolveIndexMapping(
- migrationBundles, httpClient, es, UNKNOWN_INDEX,
"does-not-exist.json");
+ String mapping =
MigrationUtils.extractMappingFromIndex(httpClient, es, UNKNOWN_INDEX);
Assert.assertTrue("copied mapping should keep itemType",
mapping.contains("itemType"));
- Assert.assertEquals("unomi996unknown",
- MigrationUtils.resolveItemType(UNKNOWN_INDEX, "context",
- Arrays.asList("profile", "event", "clusterNode",
"generic")));
+ String itemType = MigrationUtils.resolveItemType(UNKNOWN_INDEX,
"context",
+ Arrays.asList("profile", "event", "clusterNode",
"generic"));
+ Assert.assertEquals("unomi996unknown", itemType);
String destSettings =
"{\"settings\":{\"number_of_shards\":1,\"number_of_replicas\":0},\"mappings\":"
+ mapping + "}";
MigrationUtils.createIndex(httpClient, es, DEST_INDEX,
destSettings);
+ Map<String, Object> params = new HashMap<>();
+ params.put("tenantId", TEST_TENANT_ID);
+ params.put("systemTenantId", "system");
+ params.put("forceDefaultTenant", forceDefaultTenant);
+ params.put("itemType", itemType);
+ params.put("date", "2026-01-01T00:00:00Z");
+ params.put("systemItems", Collections.emptySet());
+
String painless = MigrationUtils.getFileWithoutComments(
migrationBundles,
"requestBody/4.0.0/initialize_tenant_and_audit_fields.painless");
- ObjectMapper mapper = new ObjectMapper();
- ObjectNode paramsObj = mapper.createObjectNode();
- paramsObj.put("tenantId", TEST_TENANT_ID);
- paramsObj.put("systemTenantId", "system");
- paramsObj.put("forceDefaultTenant", true);
- paramsObj.put("itemType", "generic");
- paramsObj.put("date", "2026-01-01T00:00:00Z");
- paramsObj.set("systemItems", mapper.createArrayNode());
-
- ObjectNode scriptObj = mapper.createObjectNode();
- scriptObj.put("source", painless);
- scriptObj.put("lang", "painless");
- scriptObj.set("params", paramsObj);
-
- ObjectNode reindex = mapper.createObjectNode();
- reindex.set("source", mapper.createObjectNode().put("index",
UNKNOWN_INDEX));
- reindex.set("dest", mapper.createObjectNode().put("index",
DEST_INDEX));
- reindex.set("script", scriptObj);
- HttpUtils.executePostRequest(httpClient, es +
"/_reindex?refresh=true&wait_for_completion=true",
- mapper.writeValueAsString(reindex), null);
+ MigrationUtils.moveToIndex(httpClient, migrationBundles, es,
UNKNOWN_INDEX, DEST_INDEX, painless, params);
+ HttpUtils.executePostRequest(httpClient, es + "/" + DEST_INDEX +
"/_refresh", null, null);
String search = HttpUtils.executeGetRequest(httpClient,
es + "/" + DEST_INDEX + "/_search?size=10", null);
- ArrayNode hits = (ArrayNode)
mapper.readTree(search).path("hits").path("hits");
- Assert.assertEquals(2, hits.size());
- for (JsonNode hit : hits) {
- String id = hit.path("_id").asText();
- JsonNode source = hit.path("_source");
- Assert.assertTrue("document id should use the configured
tenant: " + id,
- id.startsWith(TEST_TENANT_ID + "_"));
- Assert.assertFalse("unknown index documents must not use the
system tenant: " + id,
- id.startsWith("system_"));
- Assert.assertEquals(TEST_TENANT_ID,
source.path("tenantId").asText());
- Assert.assertNotEquals("system",
source.path("tenantId").asText());
- }
+ return new
ObjectMapper().readTree(search).path("hits").path("hits");
}
}
}
diff --git a/tools/shell-commands/pom.xml b/tools/shell-commands/pom.xml
index 4f0aef63b..17ebbcced 100644
--- a/tools/shell-commands/pom.xml
+++ b/tools/shell-commands/pom.xml
@@ -157,31 +157,30 @@
</dependencies>
<build>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ </resource>
+ <!-- Mappings needed by the migration before Unomi is started.
Kept out of META-INF/cxs/mappings
+ so the persistence services do not load them as predefined
mappings. -->
+ <resource>
+
<directory>../../persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings</directory>
+
<targetPath>META-INF/cxs/migration-mappings/elasticsearch</targetPath>
+ </resource>
+ <resource>
+
<directory>../../extensions/geonames/services/src/main/resources/META-INF/cxs/mappings</directory>
+
<targetPath>META-INF/cxs/migration-mappings/elasticsearch</targetPath>
+ </resource>
+ <resource>
+
<directory>../../persistence-opensearch/core/src/main/resources/META-INF/cxs/mappings</directory>
+
<targetPath>META-INF/cxs/migration-mappings/opensearch</targetPath>
+ </resource>
+ <resource>
+
<directory>../../extensions/geonames/services/src/main/resources/META-INF/cxs/mappings</directory>
+
<targetPath>META-INF/cxs/migration-mappings/opensearch</targetPath>
+ </resource>
+ </resources>
<plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-resources-plugin</artifactId>
- <executions>
- <execution>
- <id>copy-persistence-mappings</id>
- <phase>generate-resources</phase>
- <goals>
- <goal>copy-resources</goal>
- </goals>
- <configuration>
-
<outputDirectory>${project.build.outputDirectory}/META-INF/cxs/mappings</outputDirectory>
- <resources>
- <resource>
-
<directory>${project.basedir}/../../persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings</directory>
- </resource>
- <resource>
-
<directory>${project.basedir}/../../extensions/geonames/services/src/main/resources/META-INF/cxs/mappings</directory>
- </resource>
- </resources>
- </configuration>
- </execution>
- </executions>
- </plugin>
<plugin>
<groupId>org.apache.felix</groupId>
<artifactId>maven-bundle-plugin</artifactId>
diff --git
a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
index f2d33be07..4e8d71657 100644
---
a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
+++
b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
@@ -303,113 +303,66 @@ public class MigrationUtils {
}
public static String extractMappingFromBundles(BundleContext
bundleContext, String fileName) throws IOException {
- String mapping = findMappingInBundles(bundleContext, fileName);
- if (mapping == null) {
- throw new RuntimeException("no mapping found in bundles for: " +
fileName);
- }
- return mapping;
+ return extractMappingFromBundles(bundleContext, fileName,
"elasticsearch");
}
/**
- * Looks up {@code META-INF/cxs/mappings/{fileName}} in every OSGi bundle.
- * The migration command bundle ships a copy of the persistence mappings
so this
- * works before Unomi persistence features are started.
+ * Looks up a mapping file in every OSGi bundle: first in {@code
META-INF/cxs/mappings}
+ * (the persistence bundles, when installed), then in the per-engine copy
shipped with the
+ * migration command bundle, which is available before Unomi is started.
The copy lives in a
+ * separate folder so the persistence services do not load it as a
predefined mapping.
*
- * @return the mapping JSON, or {@code null} if no bundle contains the file
+ * @param searchEngine {@code elasticsearch} or {@code opensearch}, see
{@link #getSearchEngine}
*/
- public static String findMappingInBundles(BundleContext bundleContext,
String fileName) throws IOException {
- if (bundleContext == null || fileName == null || fileName.isBlank()) {
- return null;
- }
- for (Bundle bundle : bundleContext.getBundles()) {
- Enumeration<URL> predefinedMappings =
bundle.findEntries("META-INF/cxs/mappings", fileName, true);
- if (predefinedMappings == null) {
- continue;
- }
- if (predefinedMappings.hasMoreElements()) {
- URL predefinedMappingURL = predefinedMappings.nextElement();
- return IOUtils.toString(predefinedMappingURL,
StandardCharsets.UTF_8);
+ public static String extractMappingFromBundles(BundleContext
bundleContext, String fileName, String searchEngine) throws IOException {
+ for (String path : List.of("META-INF/cxs/mappings",
"META-INF/cxs/migration-mappings/" + searchEngine)) {
+ for (Bundle bundle : bundleContext.getBundles()) {
+ Enumeration<URL> predefinedMappings = bundle.findEntries(path,
fileName, true);
+ if (predefinedMappings != null &&
predefinedMappings.hasMoreElements()) {
+ return IOUtils.toString(predefinedMappings.nextElement(),
StandardCharsets.UTF_8);
+ }
}
}
- return null;
+ throw new RuntimeException("no mapping found in bundles for: " +
fileName);
}
/**
- * Resolves the mapping used to recreate an index during migration.
- * Prefers a bundled file when present (the 4.0 definition). Otherwise
copies
- * the mapping currently stored on the index, so unknown types do not stop
the run.
+ * Copies the mapping currently stored on an index (GET {@code
/{index}/_mapping}).
+ * Used for indices that have no bundled 4.0 mapping.
*/
- public static String resolveIndexMapping(BundleContext bundleContext,
CloseableHttpClient httpClient,
- String esAddress, String
indexName, String mappingFileName) throws IOException {
- if (mappingFileName != null && !mappingFileName.isBlank()) {
- String bundled = findMappingInBundles(bundleContext,
mappingFileName);
- if (bundled != null) {
- return bundled;
- }
- }
- return extractMappingFromIndex(httpClient, esAddress, indexName);
- }
-
public static String extractMappingFromIndex(CloseableHttpClient
httpClient, String esAddress, String indexName) throws IOException {
- if (indexName == null || indexName.isBlank()) {
- throw new IllegalArgumentException("An index name is required to
copy its mapping");
- }
- String response = HttpUtils.executeGetRequest(httpClient, esAddress +
"/" + indexName + "/_mapping", null);
- return mappingsFromIndexResponse(response);
+ return
mappingsFromIndexResponse(HttpUtils.executeGetRequest(httpClient, esAddress +
"/" + indexName + "/_mapping", null));
}
- /**
- * Reads the mappings object from a GET {@code /{index}/_mapping} body.
- */
- public static String mappingsFromIndexResponse(String responseBody) {
+ static String mappingsFromIndexResponse(String responseBody) {
if (responseBody == null || responseBody.isBlank()) {
throw new IllegalArgumentException("Index mapping response is
empty");
}
JSONObject root = new JSONObject(responseBody);
- if (root.keySet().isEmpty()) {
- throw new IllegalArgumentException("Index mapping response has no
indices");
- }
- String firstIndex = root.keys().next();
- JSONObject index = root.getJSONObject(firstIndex);
- if (!index.has("mappings")) {
- throw new IllegalArgumentException("Index mapping response has no
mappings for " + firstIndex);
+ if (root.length() != 1) {
+ throw new IllegalArgumentException("Expected the mapping of
exactly one index, got: " + root.keySet());
}
- return index.getJSONObject("mappings").toString();
+ return
root.getJSONObject(root.keys().next()).getJSONObject("mappings").toString();
}
/**
* Maps an index name such as {@code context-event-000001} or {@code
context-clusternode}
- * to an item type. Known types are matched longest-first, ignoring case.
- * Unknown names keep the suffix after the prefix (without a rollover
number).
+ * to an item type. The name after the prefix (without a rollover number)
must equal a known
+ * type, ignoring case; otherwise that name is returned as is. Names that
do not start with
+ * the prefix resolve to {@code generic}.
*/
public static String resolveItemType(String indexName, String indexPrefix,
Collection<String> knownTypes) {
- if (indexName == null || indexName.isBlank()) {
- throw new IllegalArgumentException("An index name is required");
+ String prefix = indexPrefix + "-";
+ if (!indexName.regionMatches(true, 0, prefix, 0, prefix.length())) {
+ return "generic";
}
- if (indexPrefix == null || indexPrefix.isBlank()) {
- throw new IllegalArgumentException("An index prefix is required");
- }
- String lowerIndex = indexName.toLowerCase(Locale.ROOT);
- String prefix = indexPrefix.toLowerCase(Locale.ROOT) + "-";
- if (knownTypes != null) {
- List<String> types = new ArrayList<>();
- for (String type : knownTypes) {
- if (type != null && !type.isBlank() &&
!"generic".equals(type)) {
- types.add(type);
- }
- }
- types.sort((a, b) -> Integer.compare(b.length(), a.length()));
- for (String type : types) {
- if (lowerIndex.startsWith(prefix +
type.toLowerCase(Locale.ROOT))) {
- return type;
- }
+ String name =
indexName.substring(prefix.length()).replaceFirst("-\\d{6}$", "");
+ for (String type : knownTypes) {
+ if (name.equalsIgnoreCase(type)) {
+ return type;
}
}
- if (lowerIndex.startsWith(prefix)) {
- String rest = indexName.substring(indexPrefix.length() + 1);
- return rest.replaceFirst("-\\d{6}$", "");
- }
- return "generic";
+ return name;
}
public static String buildIndexCreationRequest(String baseIndexSettings,
String mapping, MigrationContext context, boolean isMonthlyIndex) throws
IOException {
@@ -1015,6 +968,21 @@ public class MigrationUtils {
return version.split("\\.")[0]; // Return major version number
}
+ /**
+ * Detects the engine behind the given address from its root endpoint.
+ *
+ * @return {@code opensearch} or {@code elasticsearch}
+ */
+ public static String getSearchEngine(CloseableHttpClient httpClient,
String esAddress) throws IOException {
+ return
searchEngineFromRootResponse(HttpUtils.executeGetRequest(httpClient, esAddress,
null));
+ }
+
+ static String searchEngineFromRootResponse(String responseBody) {
+ // Only OpenSearch reports a distribution
+ String distribution = new
JSONObject(responseBody).getJSONObject("version").optString("distribution");
+ return "opensearch".equals(distribution) ? "opensearch" :
"elasticsearch";
+ }
+
public interface ScrollCallback {
void execute(String hits);
}
diff --git
a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-4.0.0-01-tenantDocumentIds.groovy
b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-4.0.0-01-tenantDocumentIds.groovy
index b8557b50e..f5f265e3d 100644
---
a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-4.0.0-01-tenantDocumentIds.groovy
+++
b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-4.0.0-01-tenantDocumentIds.groovy
@@ -67,15 +67,22 @@ def indexConfigs = [
mapping: "personaSession.json",
useRollover: false
],
+ "profileAlias": [
+ baseSettings: "requestBody/2.0.0/base_index_mapping.json",
+ mapping: "profileAlias.json",
+ useRollover: false
+ ],
"clusterNode": [
baseSettings: "requestBody/2.0.0/base_index_mapping.json",
mapping: "clusterNode.json",
- useRollover: false
+ useRollover: false,
+ forceDefaultTenant: true
],
"generic": [
baseSettings: "requestBody/2.0.0/base_index_mapping.json",
mapping: null, // Copy the mapping currently stored on the
index
- useRollover: false
+ useRollover: false,
+ forceDefaultTenant: true
]
]
@@ -113,6 +120,9 @@ context.performMigrationStep("4.0.0-get-all-indices", () ->
{
// Get the Painless script
String updateScript = MigrationUtils.getFileWithoutComments(bundleContext,
"requestBody/4.0.0/initialize_tenant_and_audit_fields.painless")
+ String searchEngine =
MigrationUtils.getSearchEngine(context.getHttpClient(), esAddress)
+ context.printMessage("Search engine: " + searchEngine)
+
// Process each index (reindex them)
allIndices.each { indexName ->
context.printMessage("Processing index: " + indexName)
@@ -125,13 +135,16 @@ context.performMigrationStep("4.0.0-get-all-indices", ()
-> {
Map<String, Object> params = new HashMap<>(baseParams)
params.put("itemType", itemType)
// Cluster nodes and unknown indices keep site data in the configured
tenant, never "system"
- boolean forceDefaultTenant = (itemType == "clusterNode" ||
indexConfig.mapping == null)
+ boolean forceDefaultTenant = indexConfig.forceDefaultTenant == true
params.put("forceDefaultTenant", forceDefaultTenant)
// Get base settings and mapping
String baseSettings = MigrationUtils.resourceAsString(bundleContext,
indexConfig.baseSettings)
- String mappingFile = indexConfig.mapping ?: "${itemType}.json"
- String mapping = MigrationUtils.resolveIndexMapping(bundleContext,
context.getHttpClient(), esAddress, indexName, mappingFile)
+ // A known type must find its bundled 4.0 mapping; only unknown types
copy the mapping stored on the index
+ String mapping = indexConfig.mapping ?
+ MigrationUtils.extractMappingFromBundles(bundleContext,
indexConfig.mapping, searchEngine) :
+
MigrationUtils.extractMappingFromIndex(context.getHttpClient(), esAddress,
indexName)
+ context.printMessage("Item type: ${itemType}, mapping:
${indexConfig.mapping ?: 'copied from the index'}, default tenant only:
${forceDefaultTenant}")
// Build index settings
String newIndexSettings
diff --git
a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-4.0.0-10-tenantInitialization.groovy
b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-4.0.0-10-tenantInitialization.groovy
index 995f09d69..dada0f0f4 100644
---
a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-4.0.0-10-tenantInitialization.groovy
+++
b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-4.0.0-10-tenantInitialization.groovy
@@ -71,7 +71,8 @@ if (Files.exists(secretsDir)) {
// Create the default tenant index and items
context.performMigrationStep("4.0.0-create-tenant-index", () -> {
String baseSettings = MigrationUtils.resourceAsString(bundleContext,
"requestBody/2.0.0/base_index_mapping.json")
- String mapping = MigrationUtils.extractMappingFromBundles(bundleContext,
"tenant.json")
+ String mapping = MigrationUtils.extractMappingFromBundles(bundleContext,
"tenant.json",
+ MigrationUtils.getSearchEngine(context.getHttpClient(), esAddress))
String newIndexSettings =
MigrationUtils.buildIndexCreationRequest(baseSettings, mapping, context, false)
if (!MigrationUtils.indexExists(context.getHttpClient(), esAddress,
"${indexPrefix}-tenant")) {
diff --git
a/tools/shell-commands/src/test/java/org/apache/unomi/shell/migration/utils/MigrationUtilsTest.java
b/tools/shell-commands/src/test/java/org/apache/unomi/shell/migration/utils/MigrationUtilsTest.java
index 2be643754..92b48578c 100644
---
a/tools/shell-commands/src/test/java/org/apache/unomi/shell/migration/utils/MigrationUtilsTest.java
+++
b/tools/shell-commands/src/test/java/org/apache/unomi/shell/migration/utils/MigrationUtilsTest.java
@@ -215,24 +215,57 @@ public class MigrationUtilsTest {
assertEquals("Resource not found: missing.painless", ex.getMessage());
}
- @Test
- public void
extractMappingFromBundlesFindsKnownTypeWithoutPersistenceBundle() throws
Exception {
+ private static final String MAPPING =
"{\"properties\":{\"cpuLoad\":{\"type\":\"double\"}}}";
+
+ private String extractClusterNodeMappingFrom(String path) throws Exception
{
+ return extractClusterNodeMappingFrom(path, "elasticsearch");
+ }
+
+ private String extractClusterNodeMappingFrom(String path, String
searchEngine) throws Exception {
Path mappingFile = Files.createTempFile("clusterNode", ".json");
- Files.writeString(mappingFile,
"{\"properties\":{\"cpuLoad\":{\"type\":\"double\"}}}");
- URL mappingUrl = mappingFile.toUri().toURL();
+ try {
+ Files.writeString(mappingFile, MAPPING);
+ when(bundleContext.getBundles()).thenReturn(new Bundle[]{bundle});
+ when(bundle.findEntries(path, "clusterNode.json", true))
+
.thenReturn(Collections.enumeration(Collections.singletonList(mappingFile.toUri().toURL())));
+ return MigrationUtils.extractMappingFromBundles(bundleContext,
"clusterNode.json", searchEngine);
+ } finally {
+ Files.deleteIfExists(mappingFile);
+ }
+ }
+
+ @Test
+ public void extractMappingFromBundlesFindsPersistenceMapping() throws
Exception {
+ assertEquals(MAPPING,
extractClusterNodeMappingFrom("META-INF/cxs/mappings"));
+ }
+
+ @Test
+ public void extractMappingFromBundlesFallsBackToMigrationCopy() throws
Exception {
+ assertEquals(MAPPING,
extractClusterNodeMappingFrom("META-INF/cxs/migration-mappings/elasticsearch"));
+ assertEquals(MAPPING,
extractClusterNodeMappingFrom("META-INF/cxs/migration-mappings/opensearch",
"opensearch"));
+ }
+
+ @Test
+ public void extractMappingFromBundlesIgnoresTheOtherEngineCopy() {
when(bundleContext.getBundles()).thenReturn(new Bundle[]{bundle});
- when(bundle.findEntries("META-INF/cxs/mappings", "clusterNode.json",
true))
-
.thenReturn(Collections.enumeration(Collections.singletonList(mappingUrl)));
+
when(bundle.findEntries("META-INF/cxs/migration-mappings/elasticsearch",
"event.json", true))
+
.thenReturn(Collections.enumeration(Collections.singletonList(getClass().getResource("/"))));
+
+ assertThrows(RuntimeException.class,
+ () -> MigrationUtils.extractMappingFromBundles(bundleContext,
"event.json", "opensearch"));
+ }
- String mapping =
MigrationUtils.extractMappingFromBundles(bundleContext, "clusterNode.json");
- assertEquals("{\"properties\":{\"cpuLoad\":{\"type\":\"double\"}}}",
mapping);
- Files.deleteIfExists(mappingFile);
+ @Test
+ public void searchEngineIsDetectedFromTheRootResponse() {
+ assertEquals("opensearch", MigrationUtils.searchEngineFromRootResponse(
+
"{\"version\":{\"distribution\":\"opensearch\",\"number\":\"3.0.0\"}}"));
+ assertEquals("elasticsearch",
MigrationUtils.searchEngineFromRootResponse(
+ "{\"version\":{\"number\":\"9.1.0\"}}"));
}
@Test
public void extractMappingFromBundlesThrowsWhenMissing() {
when(bundleContext.getBundles()).thenReturn(new Bundle[]{bundle});
- when(bundle.findEntries("META-INF/cxs/mappings", "generic.json",
true)).thenReturn(null);
RuntimeException ex = assertThrows(RuntimeException.class,
() -> MigrationUtils.extractMappingFromBundles(bundleContext,
"generic.json"));
@@ -240,40 +273,33 @@ public class MigrationUtilsTest {
}
@Test
- public void mappingsFromIndexResponseReadsFirstIndex() {
+ public void mappingsFromIndexResponseReadsTheIndexMappings() {
String body =
"{\"context-sfdcconfiguration\":{\"mappings\":{\"properties\":{\"itemId\":{\"type\":\"keyword\"}}}}}";
- String mappings = MigrationUtils.mappingsFromIndexResponse(body);
- assertEquals("{\"properties\":{\"itemId\":{\"type\":\"keyword\"}}}",
mappings);
+ assertEquals("{\"properties\":{\"itemId\":{\"type\":\"keyword\"}}}",
MigrationUtils.mappingsFromIndexResponse(body));
}
@Test
- public void mappingsFromIndexResponseRejectsEmptyBody() {
+ public void mappingsFromIndexResponseRejectsAnythingButOneIndex() {
assertThrows(IllegalArgumentException.class, () ->
MigrationUtils.mappingsFromIndexResponse(""));
assertThrows(IllegalArgumentException.class, () ->
MigrationUtils.mappingsFromIndexResponse("{}"));
+ assertThrows(IllegalArgumentException.class,
+ () ->
MigrationUtils.mappingsFromIndexResponse("{\"a\":{\"mappings\":{}},\"b\":{\"mappings\":{}}}"));
}
@Test
- public void resolveItemTypeMatchesClusterNodeIgnoringCase() {
- Collection<String> types = Arrays.asList("profile", "event",
"clusterNode", "generic");
+ public void
resolveItemTypeMatchesKnownTypesIgnoringCaseAndRolloverNumber() {
+ Collection<String> types = Arrays.asList("profile", "profileAlias",
"event", "clusterNode");
assertEquals("clusterNode",
MigrationUtils.resolveItemType("context-clusternode", "context", types));
- assertEquals("clusterNode",
MigrationUtils.resolveItemType("context-clusterNode", "context", types));
- }
-
- @Test
- public void resolveItemTypeMatchesRolloverEvent() {
- Collection<String> types = Arrays.asList("event", "session",
"generic");
+ assertEquals("profileAlias",
MigrationUtils.resolveItemType("context-profilealias", "context", types));
assertEquals("event",
MigrationUtils.resolveItemType("context-event-000001", "context", types));
}
@Test
- public void resolveItemTypeKeepsUnknownIndexSuffix() {
- Collection<String> types = Arrays.asList("profile", "event",
"generic");
+ public void resolveItemTypeDoesNotMatchOnKnownTypePrefix() {
+ Collection<String> types = Arrays.asList("profile", "event");
+ assertEquals("profilealias",
MigrationUtils.resolveItemType("context-profilealias", "context", types));
+ assertEquals("eventfoo",
MigrationUtils.resolveItemType("context-eventfoo-000001", "context", types));
assertEquals("sfdcconfiguration",
MigrationUtils.resolveItemType("context-sfdcconfiguration", "context", types));
+ assertEquals("generic",
MigrationUtils.resolveItemType("other-profile", "context", types));
}
-
- @Test
- public void resolveItemTypePrefersLongerKnownType() {
- Collection<String> types = Arrays.asList("profile", "profileAlias");
- assertEquals("profileAlias",
MigrationUtils.resolveItemType("context-profilealias", "context", types));
- }
-}
\ No newline at end of file
+}
