This is an automated email from the ASF dual-hosted git repository.
sergehuber pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/unomi.git
The following commit(s) were added to refs/heads/master by this push:
new 04a8142de UNOMI-996: Let the 3.0 to 4.0 migration finish (#908)
04a8142de is described below
commit 04a8142deb0c77be446185849bce53d30845af35
Author: Serge Huber <[email protected]>
AuthorDate: Tue Oct 6 20:17:29 2026 +0200
UNOMI-996: Let the 3.0 to 4.0 migration finish (#908)
Co-authored-by: Claude Opus 5.5 <[email protected]>
---
.../test/java/org/apache/unomi/itests/AllITs.java | 2 +
.../apache/unomi/itests/CorePersistenceITs.java | 2 +
.../itests/migration/Migrate40IndexMappingIT.java | 169 +++++++++++++++++++++
tools/shell-commands/pom.xml | 23 +++
.../shell/migration/utils/MigrationUtils.java | 81 ++++++++--
.../migrate-4.0.0-01-tenantDocumentIds.groovy | 39 +++--
.../migrate-4.0.0-10-tenantInitialization.groovy | 3 +-
.../initialize_tenant_and_audit_fields.painless | 10 +-
.../shell/migration/utils/MigrationUtilsTest.java | 95 +++++++++++-
9 files changed, 396 insertions(+), 28 deletions(-)
diff --git a/itests/src/test/java/org/apache/unomi/itests/AllITs.java
b/itests/src/test/java/org/apache/unomi/itests/AllITs.java
index 61e39077e..4d377c022 100644
--- a/itests/src/test/java/org/apache/unomi/itests/AllITs.java
+++ b/itests/src/test/java/org/apache/unomi/itests/AllITs.java
@@ -18,6 +18,7 @@ package org.apache.unomi.itests;
import org.apache.unomi.itests.migration.Migrate16xToCurrentVersionIT;
import org.apache.unomi.itests.graphql.*;
+import org.apache.unomi.itests.migration.Migrate40IndexMappingIT;
import org.apache.unomi.itests.migration.MigrationIT;
import org.apache.unomi.itests.shell.*;
import org.junit.runner.RunWith;
@@ -34,6 +35,7 @@ import org.junit.runners.Suite.SuiteClasses;
@SuiteClasses({
Migrate16xToCurrentVersionIT.class,
MigrationIT.class,
+ Migrate40IndexMappingIT.class,
PersistenceServiceIT.class,
BasicIT.class,
ConditionEvaluatorIT.class,
diff --git
a/itests/src/test/java/org/apache/unomi/itests/CorePersistenceITs.java
b/itests/src/test/java/org/apache/unomi/itests/CorePersistenceITs.java
index 3d19d4538..bf50ac2b2 100644
--- a/itests/src/test/java/org/apache/unomi/itests/CorePersistenceITs.java
+++ b/itests/src/test/java/org/apache/unomi/itests/CorePersistenceITs.java
@@ -18,6 +18,7 @@ package org.apache.unomi.itests;
import org.apache.unomi.itests.migration.Migrate16xToCurrentVersionIT;
import org.apache.unomi.itests.graphql.*;
+import org.apache.unomi.itests.migration.Migrate40IndexMappingIT;
import org.apache.unomi.itests.migration.MigrationIT;
import org.apache.unomi.itests.shell.*;
import org.junit.runner.RunWith;
@@ -35,6 +36,7 @@ import org.junit.runners.Suite.SuiteClasses;
@SuiteClasses({
Migrate16xToCurrentVersionIT.class,
MigrationIT.class,
+ Migrate40IndexMappingIT.class,
PersistenceServiceIT.class,
BasicIT.class,
ConditionEvaluatorIT.class,
diff --git
a/itests/src/test/java/org/apache/unomi/itests/migration/Migrate40IndexMappingIT.java
b/itests/src/test/java/org/apache/unomi/itests/migration/Migrate40IndexMappingIT.java
new file mode 100644
index 000000000..b66b472f5
--- /dev/null
+++
b/itests/src/test/java/org/apache/unomi/itests/migration/Migrate40IndexMappingIT.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
+package org.apache.unomi.itests.migration;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.unomi.itests.BaseIT;
+import org.apache.unomi.itests.persistence.SearchBackendIT;
+import org.apache.unomi.shell.migration.utils.HttpUtils;
+import org.apache.unomi.shell.migration.utils.MigrationUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.osgi.framework.Bundle;
+import org.osgi.framework.BundleContext;
+import org.osgi.framework.FrameworkUtil;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * UNOMI-996: migration must find bundled mappings (including cluster nodes)
and
+ * copy the mapping of an unknown index instead of stopping.
+ */
+@Category(SearchBackendIT.class)
+public class Migrate40IndexMappingIT extends BaseIT {
+
+ private static final String UNKNOWN_INDEX = "context-unomi996unknown";
+ private static final String DEST_INDEX = "context-unomi996unknown-dest";
+
+ @After
+ public void deleteTestIndices() throws Exception {
+ if (!persistenceCapabilities().httpAdminApi()) {
+ return;
+ }
+ try (CloseableHttpClient httpClient = createSearchEngineHttpClient()) {
+ String es = getSearchEngineBaseUrl();
+ MigrationUtils.deleteIndex(httpClient, es, UNKNOWN_INDEX);
+ MigrationUtils.deleteIndex(httpClient, es, DEST_INDEX);
+ }
+ }
+
+ @Test
+ public void mappingsAreBundledWithTheMigrationCommandForBothEngines() {
+ // Look in the migration command bundle only: the started persistence
bundle ships these files too
+ Bundle migrationBundle = FrameworkUtil.getBundle(MigrationUtils.class);
+ for (String engine : Arrays.asList(SEARCH_ENGINE_ELASTICSEARCH,
SEARCH_ENGINE_OPENSEARCH)) {
+ for (String mappingFile : Arrays.asList("clusterNode.json",
"profile.json", "event.json", "geonameEntry.json")) {
+ String path = "META-INF/cxs/migration-mappings/" + engine +
"/" + mappingFile;
+ Assert.assertNotNull(path + " should be shipped with the
migration command", migrationBundle.getEntry(path));
+ }
+ }
+ }
+
+ @Test
+ public void searchEngineIsDetected() throws Exception {
+ assumeHttpAdminApi();
+ try (CloseableHttpClient httpClient = createSearchEngineHttpClient()) {
+ Assert.assertEquals(getPersistenceBackend().providerId(),
+ MigrationUtils.getSearchEngine(httpClient,
getSearchEngineBaseUrl()));
+ }
+ }
+
+ @Test
+ public void unknownIndexKeepsDocumentsOnTheConfiguredTenant() throws
Exception {
+ JsonNode hits = migrateUnknownIndex(true);
+ Assert.assertEquals(2, hits.size());
+ for (JsonNode hit : hits) {
+ String id = hit.path("_id").asText();
+ Assert.assertTrue("document id should use the configured tenant: "
+ id, id.startsWith(TEST_TENANT_ID + "_"));
+ Assert.assertEquals(TEST_TENANT_ID,
hit.path("_source").path("tenantId").asText());
+ }
+ }
+
+ @Test
+ public void knownIndexKeepsSystemScopeDocumentsOnTheSystemTenant() throws
Exception {
+ JsonNode hits = migrateUnknownIndex(false);
+ Assert.assertEquals(2, hits.size());
+ for (JsonNode hit : hits) {
+ boolean systemScope =
"system".equals(hit.path("_source").path("scope").asText());
+ String expectedTenant = systemScope ? "system" : TEST_TENANT_ID;
+ Assert.assertEquals(expectedTenant + "_" +
hit.path("_source").path("itemId").asText(), hit.path("_id").asText());
+ Assert.assertEquals(expectedTenant,
hit.path("_source").path("tenantId").asText());
+ }
+ }
+
+ private void assumeHttpAdminApi() {
+ Assume.assumeTrue(
+ "HTTP admin API required to create migration fixture indices
(provider="
+ + getPersistenceBackend().providerId() + ")",
+ persistenceCapabilities().httpAdminApi());
+ }
+
+ /**
+ * Reindexes a fixture index holding one document without scope and one
with scope "system"
+ * through the 4.0 tenant script, the way the migration does, and returns
the resulting hits.
+ */
+ private JsonNode migrateUnknownIndex(boolean forceDefaultTenant) throws
Exception {
+ assumeHttpAdminApi();
+
+ try (CloseableHttpClient httpClient = createSearchEngineHttpClient()) {
+ String es = getSearchEngineBaseUrl();
+ BundleContext migrationBundles =
FrameworkUtil.getBundle(MigrationUtils.class).getBundleContext();
+
+ String sourceSettings = "{"
+ +
"\"settings\":{\"number_of_shards\":1,\"number_of_replicas\":0},"
+ + "\"mappings\":{\"properties\":{"
+ + "\"itemId\":{\"type\":\"keyword\"},"
+ + "\"itemType\":{\"type\":\"keyword\"},"
+ + "\"scope\":{\"type\":\"keyword\"},"
+ + "\"tenantId\":{\"type\":\"keyword\"}"
+ + "}}}";
+ HttpUtils.executePutRequest(httpClient, es + "/" + UNKNOWN_INDEX,
sourceSettings, null);
+ HttpUtils.executePutRequest(httpClient, es + "/" + UNKNOWN_INDEX
+ + "/_doc/unknown1?refresh=true",
+
"{\"itemId\":\"unknown1\",\"itemType\":\"sfdcConfiguration\"}", null);
+ HttpUtils.executePutRequest(httpClient, es + "/" + UNKNOWN_INDEX
+ + "/_doc/unknown2?refresh=true",
+
"{\"itemId\":\"unknown2\",\"itemType\":\"sfdcConfiguration\",\"scope\":\"system\"}",
null);
+
+ String mapping =
MigrationUtils.extractMappingFromIndex(httpClient, es, UNKNOWN_INDEX);
+ Assert.assertTrue("copied mapping should keep itemType",
mapping.contains("itemType"));
+
+ String itemType = MigrationUtils.resolveItemType(UNKNOWN_INDEX,
"context",
+ Arrays.asList("profile", "event", "clusterNode",
"generic"));
+ Assert.assertEquals("unomi996unknown", itemType);
+
+ String destSettings =
"{\"settings\":{\"number_of_shards\":1,\"number_of_replicas\":0},\"mappings\":"
+ + mapping + "}";
+ MigrationUtils.createIndex(httpClient, es, DEST_INDEX,
destSettings);
+
+ Map<String, Object> params = new HashMap<>();
+ params.put("tenantId", TEST_TENANT_ID);
+ params.put("systemTenantId", "system");
+ params.put("forceDefaultTenant", forceDefaultTenant);
+ params.put("itemType", itemType);
+ params.put("date", "2026-01-01T00:00:00Z");
+ params.put("systemItems", Collections.emptySet());
+
+ String painless = MigrationUtils.getFileWithoutComments(
+ migrationBundles,
"requestBody/4.0.0/initialize_tenant_and_audit_fields.painless");
+ MigrationUtils.moveToIndex(httpClient, migrationBundles, es,
UNKNOWN_INDEX, DEST_INDEX, painless, params);
+ HttpUtils.executePostRequest(httpClient, es + "/" + DEST_INDEX +
"/_refresh", null, null);
+
+ String search = HttpUtils.executeGetRequest(httpClient,
+ es + "/" + DEST_INDEX + "/_search?size=10", null);
+ return new
ObjectMapper().readTree(search).path("hits").path("hits");
+ }
+ }
+}
diff --git a/tools/shell-commands/pom.xml b/tools/shell-commands/pom.xml
index 9451764d7..17ebbcced 100644
--- a/tools/shell-commands/pom.xml
+++ b/tools/shell-commands/pom.xml
@@ -157,6 +157,29 @@
</dependencies>
<build>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ </resource>
+ <!-- Mappings needed by the migration before Unomi is started.
Kept out of META-INF/cxs/mappings
+ so the persistence services do not load them as predefined
mappings. -->
+ <resource>
+
<directory>../../persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings</directory>
+
<targetPath>META-INF/cxs/migration-mappings/elasticsearch</targetPath>
+ </resource>
+ <resource>
+
<directory>../../extensions/geonames/services/src/main/resources/META-INF/cxs/mappings</directory>
+
<targetPath>META-INF/cxs/migration-mappings/elasticsearch</targetPath>
+ </resource>
+ <resource>
+
<directory>../../persistence-opensearch/core/src/main/resources/META-INF/cxs/mappings</directory>
+
<targetPath>META-INF/cxs/migration-mappings/opensearch</targetPath>
+ </resource>
+ <resource>
+
<directory>../../extensions/geonames/services/src/main/resources/META-INF/cxs/mappings</directory>
+
<targetPath>META-INF/cxs/migration-mappings/opensearch</targetPath>
+ </resource>
+ </resources>
<plugins>
<plugin>
<groupId>org.apache.felix</groupId>
diff --git
a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
index b80055576..4e8d71657 100644
---
a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
+++
b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
@@ -303,20 +303,68 @@ public class MigrationUtils {
}
public static String extractMappingFromBundles(BundleContext
bundleContext, String fileName) throws IOException {
- for (Bundle bundle : bundleContext.getBundles()) {
- Enumeration<URL> predefinedMappings =
bundle.findEntries("META-INF/cxs/mappings", fileName, true);
- if (predefinedMappings == null) {
- continue;
- }
- if (predefinedMappings.hasMoreElements()) {
- URL predefinedMappingURL = predefinedMappings.nextElement();
- return IOUtils.toString(predefinedMappingURL);
+ return extractMappingFromBundles(bundleContext, fileName,
"elasticsearch");
+ }
+
+ /**
+ * Looks up a mapping file in every OSGi bundle: first in {@code
META-INF/cxs/mappings}
+ * (the persistence bundles, when installed), then in the per-engine copy
shipped with the
+ * migration command bundle, which is available before Unomi is started.
The copy lives in a
+ * separate folder so the persistence services do not load it as a
predefined mapping.
+ *
+ * @param searchEngine {@code elasticsearch} or {@code opensearch}, see
{@link #getSearchEngine}
+ */
+ public static String extractMappingFromBundles(BundleContext
bundleContext, String fileName, String searchEngine) throws IOException {
+ for (String path : List.of("META-INF/cxs/mappings",
"META-INF/cxs/migration-mappings/" + searchEngine)) {
+ for (Bundle bundle : bundleContext.getBundles()) {
+ Enumeration<URL> predefinedMappings = bundle.findEntries(path,
fileName, true);
+ if (predefinedMappings != null &&
predefinedMappings.hasMoreElements()) {
+ return IOUtils.toString(predefinedMappings.nextElement(),
StandardCharsets.UTF_8);
+ }
}
}
-
throw new RuntimeException("no mapping found in bundles for: " +
fileName);
}
+ /**
+ * Copies the mapping currently stored on an index (GET {@code
/{index}/_mapping}).
+ * Used for indices that have no bundled 4.0 mapping.
+ */
+ public static String extractMappingFromIndex(CloseableHttpClient
httpClient, String esAddress, String indexName) throws IOException {
+ return
mappingsFromIndexResponse(HttpUtils.executeGetRequest(httpClient, esAddress +
"/" + indexName + "/_mapping", null));
+ }
+
+ static String mappingsFromIndexResponse(String responseBody) {
+ if (responseBody == null || responseBody.isBlank()) {
+ throw new IllegalArgumentException("Index mapping response is
empty");
+ }
+ JSONObject root = new JSONObject(responseBody);
+ if (root.length() != 1) {
+ throw new IllegalArgumentException("Expected the mapping of
exactly one index, got: " + root.keySet());
+ }
+ return
root.getJSONObject(root.keys().next()).getJSONObject("mappings").toString();
+ }
+
+ /**
+ * Maps an index name such as {@code context-event-000001} or {@code
context-clusternode}
+ * to an item type. The name after the prefix (without a rollover number)
must equal a known
+ * type, ignoring case; otherwise that name is returned as is. Names that
do not start with
+ * the prefix resolve to {@code generic}.
+ */
+ public static String resolveItemType(String indexName, String indexPrefix,
Collection<String> knownTypes) {
+ String prefix = indexPrefix + "-";
+ if (!indexName.regionMatches(true, 0, prefix, 0, prefix.length())) {
+ return "generic";
+ }
+ String name =
indexName.substring(prefix.length()).replaceFirst("-\\d{6}$", "");
+ for (String type : knownTypes) {
+ if (name.equalsIgnoreCase(type)) {
+ return type;
+ }
+ }
+ return name;
+ }
+
public static String buildIndexCreationRequest(String baseIndexSettings,
String mapping, MigrationContext context, boolean isMonthlyIndex) throws
IOException {
String settings = baseIndexSettings
.replace("#numberOfShards",
context.getConfigString(isMonthlyIndex ? MONTHLY_NUMBER_OF_SHARDS :
NUMBER_OF_SHARDS))
@@ -920,6 +968,21 @@ public class MigrationUtils {
return version.split("\\.")[0]; // Return major version number
}
+ /**
+ * Detects the engine behind the given address from its root endpoint.
+ *
+ * @return {@code opensearch} or {@code elasticsearch}
+ */
+ public static String getSearchEngine(CloseableHttpClient httpClient,
String esAddress) throws IOException {
+ return
searchEngineFromRootResponse(HttpUtils.executeGetRequest(httpClient, esAddress,
null));
+ }
+
+ static String searchEngineFromRootResponse(String responseBody) {
+ // Only OpenSearch reports a distribution
+ String distribution = new
JSONObject(responseBody).getJSONObject("version").optString("distribution");
+ return "opensearch".equals(distribution) ? "opensearch" :
"elasticsearch";
+ }
+
public interface ScrollCallback {
void execute(String hits);
}
diff --git
a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-4.0.0-01-tenantDocumentIds.groovy
b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-4.0.0-01-tenantDocumentIds.groovy
index 647aae6ac..f5f265e3d 100644
---
a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-4.0.0-01-tenantDocumentIds.groovy
+++
b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-4.0.0-01-tenantDocumentIds.groovy
@@ -67,22 +67,25 @@ def indexConfigs = [
mapping: "personaSession.json",
useRollover: false
],
- "generic": [
+ "profileAlias": [
baseSettings: "requestBody/2.0.0/base_index_mapping.json",
- mapping: null, // Will be determined dynamically from resolved
item type
+ mapping: "profileAlias.json",
useRollover: false
+ ],
+ "clusterNode": [
+ baseSettings: "requestBody/2.0.0/base_index_mapping.json",
+ mapping: "clusterNode.json",
+ useRollover: false,
+ forceDefaultTenant: true
+ ],
+ "generic": [
+ baseSettings: "requestBody/2.0.0/base_index_mapping.json",
+ mapping: null, // Copy the mapping currently stored on the
index
+ useRollover: false,
+ forceDefaultTenant: true
]
]
-// Helper function to resolve item type from index name
-def resolveItemType = { String indexName ->
- def type = indexConfigs.find { type, config ->
- indexName.startsWith("${indexPrefix}-${type}")
- }
- return type ? type.key : "generic"
-}
-
-// Helper function to get index configuration
def getIndexConfig = { String itemType ->
return indexConfigs[itemType] ?: indexConfigs["generic"]
}
@@ -117,23 +120,31 @@ context.performMigrationStep("4.0.0-get-all-indices", ()
-> {
// Get the Painless script
String updateScript = MigrationUtils.getFileWithoutComments(bundleContext,
"requestBody/4.0.0/initialize_tenant_and_audit_fields.painless")
+ String searchEngine =
MigrationUtils.getSearchEngine(context.getHttpClient(), esAddress)
+ context.printMessage("Search engine: " + searchEngine)
+
// Process each index (reindex them)
allIndices.each { indexName ->
context.printMessage("Processing index: " + indexName)
// Determine item type and get configuration
- String itemType = resolveItemType(indexName)
+ String itemType = MigrationUtils.resolveItemType(indexName,
indexPrefix, indexConfigs.keySet())
def indexConfig = getIndexConfig(itemType)
// Add item type to parameters
Map<String, Object> params = new HashMap<>(baseParams)
params.put("itemType", itemType)
+ // Cluster nodes and unknown indices keep site data in the configured
tenant, never "system"
+ boolean forceDefaultTenant = indexConfig.forceDefaultTenant == true
+ params.put("forceDefaultTenant", forceDefaultTenant)
// Get base settings and mapping
String baseSettings = MigrationUtils.resourceAsString(bundleContext,
indexConfig.baseSettings)
+ // A known type must find its bundled 4.0 mapping; only unknown types
copy the mapping stored on the index
String mapping = indexConfig.mapping ?
- MigrationUtils.extractMappingFromBundles(bundleContext,
indexConfig.mapping) :
- MigrationUtils.extractMappingFromBundles(bundleContext,
"${itemType}.json")
+ MigrationUtils.extractMappingFromBundles(bundleContext,
indexConfig.mapping, searchEngine) :
+
MigrationUtils.extractMappingFromIndex(context.getHttpClient(), esAddress,
indexName)
+ context.printMessage("Item type: ${itemType}, mapping:
${indexConfig.mapping ?: 'copied from the index'}, default tenant only:
${forceDefaultTenant}")
// Build index settings
String newIndexSettings
diff --git
a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-4.0.0-10-tenantInitialization.groovy
b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-4.0.0-10-tenantInitialization.groovy
index 995f09d69..dada0f0f4 100644
---
a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-4.0.0-10-tenantInitialization.groovy
+++
b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-4.0.0-10-tenantInitialization.groovy
@@ -71,7 +71,8 @@ if (Files.exists(secretsDir)) {
// Create the default tenant index and items
context.performMigrationStep("4.0.0-create-tenant-index", () -> {
String baseSettings = MigrationUtils.resourceAsString(bundleContext,
"requestBody/2.0.0/base_index_mapping.json")
- String mapping = MigrationUtils.extractMappingFromBundles(bundleContext,
"tenant.json")
+ String mapping = MigrationUtils.extractMappingFromBundles(bundleContext,
"tenant.json",
+ MigrationUtils.getSearchEngine(context.getHttpClient(), esAddress))
String newIndexSettings =
MigrationUtils.buildIndexCreationRequest(baseSettings, mapping, context, false)
if (!MigrationUtils.indexExists(context.getHttpClient(), esAddress,
"${indexPrefix}-tenant")) {
diff --git
a/tools/shell-commands/src/main/resources/requestBody/4.0.0/initialize_tenant_and_audit_fields.painless
b/tools/shell-commands/src/main/resources/requestBody/4.0.0/initialize_tenant_and_audit_fields.painless
index 07b89377b..7b089252d 100644
---
a/tools/shell-commands/src/main/resources/requestBody/4.0.0/initialize_tenant_and_audit_fields.painless
+++
b/tools/shell-commands/src/main/resources/requestBody/4.0.0/initialize_tenant_and_audit_fields.painless
@@ -18,8 +18,10 @@
if (ctx._id != null) {
// Skip if already has tenant prefix
if (!ctx._id.startsWith(params.tenantId + '_') &&
!ctx._id.startsWith(params.systemTenantId + '_')) {
- // For geonames or items with system scope, use system tenant
- if (ctx._index.endsWith('-geonames') || (ctx._source.scope != null &&
ctx._source.scope == 'system')) {
+ // Cluster nodes and unknown indices always stay in the configured
(default) tenant
+ if (params.forceDefaultTenant == true) {
+ ctx._id = params.tenantId + '_' + ctx._id;
+ } else if (ctx._index.endsWith('-geonames') || (ctx._source.scope !=
null && ctx._source.scope == 'system')) {
// Preserve the entire original document ID (it may contain
underscores, e.g., "dummy_scope")
// Do NOT split on underscores - this was a bug in the original
migration that caused itemIds
// like "dummy_scope" to become "dummy" when the document ID was
"dummy_scope_scope"
@@ -92,7 +94,9 @@ if (!ctx._index.endsWith('-systemitems') &&
!ctx._index.endsWith('-geonames')) {
// Set tenant ID in the source document based on scope for ALL items
if (ctx._source.tenantId == null) {
- if (ctx._index.endsWith('-geonames') || (ctx._source.scope != null &&
ctx._source.scope == 'system')) {
+ if (params.forceDefaultTenant == true) {
+ ctx._source.tenantId = params.tenantId;
+ } else if (ctx._index.endsWith('-geonames') || (ctx._source.scope != null
&& ctx._source.scope == 'system')) {
ctx._source.tenantId = params.systemTenantId;
} else {
ctx._source.tenantId = params.tenantId;
diff --git
a/tools/shell-commands/src/test/java/org/apache/unomi/shell/migration/utils/MigrationUtilsTest.java
b/tools/shell-commands/src/test/java/org/apache/unomi/shell/migration/utils/MigrationUtilsTest.java
index 96129f47c..92b48578c 100644
---
a/tools/shell-commands/src/test/java/org/apache/unomi/shell/migration/utils/MigrationUtilsTest.java
+++
b/tools/shell-commands/src/test/java/org/apache/unomi/shell/migration/utils/MigrationUtilsTest.java
@@ -25,6 +25,11 @@ import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.net.URL;
import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThrows;
@@ -209,4 +214,92 @@ public class MigrationUtilsTest {
() -> MigrationUtils.getFileWithoutComments(bundleContext,
"missing.painless"));
assertEquals("Resource not found: missing.painless", ex.getMessage());
}
-}
\ No newline at end of file
+
+ private static final String MAPPING =
"{\"properties\":{\"cpuLoad\":{\"type\":\"double\"}}}";
+
+ private String extractClusterNodeMappingFrom(String path) throws Exception
{
+ return extractClusterNodeMappingFrom(path, "elasticsearch");
+ }
+
+ private String extractClusterNodeMappingFrom(String path, String
searchEngine) throws Exception {
+ Path mappingFile = Files.createTempFile("clusterNode", ".json");
+ try {
+ Files.writeString(mappingFile, MAPPING);
+ when(bundleContext.getBundles()).thenReturn(new Bundle[]{bundle});
+ when(bundle.findEntries(path, "clusterNode.json", true))
+
.thenReturn(Collections.enumeration(Collections.singletonList(mappingFile.toUri().toURL())));
+ return MigrationUtils.extractMappingFromBundles(bundleContext,
"clusterNode.json", searchEngine);
+ } finally {
+ Files.deleteIfExists(mappingFile);
+ }
+ }
+
+ @Test
+ public void extractMappingFromBundlesFindsPersistenceMapping() throws
Exception {
+ assertEquals(MAPPING,
extractClusterNodeMappingFrom("META-INF/cxs/mappings"));
+ }
+
+ @Test
+ public void extractMappingFromBundlesFallsBackToMigrationCopy() throws
Exception {
+ assertEquals(MAPPING,
extractClusterNodeMappingFrom("META-INF/cxs/migration-mappings/elasticsearch"));
+ assertEquals(MAPPING,
extractClusterNodeMappingFrom("META-INF/cxs/migration-mappings/opensearch",
"opensearch"));
+ }
+
+ @Test
+ public void extractMappingFromBundlesIgnoresTheOtherEngineCopy() {
+ when(bundleContext.getBundles()).thenReturn(new Bundle[]{bundle});
+
when(bundle.findEntries("META-INF/cxs/migration-mappings/elasticsearch",
"event.json", true))
+
.thenReturn(Collections.enumeration(Collections.singletonList(getClass().getResource("/"))));
+
+ assertThrows(RuntimeException.class,
+ () -> MigrationUtils.extractMappingFromBundles(bundleContext,
"event.json", "opensearch"));
+ }
+
+ @Test
+ public void searchEngineIsDetectedFromTheRootResponse() {
+ assertEquals("opensearch", MigrationUtils.searchEngineFromRootResponse(
+
"{\"version\":{\"distribution\":\"opensearch\",\"number\":\"3.0.0\"}}"));
+ assertEquals("elasticsearch",
MigrationUtils.searchEngineFromRootResponse(
+ "{\"version\":{\"number\":\"9.1.0\"}}"));
+ }
+
+ @Test
+ public void extractMappingFromBundlesThrowsWhenMissing() {
+ when(bundleContext.getBundles()).thenReturn(new Bundle[]{bundle});
+
+ RuntimeException ex = assertThrows(RuntimeException.class,
+ () -> MigrationUtils.extractMappingFromBundles(bundleContext,
"generic.json"));
+ assertEquals("no mapping found in bundles for: generic.json",
ex.getMessage());
+ }
+
+ @Test
+ public void mappingsFromIndexResponseReadsTheIndexMappings() {
+ String body =
"{\"context-sfdcconfiguration\":{\"mappings\":{\"properties\":{\"itemId\":{\"type\":\"keyword\"}}}}}";
+ assertEquals("{\"properties\":{\"itemId\":{\"type\":\"keyword\"}}}",
MigrationUtils.mappingsFromIndexResponse(body));
+ }
+
+ @Test
+ public void mappingsFromIndexResponseRejectsAnythingButOneIndex() {
+ assertThrows(IllegalArgumentException.class, () ->
MigrationUtils.mappingsFromIndexResponse(""));
+ assertThrows(IllegalArgumentException.class, () ->
MigrationUtils.mappingsFromIndexResponse("{}"));
+ assertThrows(IllegalArgumentException.class,
+ () ->
MigrationUtils.mappingsFromIndexResponse("{\"a\":{\"mappings\":{}},\"b\":{\"mappings\":{}}}"));
+ }
+
+ @Test
+ public void
resolveItemTypeMatchesKnownTypesIgnoringCaseAndRolloverNumber() {
+ Collection<String> types = Arrays.asList("profile", "profileAlias",
"event", "clusterNode");
+ assertEquals("clusterNode",
MigrationUtils.resolveItemType("context-clusternode", "context", types));
+ assertEquals("profileAlias",
MigrationUtils.resolveItemType("context-profilealias", "context", types));
+ assertEquals("event",
MigrationUtils.resolveItemType("context-event-000001", "context", types));
+ }
+
+ @Test
+ public void resolveItemTypeDoesNotMatchOnKnownTypePrefix() {
+ Collection<String> types = Arrays.asList("profile", "event");
+ assertEquals("profilealias",
MigrationUtils.resolveItemType("context-profilealias", "context", types));
+ assertEquals("eventfoo",
MigrationUtils.resolveItemType("context-eventfoo-000001", "context", types));
+ assertEquals("sfdcconfiguration",
MigrationUtils.resolveItemType("context-sfdcconfiguration", "context", types));
+ assertEquals("generic",
MigrationUtils.resolveItemType("other-profile", "context", types));
+ }
+}
