This is an automated email from the ASF dual-hosted git repository. asf-gitbox-commits pushed a commit to branch UNOMI-997-migration-tenants in repository https://gitbox.apache.org/repos/asf/unomi.git
commit d59bd02b4308d0cde0cd48ca5f2a2b8a99c89c9a Author: Serge Huber <[email protected]> AuthorDate: Tue Oct 6 20:45:26 2026 +0200 UNOMI-997: Put migrated items in the right tenant Shipped definitions and geonames go to system. User rules, segments, scorings and goals with systemscope stay in the configured tenant. The itemId fix matches camelCase types and logs how many documents changed. --- .../test/java/org/apache/unomi/itests/AllITs.java | 2 + .../apache/unomi/itests/CorePersistenceITs.java | 2 + .../itests/migration/Migrate40IndexMappingIT.java | 1 + .../migration/Migrate40TenantAssignmentIT.java | 247 +++++++++++++++++++++ .../shell/migration/utils/MigrationUtils.java | 73 ++++++ .../migrate-4.0.0-01-tenantDocumentIds.groovy | 7 +- .../migrate-4.0.0-05-fixSystemItemIds.groovy | 49 ++-- .../initialize_tenant_and_audit_fields.painless | 73 +++--- .../4.0.0/shipped_system_definitions.json | 123 ++++++++++ .../shell/migration/utils/MigrationUtilsTest.java | 57 +++++ 10 files changed, 576 insertions(+), 58 deletions(-) diff --git a/itests/src/test/java/org/apache/unomi/itests/AllITs.java b/itests/src/test/java/org/apache/unomi/itests/AllITs.java index 4d377c022..b072a31fd 100644 --- a/itests/src/test/java/org/apache/unomi/itests/AllITs.java +++ b/itests/src/test/java/org/apache/unomi/itests/AllITs.java @@ -19,6 +19,7 @@ package org.apache.unomi.itests; import org.apache.unomi.itests.migration.Migrate16xToCurrentVersionIT; import org.apache.unomi.itests.graphql.*; import org.apache.unomi.itests.migration.Migrate40IndexMappingIT; +import org.apache.unomi.itests.migration.Migrate40TenantAssignmentIT; import org.apache.unomi.itests.migration.MigrationIT; import org.apache.unomi.itests.shell.*; import org.junit.runner.RunWith; @@ -36,6 +37,7 @@ import org.junit.runners.Suite.SuiteClasses; Migrate16xToCurrentVersionIT.class, MigrationIT.class, Migrate40IndexMappingIT.class, + Migrate40TenantAssignmentIT.class, PersistenceServiceIT.class, BasicIT.class, ConditionEvaluatorIT.class, diff --git a/itests/src/test/java/org/apache/unomi/itests/CorePersistenceITs.java b/itests/src/test/java/org/apache/unomi/itests/CorePersistenceITs.java index bf50ac2b2..6f3c510ef 100644 --- a/itests/src/test/java/org/apache/unomi/itests/CorePersistenceITs.java +++ b/itests/src/test/java/org/apache/unomi/itests/CorePersistenceITs.java @@ -19,6 +19,7 @@ package org.apache.unomi.itests; import org.apache.unomi.itests.migration.Migrate16xToCurrentVersionIT; import org.apache.unomi.itests.graphql.*; import org.apache.unomi.itests.migration.Migrate40IndexMappingIT; +import org.apache.unomi.itests.migration.Migrate40TenantAssignmentIT; import org.apache.unomi.itests.migration.MigrationIT; import org.apache.unomi.itests.shell.*; import org.junit.runner.RunWith; @@ -37,6 +38,7 @@ import org.junit.runners.Suite.SuiteClasses; Migrate16xToCurrentVersionIT.class, MigrationIT.class, Migrate40IndexMappingIT.class, + Migrate40TenantAssignmentIT.class, PersistenceServiceIT.class, BasicIT.class, ConditionEvaluatorIT.class, diff --git a/itests/src/test/java/org/apache/unomi/itests/migration/Migrate40IndexMappingIT.java b/itests/src/test/java/org/apache/unomi/itests/migration/Migrate40IndexMappingIT.java index b66b472f5..dc2f0ea70 100644 --- a/itests/src/test/java/org/apache/unomi/itests/migration/Migrate40IndexMappingIT.java +++ b/itests/src/test/java/org/apache/unomi/itests/migration/Migrate40IndexMappingIT.java @@ -155,6 +155,7 @@ public class Migrate40IndexMappingIT extends BaseIT { params.put("itemType", itemType); params.put("date", "2026-01-01T00:00:00Z"); params.put("systemItems", Collections.emptySet()); + params.put("shippedDefinitionIds", Collections.emptySet()); String painless = MigrationUtils.getFileWithoutComments( migrationBundles, "requestBody/4.0.0/initialize_tenant_and_audit_fields.painless"); diff --git a/itests/src/test/java/org/apache/unomi/itests/migration/Migrate40TenantAssignmentIT.java b/itests/src/test/java/org/apache/unomi/itests/migration/Migrate40TenantAssignmentIT.java new file mode 100644 index 000000000..16d24c8f8 --- /dev/null +++ b/itests/src/test/java/org/apache/unomi/itests/migration/Migrate40TenantAssignmentIT.java @@ -0,0 +1,247 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License + */ +package org.apache.unomi.itests.migration; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.unomi.itests.BaseIT; +import org.apache.unomi.itests.persistence.SearchBackendIT; +import org.apache.unomi.shell.migration.utils.HttpUtils; +import org.apache.unomi.shell.migration.utils.MigrationUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Assume; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.osgi.framework.BundleContext; +import org.osgi.framework.FrameworkUtil; + +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** + * UNOMI-997: after migration, shipped definitions and geonames land in system, + * while user content with systemscope stays in the configured tenant. + */ +@Category(SearchBackendIT.class) +public class Migrate40TenantAssignmentIT extends BaseIT { + + private static final String SOURCE_INDEX = "context-unomi997systemitems"; + private static final String DEST_INDEX = "context-unomi997systemitems-dest"; + private static final String GEO_SOURCE = "context-unomi997geonameentry"; + private static final String GEO_DEST = "context-unomi997geonameentry-dest"; + + @After + public void deleteTestIndices() throws Exception { + if (!persistenceCapabilities().httpAdminApi()) { + return; + } + try (CloseableHttpClient httpClient = createSearchEngineHttpClient()) { + String es = getSearchEngineBaseUrl(); + MigrationUtils.deleteIndex(httpClient, es, SOURCE_INDEX); + MigrationUtils.deleteIndex(httpClient, es, DEST_INDEX); + MigrationUtils.deleteIndex(httpClient, es, GEO_SOURCE); + MigrationUtils.deleteIndex(httpClient, es, GEO_DEST); + } + } + + @Test + public void shippedDefinitionsListIsPresentInTheMigrationBundle() throws Exception { + Set<String> shipped = MigrationUtils.loadShippedDefinitionIds( + FrameworkUtil.getBundle(MigrationUtils.class).getBundleContext()); + Assert.assertTrue(shipped.contains("conditiontype:booleancondition")); + Assert.assertTrue(shipped.contains("conditiontype:pasteventcondition")); + Assert.assertTrue(shipped.contains("conditiontype:profilepropertycondition")); + Assert.assertFalse("user content must not be in the shipped list by id alone", + shipped.contains("rule:my-user-rule")); + } + + @Test + public void migrationPutsEachKindOfItemInTheRightTenant() throws Exception { + Assume.assumeTrue( + "HTTP admin API required (provider=" + getPersistenceBackend().providerId() + ")", + persistenceCapabilities().httpAdminApi()); + + try (CloseableHttpClient httpClient = createSearchEngineHttpClient()) { + String es = getSearchEngineBaseUrl(); + BundleContext migrationBundles = FrameworkUtil.getBundle(MigrationUtils.class).getBundleContext(); + Set<String> shipped = MigrationUtils.loadShippedDefinitionIds(migrationBundles); + + String mapping = "{" + + "\"settings\":{\"number_of_shards\":1,\"number_of_replicas\":0}," + + "\"mappings\":{\"properties\":{" + + "\"itemId\":{\"type\":\"keyword\"}," + + "\"itemType\":{\"type\":\"keyword\"}," + + "\"scope\":{\"type\":\"keyword\"}," + + "\"tenantId\":{\"type\":\"keyword\"}" + + "}}}"; + HttpUtils.executePutRequest(httpClient, es + "/" + SOURCE_INDEX, mapping, null); + HttpUtils.executePutRequest(httpClient, es + "/" + SOURCE_INDEX + + "/_doc/booleanCondition?refresh=true", + "{\"itemId\":\"booleanCondition\",\"itemType\":\"conditionType\"}", null); + HttpUtils.executePutRequest(httpClient, es + "/" + SOURCE_INDEX + + "/_doc/my-user-rule?refresh=true", + "{\"itemId\":\"my-user-rule\",\"itemType\":\"rule\",\"scope\":\"systemscope\"}", null); + HttpUtils.executePutRequest(httpClient, es + "/" + SOURCE_INDEX + + "/_doc/my-segment?refresh=true", + "{\"itemId\":\"my-segment\",\"itemType\":\"segment\",\"scope\":\"systemscope\"}", null); + HttpUtils.executePutRequest(httpClient, es + "/" + SOURCE_INDEX + + "/_doc/my-scoring?refresh=true", + "{\"itemId\":\"my-scoring\",\"itemType\":\"scoring\",\"scope\":\"systemscope\"}", null); + HttpUtils.executePutRequest(httpClient, es + "/" + SOURCE_INDEX + + "/_doc/my-goal?refresh=true", + "{\"itemId\":\"my-goal\",\"itemType\":\"goal\",\"scope\":\"systemscope\"}", null); + + MigrationUtils.createIndex(httpClient, es, DEST_INDEX, mapping); + + Set<String> systemItems = new HashSet<>(); + Collections.addAll(systemItems, "conditiontype", "rule", "segment", "scoring", "goal"); + + Map<String, Object> params = new HashMap<>(); + params.put("tenantId", TEST_TENANT_ID); + params.put("systemTenantId", "system"); + params.put("forceDefaultTenant", false); + params.put("itemType", "systemitems"); + params.put("date", "2026-01-01T00:00:00Z"); + params.put("systemItems", systemItems); + params.put("shippedDefinitionIds", shipped); + + String painless = MigrationUtils.getFileWithoutComments( + migrationBundles, "requestBody/4.0.0/initialize_tenant_and_audit_fields.painless"); + MigrationUtils.moveToIndex(httpClient, migrationBundles, es, SOURCE_INDEX, DEST_INDEX, painless, params); + HttpUtils.executePostRequest(httpClient, es + "/" + DEST_INDEX + "/_refresh", null, null); + + JsonNode hits = new ObjectMapper().readTree(HttpUtils.executeGetRequest(httpClient, + es + "/" + DEST_INDEX + "/_search?size=20", null)).path("hits").path("hits"); + Assert.assertEquals(5, hits.size()); + + Map<String, JsonNode> byItemId = new HashMap<>(); + for (JsonNode hit : hits) { + byItemId.put(hit.path("_source").path("itemId").asText(), hit); + } + + assertTenant(byItemId.get("booleanCondition"), "system"); + assertTenant(byItemId.get("my-user-rule"), TEST_TENANT_ID); + assertTenant(byItemId.get("my-segment"), TEST_TENANT_ID); + assertTenant(byItemId.get("my-scoring"), TEST_TENANT_ID); + assertTenant(byItemId.get("my-goal"), TEST_TENANT_ID); + Assert.assertFalse("no user-created item may land in system", + byItemId.get("my-user-rule").path("_id").asText().startsWith("system_")); + } + } + + @Test + public void geonamesGoToTheSystemTenant() throws Exception { + Assume.assumeTrue( + "HTTP admin API required (provider=" + getPersistenceBackend().providerId() + ")", + persistenceCapabilities().httpAdminApi()); + + try (CloseableHttpClient httpClient = createSearchEngineHttpClient()) { + String es = getSearchEngineBaseUrl(); + BundleContext migrationBundles = FrameworkUtil.getBundle(MigrationUtils.class).getBundleContext(); + + String mapping = "{" + + "\"settings\":{\"number_of_shards\":1,\"number_of_replicas\":0}," + + "\"mappings\":{\"properties\":{" + + "\"itemId\":{\"type\":\"keyword\"}," + + "\"itemType\":{\"type\":\"keyword\"}," + + "\"tenantId\":{\"type\":\"keyword\"}" + + "}}}"; + HttpUtils.executePutRequest(httpClient, es + "/" + GEO_SOURCE, mapping, null); + HttpUtils.executePutRequest(httpClient, es + "/" + GEO_SOURCE + + "/_doc/paris?refresh=true", + "{\"itemId\":\"paris\",\"itemType\":\"geonameEntry\"}", null); + MigrationUtils.createIndex(httpClient, es, GEO_DEST, mapping); + + Map<String, Object> params = new HashMap<>(); + params.put("tenantId", TEST_TENANT_ID); + params.put("systemTenantId", "system"); + params.put("forceDefaultTenant", false); + params.put("itemType", "geonameentry"); + params.put("date", "2026-01-01T00:00:00Z"); + params.put("systemItems", Collections.emptySet()); + params.put("shippedDefinitionIds", Collections.emptySet()); + + String painless = MigrationUtils.getFileWithoutComments( + migrationBundles, "requestBody/4.0.0/initialize_tenant_and_audit_fields.painless"); + MigrationUtils.moveToIndex(httpClient, migrationBundles, es, GEO_SOURCE, GEO_DEST, painless, params); + HttpUtils.executePostRequest(httpClient, es + "/" + GEO_DEST + "/_refresh", null, null); + + JsonNode hit = new ObjectMapper().readTree(HttpUtils.executeGetRequest(httpClient, + es + "/" + GEO_DEST + "/_search?size=1", null)).path("hits").path("hits").get(0); + assertTenant(hit, "system"); + } + } + + @Test + public void camelCaseItemTypeKeywordQueryUpdatesDocuments() throws Exception { + Assume.assumeTrue( + "HTTP admin API required (provider=" + getPersistenceBackend().providerId() + ")", + persistenceCapabilities().httpAdminApi()); + + try (CloseableHttpClient httpClient = createSearchEngineHttpClient()) { + String es = getSearchEngineBaseUrl(); + BundleContext migrationBundles = FrameworkUtil.getBundle(MigrationUtils.class).getBundleContext(); + + String index = "context-unomi997camelcase"; + try { + String mapping = "{" + + "\"settings\":{" + + "\"number_of_shards\":1,\"number_of_replicas\":0," + + "\"analysis\":{\"analyzer\":{\"folding\":{\"type\":\"custom\",\"tokenizer\":\"keyword\",\"filter\":[\"lowercase\",\"asciifolding\"]}}}" + + "}," + + "\"mappings\":{\"properties\":{" + + "\"itemId\":{\"type\":\"keyword\"}," + + "\"itemType\":{\"type\":\"text\",\"analyzer\":\"folding\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}}" + + "}}}"; + HttpUtils.executePutRequest(httpClient, es + "/" + index, mapping, null); + HttpUtils.executePutRequest(httpClient, es + "/" + index + + "/_doc/system_booleanCondition_conditiontype?refresh=true", + "{\"itemId\":\"wrong\",\"itemType\":\"conditionType\"}", null); + + String fixScript = MigrationUtils.getFileWithoutComments( + migrationBundles, "requestBody/4.0.0/fix_system_item_ids.painless"); + String body = "{\"script\":{\"source\":" + + new ObjectMapper().writeValueAsString(fixScript) + + ",\"lang\":\"painless\"}," + + "\"query\":{\"bool\":{\"should\":[" + + "{\"term\":{\"itemType.keyword\":\"conditionType\"}}," + + "{\"term\":{\"itemType\":\"conditiontype\"}}" + + "],\"minimum_should_match\":1}}}"; + long updated = MigrationUtils.updateByQueryAndCount(httpClient, es, index, body); + Assert.assertEquals(1L, updated); + + JsonNode source = new ObjectMapper().readTree(HttpUtils.executeGetRequest(httpClient, + es + "/" + index + "/_doc/system_booleanCondition_conditiontype", null)) + .path("_source"); + Assert.assertEquals("booleanCondition_conditiontype", source.path("itemId").asText()); + } finally { + MigrationUtils.deleteIndex(httpClient, es, index); + } + } + } + + private static void assertTenant(JsonNode hit, String expectedTenant) { + Assert.assertNotNull(hit); + Assert.assertTrue(hit.path("_id").asText().startsWith(expectedTenant + "_")); + Assert.assertEquals(expectedTenant, hit.path("_source").path("tenantId").asText()); + } +} diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java index 4e8d71657..fbee87829 100644 --- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java +++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java @@ -365,6 +365,79 @@ public class MigrationUtils { return name; } + /** + * Loads the set of definitions Unomi ships and reloads at startup. + * Keys are {@code itemType:itemId} in lower case (for example {@code conditiontype:booleanCondition}). + * The migrate command bundle ships a snapshot of that list so it is available before plugins start. + */ + public static Set<String> loadShippedDefinitionIds(BundleContext bundleContext) throws IOException { + Set<String> ids = new LinkedHashSet<>(); + String resource = resourceAsString(bundleContext, "requestBody/4.0.0/shipped_system_definitions.json"); + JSONArray array = new JSONArray(resource); + for (int i = 0; i < array.length(); i++) { + String key = array.optString(i, null); + if (key != null && !key.isBlank()) { + ids.add(key.toLowerCase(Locale.ROOT)); + } + } + return ids; + } + + /** + * Strips a trailing {@code _itemType} suffix (any case) from a document or item id. + */ + public static String resolveBaseItemId(String documentOrItemId, String itemType) { + if (documentOrItemId == null || documentOrItemId.isBlank()) { + return documentOrItemId; + } + if (itemType == null || itemType.isBlank()) { + return documentOrItemId; + } + String suffix = "_" + itemType; + if (documentOrItemId.regionMatches(true, documentOrItemId.length() - suffix.length(), suffix, 0, suffix.length())) { + return documentOrItemId.substring(0, documentOrItemId.length() - suffix.length()); + } + return documentOrItemId; + } + + /** + * Decides whether a 3.0 item must land in the system tenant after migration. + * Shipped definitions and geonames go to system. User-created rules, segments, + * scorings and goals stay in the configured tenant even when their 3.0 scope is + * {@code systemscope}. Unknown indices forced to the default tenant never use system. + */ + public static boolean shouldAssignToSystemTenant(String itemType, String itemId, String scope, + String indexName, boolean forceDefaultTenant, + Collection<String> shippedDefinitionIds) { + if (forceDefaultTenant) { + return false; + } + String typeLower = itemType == null ? "" : itemType.toLowerCase(Locale.ROOT); + if ("geonameentry".equals(typeLower) + || (indexName != null && indexName.toLowerCase(Locale.ROOT).endsWith("-geonameentry"))) { + return true; + } + String baseId = resolveBaseItemId(itemId, itemType); + if (baseId != null && shippedDefinitionIds != null && !typeLower.isEmpty()) { + String key = typeLower + ":" + baseId.toLowerCase(Locale.ROOT); + if (shippedDefinitionIds.contains(key)) { + return true; + } + } + // Literal scope "system" only — not the 3.0 default "systemscope" used by user content + return "system".equals(scope); + } + + /** + * Same as {@link #updateByQuery(CloseableHttpClient, String, String, String)} but returns how + * many documents were updated, so callers can log the count. + */ + public static long updateByQueryAndCount(CloseableHttpClient httpClient, String esAddress, String indexName, String requestBody) throws Exception { + JSONObject response = new JSONObject(HttpUtils.executePostRequest( + httpClient, esAddress + "/" + indexName + "/_update_by_query?refresh=true&conflicts=proceed", requestBody, null)); + return response.optLong("updated", 0L); + } + public static String buildIndexCreationRequest(String baseIndexSettings, String mapping, MigrationContext context, boolean isMonthlyIndex) throws IOException { String settings = baseIndexSettings .replace("#numberOfShards", context.getConfigString(isMonthlyIndex ? MONTHLY_NUMBER_OF_SHARDS : NUMBER_OF_SHARDS)) diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-4.0.0-01-tenantDocumentIds.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-4.0.0-01-tenantDocumentIds.groovy index f5f265e3d..65e927682 100644 --- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-4.0.0-01-tenantDocumentIds.groovy +++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-4.0.0-01-tenantDocumentIds.groovy @@ -104,16 +104,21 @@ context.performMigrationStep("4.0.0-get-all-indices", () -> { Set<String> allItemTypes = MigrationUtils.getAllItemTypes(context.getHttpClient(), esAddress, indexPrefix, "*", bundleContext) context.printMessage("Found " + allItemTypes.size() + " item types") - // Get all system items from the systemitems index + // Get all system items from the systemitems index (store lower-case for painless checks) Set<String> systemItems = MigrationUtils.getAllItemTypes(context.getHttpClient(), esAddress, indexPrefix, "systemitems", bundleContext) + .collect { it.toLowerCase(java.util.Locale.ROOT) } as Set context.printMessage("Found " + systemItems.size() + " system items") + Set<String> shippedDefinitionIds = MigrationUtils.loadShippedDefinitionIds(bundleContext) + context.printMessage("Loaded " + shippedDefinitionIds.size() + " shipped definition ids") + // Create base parameters Map<String, Object> baseParams = new HashMap<>() baseParams.put("date", isoDate) baseParams.put("tenantId", tenantId) baseParams.put("systemTenantId", systemTenantId) baseParams.put("systemItems", systemItems) + baseParams.put("shippedDefinitionIds", shippedDefinitionIds) context.printMessage("Using tenant ID: " + tenantId) diff --git a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-4.0.0-05-fixSystemItemIds.groovy b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-4.0.0-05-fixSystemItemIds.groovy index c92606ddc..e5c8fbdd3 100644 --- a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-4.0.0-05-fixSystemItemIds.groovy +++ b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-4.0.0-05-fixSystemItemIds.groovy @@ -25,7 +25,7 @@ MigrationContext context = migrationContext String esAddress = context.getConfigString(CONFIG_ES_ADDRESS) String indexPrefix = context.getConfigString(INDEX_PREFIX) -// Get all system item types +// Get all system item types (keep the casing returned by the index for keyword queries) Set<String> systemItems = MigrationUtils.getAllItemTypes(context.getHttpClient(), esAddress, indexPrefix, "systemitems", bundleContext) context.printMessage("Found " + systemItems.size() + " system item types") @@ -38,48 +38,47 @@ context.printMessage("Found " + systemItems.size() + " system item types") // incorrectly processed the baseId, we need to fix the itemId in the source to match the document ID. context.performMigrationStep("4.0.0-fix-system-item-ids", () -> { String systemItemsIndex = "${indexPrefix}-systemitems" - + if (MigrationUtils.indexExists(context.getHttpClient(), esAddress, systemItemsIndex)) { context.printMessage("Fixing itemIds in systemitems index that end with itemType suffix") - - // Process each system item type + + long totalUpdated = 0L + + // Process each system item type. Match on itemType.keyword so camelCase values + // such as conditionType / actionType / propertyType are not missed (UNOMI-997). systemItems.each { itemType -> context.printMessage("Fixing items of type: ${itemType}") - - // Get the Painless script from file + String fixScript = MigrationUtils.getFileWithoutComments(bundleContext, "requestBody/4.0.0/fix_system_item_ids.painless") - - // Build the update request using JSONObject to properly escape the script - // This is the same approach used in MigrationUtils.getScriptPart() and other migrations + JSONObject scriptObj = new JSONObject() scriptObj.put("source", fixScript) scriptObj.put("lang", "painless") - - JSONObject queryObj = new JSONObject() - JSONObject termObj = new JSONObject() - termObj.put("itemType", itemType) - queryObj.put("term", termObj) - + + // Match camelCase item types on both the keyword sub-field and the folded text field + JSONObject keywordTerm = new JSONObject().put("term", new JSONObject().put("itemType.keyword", itemType)) + JSONObject foldedTerm = new JSONObject().put("term", new JSONObject().put("itemType", itemType.toLowerCase(java.util.Locale.ROOT))) + JSONObject boolQuery = new JSONObject() + boolQuery.put("should", [keywordTerm, foldedTerm]) + boolQuery.put("minimum_should_match", 1) + JSONObject queryObj = new JSONObject().put("bool", boolQuery) + JSONObject updateRequestObj = new JSONObject() updateRequestObj.put("script", scriptObj) updateRequestObj.put("query", queryObj) - - String updateRequest = updateRequestObj.toString() - + try { - MigrationUtils.updateByQuery(context.getHttpClient(), esAddress, systemItemsIndex, updateRequest) - context.printMessage("Fixed itemIds for item type: ${itemType}") + long updated = MigrationUtils.updateByQueryAndCount(context.getHttpClient(), esAddress, systemItemsIndex, updateRequestObj.toString()) + totalUpdated += updated + context.printMessage("Fixed ${updated} itemId(s) for item type: ${itemType}") } catch (Exception e) { context.printMessage("Warning: Could not fix itemIds for item type ${itemType}: ${e.getMessage()}") - // Continue with other item types even if one fails } } - - // Refresh the index to make changes visible + HttpUtils.executePostRequest(context.getHttpClient(), esAddress + "/${systemItemsIndex}/_refresh", null, null) - context.printMessage("Fixed itemIds in systemitems index") + context.printMessage("Fixed ${totalUpdated} itemId(s) in systemitems index") } else { context.printMessage("Systemitems index does not exist, skipping itemId fix") } }) - diff --git a/tools/shell-commands/src/main/resources/requestBody/4.0.0/initialize_tenant_and_audit_fields.painless b/tools/shell-commands/src/main/resources/requestBody/4.0.0/initialize_tenant_and_audit_fields.painless index 7b089252d..63521c208 100644 --- a/tools/shell-commands/src/main/resources/requestBody/4.0.0/initialize_tenant_and_audit_fields.painless +++ b/tools/shell-commands/src/main/resources/requestBody/4.0.0/initialize_tenant_and_audit_fields.painless @@ -14,37 +14,52 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +// Decide the tenant for this document (UNOMI-997). +// Shipped definitions and geonames go to system. User-created rules/segments/scorings/goals +// stay in the configured tenant even when their 3.0 scope is "systemscope". +boolean useSystemTenant = false; +if (params.forceDefaultTenant != true) { + def itemTypeLower = ctx._source.itemType != null ? ctx._source.itemType.toLowerCase() : ''; + def rawId = ctx._source.itemId != null ? ctx._source.itemId : ctx._id; + def baseId = rawId; + if (baseId != null && itemTypeLower != '' && baseId.toLowerCase().endsWith('_' + itemTypeLower)) { + baseId = baseId.substring(0, baseId.length() - itemTypeLower.length() - 1); + } + if (itemTypeLower == 'geonameentry' || (ctx._index != null && ctx._index.toLowerCase().endsWith('-geonameentry'))) { + useSystemTenant = true; + } else if (baseId != null && params.shippedDefinitionIds != null + && params.shippedDefinitionIds.contains(itemTypeLower + ':' + baseId.toLowerCase())) { + useSystemTenant = true; + } else if (ctx._source.scope != null && ctx._source.scope == 'system') { + // Literal "system" only — not the 3.0 default "systemscope" + useSystemTenant = true; + } +} +def assignedTenantId = useSystemTenant ? params.systemTenantId : params.tenantId; + // Update document ID with tenant prefix if (ctx._id != null) { // Skip if already has tenant prefix if (!ctx._id.startsWith(params.tenantId + '_') && !ctx._id.startsWith(params.systemTenantId + '_')) { - // Cluster nodes and unknown indices always stay in the configured (default) tenant - if (params.forceDefaultTenant == true) { - ctx._id = params.tenantId + '_' + ctx._id; - } else if (ctx._index.endsWith('-geonames') || (ctx._source.scope != null && ctx._source.scope == 'system')) { - // Preserve the entire original document ID (it may contain underscores, e.g., "dummy_scope") - // Do NOT split on underscores - this was a bug in the original migration that caused itemIds - // like "dummy_scope" to become "dummy" when the document ID was "dummy_scope_scope" - def baseId = ctx._id; - if (ctx._index.endsWith('-systemitems')) { - // For system items, ensure itemType is lowercase and matches the format in ElasticSearchPersistenceServiceImpl - def itemType = ctx._source.itemType != null ? ctx._source.itemType.toLowerCase() : null; - if (itemType != null && params.systemItems.contains(itemType)) { - ctx._id = ctx._source.scope == 'system' ? params.systemTenantId + '_' + baseId + '_' + itemType : params.tenantId + '_' + baseId + '_' + itemType; - } else { - ctx._id = ctx._source.scope == 'system' ? params.systemTenantId + '_' + baseId : params.tenantId + '_' + baseId; - } + def baseId = ctx._id; + if (ctx._index != null && ctx._index.toLowerCase().endsWith('-systemitems')) { + // For system items, ensure itemType is lowercase and matches the format in ElasticSearchPersistenceServiceImpl + def itemType = ctx._source.itemType != null ? ctx._source.itemType.toLowerCase() : null; + if (itemType != null && params.systemItems != null && params.systemItems.contains(itemType)) { + ctx._id = assignedTenantId + '_' + baseId + '_' + itemType; } else { - ctx._id = params.systemTenantId + '_' + baseId; + ctx._id = assignedTenantId + '_' + baseId; } } else { - ctx._id = params.tenantId + '_' + ctx._id; + ctx._id = assignedTenantId + '_' + baseId; } } } -// Update audit fields -if (!ctx._index.endsWith('-systemitems') && !ctx._index.endsWith('-geonames')) { +// Update audit fields (not for systemitems or geonames) +def indexLower = ctx._index != null ? ctx._index.toLowerCase() : ''; +if (!indexLower.endsWith('-systemitems') && !indexLower.endsWith('-geonameentry')) { // Handle creation date based on item type if (ctx._source.creationDate == null) { if (params.itemType == 'profile' && ctx._source.properties != null) { @@ -59,7 +74,7 @@ if (!ctx._index.endsWith('-systemitems') && !ctx._index.endsWith('-geonames')) { ctx._source.creationDate = params.date; } } - + // Handle last modification date based on item type if (ctx._source.lastModificationDate == null) { if (params.itemType == 'profile' && ctx._source.properties != null) { @@ -74,7 +89,7 @@ if (!ctx._index.endsWith('-systemitems') && !ctx._index.endsWith('-geonames')) { ctx._source.lastModificationDate = ctx._source.creationDate; } } - + // Set creator fields if (ctx._source.createdBy == null) { ctx._source.createdBy = 'system-migration-4.0.0'; @@ -82,7 +97,7 @@ if (!ctx._index.endsWith('-systemitems') && !ctx._index.endsWith('-geonames')) { if (ctx._source.lastModifiedBy == null) { ctx._source.lastModifiedBy = 'system-migration-4.0.0'; } - + // Initialize source tracking fields if (ctx._source.sourceInstanceId == null) { ctx._source.sourceInstanceId = null; @@ -92,13 +107,7 @@ if (!ctx._index.endsWith('-systemitems') && !ctx._index.endsWith('-geonames')) { } } -// Set tenant ID in the source document based on scope for ALL items +// Set tenant ID in the source document if (ctx._source.tenantId == null) { - if (params.forceDefaultTenant == true) { - ctx._source.tenantId = params.tenantId; - } else if (ctx._index.endsWith('-geonames') || (ctx._source.scope != null && ctx._source.scope == 'system')) { - ctx._source.tenantId = params.systemTenantId; - } else { - ctx._source.tenantId = params.tenantId; - } -} \ No newline at end of file + ctx._source.tenantId = assignedTenantId; +} diff --git a/tools/shell-commands/src/main/resources/requestBody/4.0.0/shipped_system_definitions.json b/tools/shell-commands/src/main/resources/requestBody/4.0.0/shipped_system_definitions.json new file mode 100644 index 000000000..22f1bfda8 --- /dev/null +++ b/tools/shell-commands/src/main/resources/requestBody/4.0.0/shipped_system_definitions.json @@ -0,0 +1,123 @@ +[ + "actiontype:addtolistsaction", + "actiontype:alleventtoprofilepropertiesaction", + "actiontype:cdpsessioneventaction", + "actiontype:copypropertiesaction", + "actiontype:evaluateprofileageaction", + "actiontype:evaluateprofilesegmentsaction", + "actiontype:evaluatevisitpropertiesaction", + "actiontype:eventtoprofilepropertyaction", + "actiontype:incrementpropertyaction", + "actiontype:incrementtweetnumberaction", + "actiontype:mergeprofilesonpropertyaction", + "actiontype:modifyconsentaction", + "actiontype:requestheadertoprofilepropertyaction", + "actiontype:requestparametertoprofilepropertyaction", + "actiontype:sendeventaction", + "actiontype:sendmailaction", + "actiontype:seteventoccurencecountaction", + "actiontype:setpropertyaction", + "actiontype:setremotehostinfoaction", + "actiontype:sfdccreateorupdateleadaction", + "actiontype:sfdcupdateprofilefromleadaction", + "actiontype:testgroovyaction", + "actiontype:trainingnotifaction", + "actiontype:updateconsenteventaction", + "actiontype:updatelistseventaction", + "actiontype:updatepropertiesaction", + "actiontype:weatherupdateaction", + "conditiontype:booleancondition", + "conditiontype:cdpsessioneventcondition", + "conditiontype:eventpropertycondition", + "conditiontype:eventtypecondition", + "conditiontype:formeventcondition", + "conditiontype:geolocationbypointsessioncondition", + "conditiontype:geolocationsessioncondition", + "conditiontype:goalmatchcondition", + "conditiontype:hovereventcondition", + "conditiontype:idscondition", + "conditiontype:matchallcondition", + "conditiontype:modifyanyconsenteventcondition", + "conditiontype:modifyconsenteventcondition", + "conditiontype:nestedcondition", + "conditiontype:newvisitorcondition", + "conditiontype:notcondition", + "conditiontype:pasteventcondition", + "conditiontype:profilealiasespropertycondition", + "conditiontype:profilepropertycondition", + "conditiontype:profilesegmentcondition", + "conditiontype:profileupdatedeventcondition", + "conditiontype:profileuserlistcondition", + "conditiontype:returningvisitorcondition", + "conditiontype:scoringcondition", + "conditiontype:sessioncreatedeventcondition", + "conditiontype:sessiondurationcondition", + "conditiontype:sessionpropertycondition", + "conditiontype:sourceeventpropertycondition", + "conditiontype:topicpropertycondition", + "conditiontype:updateconsenteventcondition", + "conditiontype:updatelistseventcondition", + "conditiontype:updatepropertieseventcondition", + "conditiontype:userlistpropertycondition", + "conditiontype:videovieweventcondition", + "propertytype:address", + "propertytype:age", + "propertytype:birthdate", + "propertytype:city", + "propertytype:company", + "propertytype:countryname", + "propertytype:devicecategory", + "propertytype:email", + "propertytype:facebookid", + "propertytype:firstname", + "propertytype:firstvisit", + "propertytype:gender", + "propertytype:googleid", + "propertytype:income", + "propertytype:jobtitle", + "propertytype:kids", + "propertytype:lastname", + "propertytype:lastvisit", + "propertytype:latitude", + "propertytype:leadassignedto", + "propertytype:linkedinid", + "propertytype:longitude", + "propertytype:maritalstatus", + "propertytype:nationality", + "propertytype:nbofvisits", + "propertytype:operatingsystemfamily", + "propertytype:operatingsystemname", + "propertytype:pagereferringurl", + "propertytype:phonenumber", + "propertytype:previousvisit", + "propertytype:sessionadminsubdiv1", + "propertytype:sessionadminsubdiv2", + "propertytype:sessioncity", + "propertytype:sessioncountrycode", + "propertytype:sessioncountryname", + "propertytype:totalnbofvisits", + "propertytype:twitterid", + "propertytype:useragentname", + "propertytype:useragentversion", + "propertytype:weatherlike", + "propertytype:weathertemperature", + "propertytype:weatherwinddirection", + "propertytype:weatherwindspeed", + "propertytype:zipcode", + "rule:_8s3odnl0n_sessionreferrer", + "rule:_ajhg9u2s5_sessionassigned", + "rule:_ed2tvbd9p_evaluateprofileage.json", + "rule:_i6iqbc3x8_sessionpagereferrer", + "rule:_t7caz8npj_evaluateprofilesegments", + "rule:_z21qiv8b1_newsession", + "rule:cdp_sessionevent", + "rule:examplelogin", + "rule:modifyanyconsent", + "rule:optimizationtest", + "rule:smp:incrementtweetnumber", + "rule:smp:trainedrule", + "rule:testgroovyactionrule", + "rule:updateconsent", + "rule:updatelists", + "rule:updateproperties" +] diff --git a/tools/shell-commands/src/test/java/org/apache/unomi/shell/migration/utils/MigrationUtilsTest.java b/tools/shell-commands/src/test/java/org/apache/unomi/shell/migration/utils/MigrationUtilsTest.java index 92b48578c..cb0322d1e 100644 --- a/tools/shell-commands/src/test/java/org/apache/unomi/shell/migration/utils/MigrationUtilsTest.java +++ b/tools/shell-commands/src/test/java/org/apache/unomi/shell/migration/utils/MigrationUtilsTest.java @@ -32,7 +32,9 @@ import java.util.Collection; import java.util.Collections; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.*; public class MigrationUtilsTest { @@ -302,4 +304,59 @@ public class MigrationUtilsTest { assertEquals("sfdcconfiguration", MigrationUtils.resolveItemType("context-sfdcconfiguration", "context", types)); assertEquals("generic", MigrationUtils.resolveItemType("other-profile", "context", types)); } + + @Test + public void resolveBaseItemIdStripsItemTypeSuffixIgnoringCase() { + assertEquals("booleanCondition", + MigrationUtils.resolveBaseItemId("booleanCondition_conditionType", "conditionType")); + assertEquals("booleanCondition", + MigrationUtils.resolveBaseItemId("booleanCondition_conditiontype", "conditionType")); + assertEquals("booleanCondition", + MigrationUtils.resolveBaseItemId("booleanCondition", "conditionType")); + } + + @Test + public void shouldAssignToSystemTenantForShippedConditionWithoutScope() { + Collection<String> shipped = Collections.singleton("conditiontype:booleancondition"); + assertTrue(MigrationUtils.shouldAssignToSystemTenant( + "conditionType", "booleanCondition", null, "context-systemitems", false, shipped)); + } + + @Test + public void shouldAssignUserContentWithSystemScopeToDefaultTenant() { + Collection<String> shipped = Collections.singleton("conditiontype:booleancondition"); + assertFalse(MigrationUtils.shouldAssignToSystemTenant( + "rule", "my-user-rule", "systemscope", "context-systemitems", false, shipped)); + assertFalse(MigrationUtils.shouldAssignToSystemTenant( + "segment", "my-segment", "systemscope", "context-systemitems", false, shipped)); + assertFalse(MigrationUtils.shouldAssignToSystemTenant( + "scoring", "my-scoring", "systemscope", "context-systemitems", false, shipped)); + assertFalse(MigrationUtils.shouldAssignToSystemTenant( + "goal", "my-goal", "systemscope", "context-systemitems", false, shipped)); + } + + @Test + public void shouldAssignGeonamesToSystemTenant() { + assertTrue(MigrationUtils.shouldAssignToSystemTenant( + "geonameEntry", "paris", null, "context-geonameentry", false, Collections.emptySet())); + assertTrue(MigrationUtils.shouldAssignToSystemTenant( + null, "paris", null, "context-geonameentry", false, Collections.emptySet())); + } + + @Test + public void shouldAssignProfilesEventsAndForcedDefaultToConfiguredTenant() { + Collection<String> shipped = Collections.singleton("conditiontype:booleancondition"); + assertFalse(MigrationUtils.shouldAssignToSystemTenant( + "profile", "p1", null, "context-profile", false, shipped)); + assertFalse(MigrationUtils.shouldAssignToSystemTenant( + "event", "e1", null, "context-event-000001", false, shipped)); + assertFalse(MigrationUtils.shouldAssignToSystemTenant( + "clusterNode", "node1", "system", "context-clusternode", true, shipped)); + } + + @Test + public void shouldAssignLiteralSystemScopeToSystemTenant() { + assertTrue(MigrationUtils.shouldAssignToSystemTenant( + "sfdcConfiguration", "cfg1", "system", "context-sfdcconfiguration", false, Collections.emptySet())); + } }
