This is an automated email from the ASF dual-hosted git repository.
sergehuber pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/unomi.git
The following commit(s) were added to refs/heads/master by this push:
new a01866516 UNOMI-997: Put migrated items in the right tenant (#909)
a01866516 is described below
commit a01866516ff54e61d48cc2058ac1e2ffeaeb7ca6
Author: Serge Huber <[email protected]>
AuthorDate: Wed Oct 7 12:30:43 2026 +0200
UNOMI-997: Put migrated items in the right tenant (#909)
Co-authored-by: Claude Opus 5.5 <[email protected]>
---
.../test/java/org/apache/unomi/itests/AllITs.java | 2 +
.../apache/unomi/itests/CorePersistenceITs.java | 2 +
.../migration/Migrate16xToCurrentVersionIT.java | 17 +-
.../itests/migration/Migrate40IndexMappingIT.java | 1 +
.../migration/Migrate40TenantAssignmentIT.java | 280 +++++++++++++++++++++
rest/pom.xml | 6 -
.../cache/AbstractMultiTypeCachingService.java | 23 +-
.../cache/AbstractMultiTypeCachingServiceTest.java | 50 ++++
.../shell/migration/utils/MigrationUtils.java | 60 +++++
.../migrate-4.0.0-01-tenantDocumentIds.groovy | 9 +-
.../migrate-4.0.0-05-fixSystemItemIds.groovy | 54 ++--
.../requestBody/4.0.0/fix_system_item_ids.painless | 37 +--
.../initialize_tenant_and_audit_fields.painless | 84 ++++---
.../4.0.0/shipped_system_definitions.json | 127 ++++++++++
.../shell/migration/utils/MigrationUtilsTest.java | 107 ++++++++
15 files changed, 741 insertions(+), 118 deletions(-)
diff --git a/itests/src/test/java/org/apache/unomi/itests/AllITs.java
b/itests/src/test/java/org/apache/unomi/itests/AllITs.java
index 4d377c022..b072a31fd 100644
--- a/itests/src/test/java/org/apache/unomi/itests/AllITs.java
+++ b/itests/src/test/java/org/apache/unomi/itests/AllITs.java
@@ -19,6 +19,7 @@ package org.apache.unomi.itests;
import org.apache.unomi.itests.migration.Migrate16xToCurrentVersionIT;
import org.apache.unomi.itests.graphql.*;
import org.apache.unomi.itests.migration.Migrate40IndexMappingIT;
+import org.apache.unomi.itests.migration.Migrate40TenantAssignmentIT;
import org.apache.unomi.itests.migration.MigrationIT;
import org.apache.unomi.itests.shell.*;
import org.junit.runner.RunWith;
@@ -36,6 +37,7 @@ import org.junit.runners.Suite.SuiteClasses;
Migrate16xToCurrentVersionIT.class,
MigrationIT.class,
Migrate40IndexMappingIT.class,
+ Migrate40TenantAssignmentIT.class,
PersistenceServiceIT.class,
BasicIT.class,
ConditionEvaluatorIT.class,
diff --git
a/itests/src/test/java/org/apache/unomi/itests/CorePersistenceITs.java
b/itests/src/test/java/org/apache/unomi/itests/CorePersistenceITs.java
index bf50ac2b2..6f3c510ef 100644
--- a/itests/src/test/java/org/apache/unomi/itests/CorePersistenceITs.java
+++ b/itests/src/test/java/org/apache/unomi/itests/CorePersistenceITs.java
@@ -19,6 +19,7 @@ package org.apache.unomi.itests;
import org.apache.unomi.itests.migration.Migrate16xToCurrentVersionIT;
import org.apache.unomi.itests.graphql.*;
import org.apache.unomi.itests.migration.Migrate40IndexMappingIT;
+import org.apache.unomi.itests.migration.Migrate40TenantAssignmentIT;
import org.apache.unomi.itests.migration.MigrationIT;
import org.apache.unomi.itests.shell.*;
import org.junit.runner.RunWith;
@@ -37,6 +38,7 @@ import org.junit.runners.Suite.SuiteClasses;
Migrate16xToCurrentVersionIT.class,
MigrationIT.class,
Migrate40IndexMappingIT.class,
+ Migrate40TenantAssignmentIT.class,
PersistenceServiceIT.class,
BasicIT.class,
ConditionEvaluatorIT.class,
diff --git
a/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xToCurrentVersionIT.java
b/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xToCurrentVersionIT.java
index 3a66818f5..63e0755ef 100644
---
a/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xToCurrentVersionIT.java
+++
b/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xToCurrentVersionIT.java
@@ -611,7 +611,7 @@ public class Migrate16xToCurrentVersionIT extends BaseIT {
}
// Check audit metadata
- checkAuditMetadata(source);
+ checkAuditMetadata(source, itemId);
}
}
}
@@ -619,22 +619,23 @@ public class Migrate16xToCurrentVersionIT extends BaseIT {
/**
* Helper method to check audit metadata fields
* @param source The document source containing the metadata
+ * @param itemId Document id included in assertion messages
*/
- private void checkAuditMetadata(JsonNode source) {
- Assert.assertNotNull("Created by should be set",
source.get("createdBy"));
+ private void checkAuditMetadata(JsonNode source, String itemId) {
+ Assert.assertNotNull("Created by should be set for " + itemId,
source.get("createdBy"));
String createdBy = source.get("createdBy").asText();
// After migration, documents may be refreshed by bundles during
startup,
// which changes createdBy from system-migration-4.0.0 to system-bundle
// Both are valid - migration sets it, bundles may refresh it
boolean isValidCreatedBy = "system-migration-4.0.0".equals(createdBy)
|| "system-bundle".equals(createdBy);
- Assert.assertTrue("Created by should be system-migration-4.0.0 or
system-bundle, but was: " + createdBy, isValidCreatedBy);
- Assert.assertNotNull("Creation date should be set",
source.get("creationDate"));
- Assert.assertNotNull("Last modified by should be set",
source.get("lastModifiedBy"));
+ Assert.assertTrue("Created by should be system-migration-4.0.0 or
system-bundle for " + itemId + ", but was: " + createdBy, isValidCreatedBy);
+ Assert.assertNotNull("Creation date should be set for " + itemId,
source.get("creationDate"));
+ Assert.assertNotNull("Last modified by should be set for " + itemId,
source.get("lastModifiedBy"));
String lastModifiedBy = source.get("lastModifiedBy").asText();
// Similarly, lastModifiedBy may be updated by bundles after migration
boolean isValidLastModifiedBy =
"system-migration-4.0.0".equals(lastModifiedBy) ||
"system-bundle".equals(lastModifiedBy);
- Assert.assertTrue("Last modified by should be system-migration-4.0.0
or system-bundle, but was: " + lastModifiedBy, isValidLastModifiedBy);
- Assert.assertNotNull("Last modification date should be set",
source.get("lastModificationDate"));
+ Assert.assertTrue("Last modified by should be system-migration-4.0.0
or system-bundle for " + itemId + ", but was: " + lastModifiedBy,
isValidLastModifiedBy);
+ Assert.assertNotNull("Last modification date should be set for " +
itemId, source.get("lastModificationDate"));
}
/**
diff --git
a/itests/src/test/java/org/apache/unomi/itests/migration/Migrate40IndexMappingIT.java
b/itests/src/test/java/org/apache/unomi/itests/migration/Migrate40IndexMappingIT.java
index b66b472f5..04fba0729 100644
---
a/itests/src/test/java/org/apache/unomi/itests/migration/Migrate40IndexMappingIT.java
+++
b/itests/src/test/java/org/apache/unomi/itests/migration/Migrate40IndexMappingIT.java
@@ -155,6 +155,7 @@ public class Migrate40IndexMappingIT extends BaseIT {
params.put("itemType", itemType);
params.put("date", "2026-01-01T00:00:00Z");
params.put("systemItems", Collections.emptySet());
+ params.put("shippedDefinitionIds", Collections.emptyMap());
String painless = MigrationUtils.getFileWithoutComments(
migrationBundles,
"requestBody/4.0.0/initialize_tenant_and_audit_fields.painless");
diff --git
a/itests/src/test/java/org/apache/unomi/itests/migration/Migrate40TenantAssignmentIT.java
b/itests/src/test/java/org/apache/unomi/itests/migration/Migrate40TenantAssignmentIT.java
new file mode 100644
index 000000000..4c466756d
--- /dev/null
+++
b/itests/src/test/java/org/apache/unomi/itests/migration/Migrate40TenantAssignmentIT.java
@@ -0,0 +1,280 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
+package org.apache.unomi.itests.migration;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.unomi.itests.BaseIT;
+import org.apache.unomi.itests.persistence.SearchBackendIT;
+import org.apache.unomi.shell.migration.utils.HttpUtils;
+import org.apache.unomi.shell.migration.utils.MigrationUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.osgi.framework.BundleContext;
+import org.osgi.framework.FrameworkUtil;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * UNOMI-997: after migration, shipped definitions and geonames land in system,
+ * while user content with systemscope stays in the configured tenant.
+ */
+@Category(SearchBackendIT.class)
+public class Migrate40TenantAssignmentIT extends BaseIT {
+
+ private static final String SOURCE_INDEX =
"context-unomi997source-systemitems";
+ private static final String DEST_INDEX =
"context-unomi997dest-systemitems";
+ private static final String GEO_SOURCE =
"context-unomi997source-geonameentry";
+ private static final String GEO_DEST = "context-unomi997dest-geonameentry";
+ private static final String DATA_SOURCE = "context-unomi997source-profile";
+ private static final String DATA_DEST = "context-unomi997dest-profile";
+ private static final String CAMEL_CASE_INDEX = "context-unomi997camelcase";
+
+ private static final String MAPPING = "{"
+ + "\"settings\":{\"number_of_shards\":1,\"number_of_replicas\":0},"
+ + "\"mappings\":{\"properties\":{"
+ + "\"itemId\":{\"type\":\"keyword\"},"
+ + "\"itemType\":{\"type\":\"keyword\"},"
+ + "\"scope\":{\"type\":\"keyword\"},"
+ + "\"tenantId\":{\"type\":\"keyword\"}"
+ + "}}}";
+
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+
+ @After
+ public void deleteTestIndices() throws Exception {
+ if (!persistenceCapabilities().httpAdminApi()) {
+ return;
+ }
+ try (CloseableHttpClient httpClient = createSearchEngineHttpClient()) {
+ String es = getSearchEngineBaseUrl();
+ for (String index : new String[]{SOURCE_INDEX, DEST_INDEX,
GEO_SOURCE, GEO_DEST, DATA_SOURCE, DATA_DEST, CAMEL_CASE_INDEX}) {
+ MigrationUtils.deleteIndex(httpClient, es, index);
+ }
+ }
+ }
+
+ @Test
+ public void shippedDefinitionsListIsPresentInTheMigrationBundle() {
+ Map<String, Set<String>> shipped =
MigrationUtils.loadShippedDefinitionIds(migrationBundle());
+
Assert.assertTrue(shipped.get("conditiontype").contains("booleancondition"));
+
Assert.assertTrue(shipped.get("conditiontype").contains("pasteventcondition"));
+
Assert.assertTrue(shipped.get("conditiontype").contains("profilepropertycondition"));
+ Assert.assertFalse("user content must not be in the shipped list by id
alone",
+ shipped.get("rule").contains("my-user-rule"));
+ }
+
+ @Test
+ public void migrationPutsEachKindOfItemInTheRightTenant() throws Exception
{
+ assumeHttpAdminApi();
+
+ try (CloseableHttpClient httpClient = createSearchEngineHttpClient()) {
+ String es = getSearchEngineBaseUrl();
+ HttpUtils.executePutRequest(httpClient, es + "/" + SOURCE_INDEX,
MAPPING, null);
+ // Shipped definitions: without scope, with the 3.0 default scope,
and with the 3.0 _itemtype id suffix
+ indexDocument(httpClient, SOURCE_INDEX, "booleanCondition",
"booleanCondition", "conditionType", null);
+ indexDocument(httpClient, SOURCE_INDEX, "updateProperties",
"updateProperties", "rule", "systemscope");
+ indexDocument(httpClient, SOURCE_INDEX,
"pastEventCondition_conditiontype", "pastEventCondition_conditiontype",
"conditionType", null);
+ // User content with the 3.0 default scope
+ indexDocument(httpClient, SOURCE_INDEX, "my-user-rule",
"my-user-rule", "rule", "systemscope");
+ indexDocument(httpClient, SOURCE_INDEX, "my-segment",
"my-segment", "segment", "systemscope");
+ indexDocument(httpClient, SOURCE_INDEX, "my-scoring",
"my-scoring", "scoring", "systemscope");
+ indexDocument(httpClient, SOURCE_INDEX, "my-goal_goal", "my-goal",
"goal", "systemscope");
+ // Literal system scope on a type that is neither shipped nor a
system item type
+ indexDocument(httpClient, SOURCE_INDEX, "cfg1", "cfg1",
"sfdcConfiguration", "system");
+
+ Map<String, JsonNode> byItemId = migrate(httpClient, SOURCE_INDEX,
DEST_INDEX, "systemitems", false,
+ Set.of("conditiontype", "rule", "segment", "scoring",
"goal"),
+
MigrationUtils.loadShippedDefinitionIds(migrationBundle()));
+ Assert.assertEquals(8, byItemId.size());
+
+ assertTenant(byItemId.get("booleanCondition"), "system");
+ assertTenant(byItemId.get("updateProperties"), "system");
+ assertTenant(byItemId.get("pastEventCondition_conditiontype"),
"system");
+ assertTenant(byItemId.get("my-user-rule"), TEST_TENANT_ID);
+ assertTenant(byItemId.get("my-segment"), TEST_TENANT_ID);
+ assertTenant(byItemId.get("my-scoring"), TEST_TENANT_ID);
+ assertTenant(byItemId.get("my-goal"), TEST_TENANT_ID);
+ assertTenant(byItemId.get("cfg1"), "system");
+
+ // System item types get the _itemtype suffix exactly once, other
types none
+ Assert.assertEquals("system_booleanCondition_conditiontype",
byItemId.get("booleanCondition").path("_id").asText());
+ Assert.assertEquals("system_pastEventCondition_conditiontype",
+
byItemId.get("pastEventCondition_conditiontype").path("_id").asText());
+ Assert.assertEquals(TEST_TENANT_ID + "_my-goal_goal",
byItemId.get("my-goal").path("_id").asText());
+ Assert.assertEquals("system_cfg1",
byItemId.get("cfg1").path("_id").asText());
+
+ // Migrated system items carry create audit, so the bundle
redeploy is an update
+ JsonNode source = byItemId.get("booleanCondition").path("_source");
+ Assert.assertEquals("system-migration-4.0.0",
source.path("createdBy").asText());
+ Assert.assertEquals("system-migration-4.0.0",
source.path("lastModifiedBy").asText());
+ Assert.assertEquals("2026-01-01T00:00:00Z",
source.path("creationDate").asText());
+ }
+ }
+
+ @Test
+ public void forcedDefaultTenantNeverUsesTheSystemTenant() throws Exception
{
+ assumeHttpAdminApi();
+
+ try (CloseableHttpClient httpClient = createSearchEngineHttpClient()) {
+ String es = getSearchEngineBaseUrl();
+ HttpUtils.executePutRequest(httpClient, es + "/" + SOURCE_INDEX,
MAPPING, null);
+ indexDocument(httpClient, SOURCE_INDEX, "node1", "node1",
"clusterNode", "system");
+ indexDocument(httpClient, SOURCE_INDEX, "booleanCondition",
"booleanCondition", "conditionType", null);
+
+ Map<String, JsonNode> byItemId = migrate(httpClient, SOURCE_INDEX,
DEST_INDEX, "clusterNode", true,
+ Collections.emptySet(),
MigrationUtils.loadShippedDefinitionIds(migrationBundle()));
+ Assert.assertEquals(2, byItemId.size());
+ assertTenant(byItemId.get("node1"), TEST_TENANT_ID);
+ assertTenant(byItemId.get("booleanCondition"), TEST_TENANT_ID);
+ }
+ }
+
+ @Test
+ public void profilesSessionsAndEventsStayInTheConfiguredTenant() throws
Exception {
+ assumeHttpAdminApi();
+
+ try (CloseableHttpClient httpClient = createSearchEngineHttpClient()) {
+ String es = getSearchEngineBaseUrl();
+ HttpUtils.executePutRequest(httpClient, es + "/" + DATA_SOURCE,
MAPPING, null);
+ indexDocument(httpClient, DATA_SOURCE, "p1", "p1", "profile",
null);
+ indexDocument(httpClient, DATA_SOURCE, "s1", "s1", "session",
"systemscope");
+ indexDocument(httpClient, DATA_SOURCE, "e1", "e1", "event",
"systemscope");
+
+ Map<String, JsonNode> byItemId = migrate(httpClient, DATA_SOURCE,
DATA_DEST, "profile", false,
+ Set.of("conditiontype", "rule"),
MigrationUtils.loadShippedDefinitionIds(migrationBundle()));
+ Assert.assertEquals(3, byItemId.size());
+ for (String itemId : new String[]{"p1", "s1", "e1"}) {
+ assertTenant(byItemId.get(itemId), TEST_TENANT_ID);
+ Assert.assertEquals(TEST_TENANT_ID + "_" + itemId,
byItemId.get(itemId).path("_id").asText());
+ }
+ }
+ }
+
+ @Test
+ public void geonamesGoToTheSystemTenant() throws Exception {
+ assumeHttpAdminApi();
+
+ try (CloseableHttpClient httpClient = createSearchEngineHttpClient()) {
+ String es = getSearchEngineBaseUrl();
+ HttpUtils.executePutRequest(httpClient, es + "/" + GEO_SOURCE,
MAPPING, null);
+ indexDocument(httpClient, GEO_SOURCE, "paris", "paris",
"geonameEntry", null);
+
+ Map<String, JsonNode> byItemId = migrate(httpClient, GEO_SOURCE,
GEO_DEST, "geonameentry", false,
+ Collections.emptySet(), Collections.emptyMap());
+ assertTenant(byItemId.get("paris"), "system");
+ }
+ }
+
+ @Test
+ public void
itemIdFixMatchesCamelCaseItemTypesAndCountsOnlyChangedDocuments() throws
Exception {
+ assumeHttpAdminApi();
+
+ try (CloseableHttpClient httpClient = createSearchEngineHttpClient()) {
+ String es = getSearchEngineBaseUrl();
+ String mapping = "{"
+ + "\"settings\":{"
+ + "\"number_of_shards\":1,\"number_of_replicas\":0,"
+ +
"\"analysis\":{\"analyzer\":{\"folding\":{\"type\":\"custom\",\"tokenizer\":\"keyword\",\"filter\":[\"lowercase\",\"asciifolding\"]}}}"
+ + "},"
+ + "\"mappings\":{\"properties\":{"
+ + "\"itemId\":{\"type\":\"keyword\"},"
+ +
"\"itemType\":{\"type\":\"text\",\"analyzer\":\"folding\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}}"
+ + "}}}";
+ HttpUtils.executePutRequest(httpClient, es + "/" +
CAMEL_CASE_INDEX, mapping, null);
+ indexDocument(httpClient, CAMEL_CASE_INDEX,
"system_booleanCondition_conditiontype", "wrong", "conditionType", null);
+ // Already consistent: must be left untouched and not counted
+ indexDocument(httpClient, CAMEL_CASE_INDEX,
"system_pastEventCondition_conditiontype",
+ "pastEventCondition_conditiontype", "conditionType", null);
+
+ // Same request as migrate-4.0.0-05-fixSystemItemIds.groovy
+ String fixScript = MigrationUtils.getFileWithoutComments(
+ migrationBundle(),
"requestBody/4.0.0/fix_system_item_ids.painless");
+ long updated = MigrationUtils.updateByQueryAndCount(httpClient,
es, CAMEL_CASE_INDEX,
+ MigrationUtils.buildItemTypeUpdateRequest(fixScript,
"conditionType"));
+ Assert.assertEquals(1L, updated);
+
+ JsonNode source =
MAPPER.readTree(HttpUtils.executeGetRequest(httpClient,
+ es + "/" + CAMEL_CASE_INDEX +
"/_doc/system_booleanCondition_conditiontype", null))
+ .path("_source");
+ Assert.assertEquals("booleanCondition_conditiontype",
source.path("itemId").asText());
+ }
+ }
+
+ private void assumeHttpAdminApi() {
+ Assume.assumeTrue(
+ "HTTP admin API required (provider=" +
getPersistenceBackend().providerId() + ")",
+ persistenceCapabilities().httpAdminApi());
+ }
+
+ private static BundleContext migrationBundle() {
+ return
FrameworkUtil.getBundle(MigrationUtils.class).getBundleContext();
+ }
+
+ private void indexDocument(CloseableHttpClient httpClient, String index,
String documentId, String itemId,
+ String itemType, String scope) throws Exception
{
+ String source = "{\"itemId\":\"" + itemId + "\",\"itemType\":\"" +
itemType + "\""
+ + (scope != null ? ",\"scope\":\"" + scope + "\"" : "") + "}";
+ HttpUtils.executePutRequest(httpClient,
+ getSearchEngineBaseUrl() + "/" + index + "/_doc/" + documentId
+ "?refresh=true", source, null);
+ }
+
+ /**
+ * Runs the tenant assignment script from {@code source} to {@code dest},
as migrate-4.0.0-01 does,
+ * and returns the migrated hits by source itemId.
+ */
+ private Map<String, JsonNode> migrate(CloseableHttpClient httpClient,
String source, String dest, String itemType,
+ boolean forceDefaultTenant,
Set<String> systemItems,
+ Map<String, Set<String>>
shippedDefinitionIds) throws Exception {
+ String es = getSearchEngineBaseUrl();
+ MigrationUtils.createIndex(httpClient, es, dest, MAPPING);
+
+ Map<String, Object> params = new HashMap<>();
+ params.put("tenantId", TEST_TENANT_ID);
+ params.put("systemTenantId", "system");
+ params.put("forceDefaultTenant", forceDefaultTenant);
+ params.put("itemType", itemType);
+ params.put("date", "2026-01-01T00:00:00Z");
+ params.put("systemItems", systemItems);
+ params.put("shippedDefinitionIds", shippedDefinitionIds);
+
+ String painless = MigrationUtils.getFileWithoutComments(
+ migrationBundle(),
"requestBody/4.0.0/initialize_tenant_and_audit_fields.painless");
+ MigrationUtils.moveToIndex(httpClient, migrationBundle(), es, source,
dest, painless, params);
+ HttpUtils.executePostRequest(httpClient, es + "/" + dest +
"/_refresh", null, null);
+
+ Map<String, JsonNode> byItemId = new HashMap<>();
+ for (JsonNode hit :
MAPPER.readTree(HttpUtils.executeGetRequest(httpClient,
+ es + "/" + dest + "/_search?size=20",
null)).path("hits").path("hits")) {
+ byItemId.put(hit.path("_source").path("itemId").asText(), hit);
+ }
+ return byItemId;
+ }
+
+ private static void assertTenant(JsonNode hit, String expectedTenant) {
+ Assert.assertNotNull(hit);
+ Assert.assertTrue(hit.path("_id").asText().startsWith(expectedTenant +
"_"));
+ Assert.assertEquals(expectedTenant,
hit.path("_source").path("tenantId").asText());
+ }
+}
diff --git a/rest/pom.xml b/rest/pom.xml
index 2deb35fd4..625556b2f 100644
--- a/rest/pom.xml
+++ b/rest/pom.xml
@@ -182,12 +182,6 @@
<version>${cxf.version}</version>
<scope>provided</scope>
</dependency>
- <dependency>
- <groupId>org.apache.cxf</groupId>
- <artifactId>cxf-rt-frontend-jaxrs</artifactId>
- <version>${cxf.version}</version>
- <scope>provided</scope>
- </dependency>
<dependency>
<groupId>org.slf4j</groupId>
diff --git
a/services-common/src/main/java/org/apache/unomi/services/common/cache/AbstractMultiTypeCachingService.java
b/services-common/src/main/java/org/apache/unomi/services/common/cache/AbstractMultiTypeCachingService.java
index f1a7ea87c..8cf444157 100644
---
a/services-common/src/main/java/org/apache/unomi/services/common/cache/AbstractMultiTypeCachingService.java
+++
b/services-common/src/main/java/org/apache/unomi/services/common/cache/AbstractMultiTypeCachingService.java
@@ -932,22 +932,25 @@ public abstract class AbstractMultiTypeCachingService
extends AbstractContextAwa
Class<T> itemClass = (Class<T>) item.getClass();
T existingItem = persistenceService.load(itemId, itemClass);
- boolean itemExists = false;
- if (existingItem != null) {
- // Item exists in persistence, check if it has audit metadata
- itemExists = existingItem.getCreatedBy() != null &&
existingItem.getCreationDate() != null;
- } else {
- // Item doesn't exist in persistence, check if current item has
audit metadata (might be a reload from cache)
- itemExists = item.getCreatedBy() != null && item.getCreationDate()
!= null;
+ // Bundle JSON definitions have no creator, and Item() defaults their
creation date and
+ // version to "now" and 0. If persistence already has create audit
(e.g. from migration),
+ // carry it over to the incoming item so the save below does not
overwrite it (UNOMI-997).
+ if (existingItem != null && item.getCreatedBy() == null
+ && existingItem.getCreatedBy() != null &&
existingItem.getCreationDate() != null) {
+ item.setCreatedBy(existingItem.getCreatedBy());
+ item.setCreationDate(existingItem.getCreationDate());
+ if (existingItem.getVersion() != null) {
+ item.setVersion(existingItem.getVersion());
+ }
}
+ boolean hasCreateAudit = item.getCreatedBy() != null &&
item.getCreationDate() != null;
+
// Set audit metadata for bundle-deployed items
if (auditService != null) {
- if (itemExists) {
- // Item exists, this is an update
+ if (hasCreateAudit) {
auditService.auditUpdate(item, "system-bundle");
} else {
- // New item, this is a create
auditService.auditCreate(item, "system-bundle");
}
}
diff --git
a/services-common/src/test/java/org/apache/unomi/services/common/cache/AbstractMultiTypeCachingServiceTest.java
b/services-common/src/test/java/org/apache/unomi/services/common/cache/AbstractMultiTypeCachingServiceTest.java
index 57923a771..ad962c909 100644
---
a/services-common/src/test/java/org/apache/unomi/services/common/cache/AbstractMultiTypeCachingServiceTest.java
+++
b/services-common/src/test/java/org/apache/unomi/services/common/cache/AbstractMultiTypeCachingServiceTest.java
@@ -21,6 +21,7 @@ import org.apache.unomi.api.Item;
import org.apache.unomi.api.services.ExecutionContextManager;
import org.apache.unomi.api.services.cache.CacheableTypeConfig;
import org.apache.unomi.api.services.cache.MultiTypeCacheService;
+import org.apache.unomi.api.tenants.AuditService;
import org.apache.unomi.api.tenants.Tenant;
import org.apache.unomi.api.tenants.TenantService;
import org.apache.unomi.persistence.spi.PersistenceService;
@@ -62,6 +63,9 @@ public class AbstractMultiTypeCachingServiceTest {
@Mock
private TenantService tenantService;
+ @Mock
+ private AuditService auditService;
+
private TestCachingServiceImpl testCachingService;
// Simple test class that implements Serializable
@@ -194,6 +198,7 @@ public class AbstractMultiTypeCachingServiceTest {
testCachingService.setContextManager(contextManager);
testCachingService.setCacheService(cacheService);
testCachingService.setTenantService(tenantService);
+ testCachingService.setAuditService(auditService);
testCachingService.makeConfigPersistable();
// Mock tenant service to return tenant list
@@ -467,4 +472,49 @@ public class AbstractMultiTypeCachingServiceTest {
assertFalse("saveItem() must not proceed until the concurrent refresh
released its write lock",
refreshStillRunningWhenSaveReturned.get());
}
+
+ /**
+ * Bundle redeploy loads JSON without a creator, with the creation date
and version defaulted by
+ * {@code Item()}. If persistence already has create audit (for example
after migration),
+ * saveItem must keep it instead of overwriting it with those defaults.
+ */
+ @Test
+ public void testSaveItemPreservesMigratedCreateAuditOnBundleRedeploy() {
+ Date creationDate = new Date(1_700_000_000_000L);
+ TestItem existing = new TestItem("item1");
+ existing.setCreatedBy("system-migration-4.0.0");
+ existing.setCreationDate(creationDate);
+ existing.setVersion(3L);
+ when(persistenceService.load("item1",
TestItem.class)).thenReturn(existing);
+
+ TestItem fromBundle = new TestItem("item1");
+ assertNull(fromBundle.getCreatedBy());
+ assertNotNull("Item() defaults the creation date",
fromBundle.getCreationDate());
+
+ testCachingService.callSaveItem(fromBundle);
+
+ ArgumentCaptor<TestItem> saved =
ArgumentCaptor.forClass(TestItem.class);
+ verify(persistenceService).save(saved.capture());
+ assertEquals("system-migration-4.0.0",
saved.getValue().getCreatedBy());
+ assertEquals(creationDate, saved.getValue().getCreationDate());
+ assertEquals(Long.valueOf(3L), saved.getValue().getVersion());
+ verify(auditService).auditUpdate(same(fromBundle),
eq("system-bundle"));
+ verify(auditService, never()).auditCreate(any(), anyString());
+ }
+
+ /**
+ * A persisted item without complete create audit is still audited as a
creation.
+ */
+ @Test
+ public void testSaveItemAuditsCreateWhenExistingItemHasNoCreateAudit() {
+ TestItem existing = new TestItem("item1");
+ existing.setCreatedBy(null);
+ when(persistenceService.load("item1",
TestItem.class)).thenReturn(existing);
+
+ TestItem fromBundle = new TestItem("item1");
+ testCachingService.callSaveItem(fromBundle);
+
+ verify(auditService).auditCreate(same(fromBundle),
eq("system-bundle"));
+ verify(auditService, never()).auditUpdate(any(), anyString());
+ }
}
diff --git
a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
index 4e8d71657..5ab1a8bc0 100644
---
a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
+++
b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java
@@ -365,6 +365,66 @@ public class MigrationUtils {
return name;
}
+ /**
+ * Loads the definitions Unomi ships and reloads at startup, as lower-case
item ids grouped by
+ * lower-case item type (for example {@code conditiontype ->
booleancondition}).
+ * The migrate command bundle ships a snapshot of that list so it is
available before plugins start.
+ * <p>
+ * Matching is by type and id only: a 3.0 item that shares both with a
shipped definition (such as a
+ * built-in rule edited in place) is migrated to the system tenant and
replaced by the 4.0 version
+ * at the next startup.
+ *
+ * @throws IllegalStateException if the list is empty or an entry is not
{@code itemType:itemId}
+ */
+ public static Map<String, Set<String>>
loadShippedDefinitionIds(BundleContext bundleContext) {
+ String resource = "requestBody/4.0.0/shipped_system_definitions.json";
+ JSONArray array = new JSONArray(resourceAsString(bundleContext,
resource));
+ Map<String, Set<String>> idsByType = new TreeMap<>();
+ for (int i = 0; i < array.length(); i++) {
+ String key = String.valueOf(array.get(i)).toLowerCase(Locale.ROOT);
+ int separator = key.indexOf(':');
+ if (separator <= 0 || separator == key.length() - 1) {
+ throw new IllegalStateException("Invalid entry in " + resource
+ ": " + key);
+ }
+ idsByType.computeIfAbsent(key.substring(0, separator), type -> new
TreeSet<>()).add(key.substring(separator + 1));
+ }
+ if (idsByType.isEmpty()) {
+ throw new IllegalStateException(resource + " is empty: shipped
definitions would be migrated to the wrong tenant");
+ }
+ return idsByType;
+ }
+
+ /**
+ * Builds an update by query request running a Painless script on the
documents of an item type.
+ * The item type is matched on both the keyword sub-field and the folded
text field, so camelCase
+ * values such as {@code conditionType} are not missed.
+ */
+ public static String buildItemTypeUpdateRequest(String painlessScript,
String itemType) {
+ JSONArray should = new JSONArray()
+ .put(new JSONObject().put("term", new
JSONObject().put("itemType.keyword", itemType)))
+ .put(new JSONObject().put("term", new
JSONObject().put("itemType", itemType.toLowerCase(Locale.ROOT))));
+ return new JSONObject()
+ .put("script", new JSONObject().put("source",
painlessScript).put("lang", "painless"))
+ .put("query", new JSONObject().put("bool", new
JSONObject().put("should", should).put("minimum_should_match", 1)))
+ .toString();
+ }
+
+ /**
+ * Runs an update by query synchronously, refreshes the index and returns
how many documents were
+ * actually changed (documents the script marks as noop are not counted).
+ *
+ * @throws IOException if the update timed out or reported failures;
version conflicts abort the request
+ */
+ public static long updateByQueryAndCount(CloseableHttpClient httpClient,
String esAddress, String indexName, String requestBody) throws IOException {
+ JSONObject response = new JSONObject(HttpUtils.executePostRequest(
+ httpClient, esAddress + "/" + indexName +
"/_update_by_query?refresh=true", requestBody, null));
+ JSONArray failures = response.optJSONArray("failures");
+ if (response.optBoolean("timed_out") || (failures != null &&
!failures.isEmpty())) {
+ throw new IOException("Update by query on " + indexName + " did
not complete: " + response);
+ }
+ return response.optLong("updated", 0L);
+ }
+
public static String buildIndexCreationRequest(String baseIndexSettings,
String mapping, MigrationContext context, boolean isMonthlyIndex) throws
IOException {
String settings = baseIndexSettings
.replace("#numberOfShards",
context.getConfigString(isMonthlyIndex ? MONTHLY_NUMBER_OF_SHARDS :
NUMBER_OF_SHARDS))
diff --git
a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-4.0.0-01-tenantDocumentIds.groovy
b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-4.0.0-01-tenantDocumentIds.groovy
index f5f265e3d..6f3c755b4 100644
---
a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-4.0.0-01-tenantDocumentIds.groovy
+++
b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-4.0.0-01-tenantDocumentIds.groovy
@@ -26,7 +26,7 @@ MigrationContext context = migrationContext
String esAddress = context.getConfigString(CONFIG_ES_ADDRESS)
String indexPrefix = context.getConfigString(INDEX_PREFIX)
String tenantId = context.getConfigString(TENANT_ID)
-String systemTenantId = "system" // System tenant ID for system-level items
+String systemTenantId = "system"
String rolloverPolicyName = indexPrefix + "-unomi-rollover-policy"
String rolloverSessionAlias = indexPrefix + "-session"
String rolloverEventAlias = indexPrefix + "-event"
@@ -104,16 +104,21 @@ context.performMigrationStep("4.0.0-get-all-indices", ()
-> {
Set<String> allItemTypes =
MigrationUtils.getAllItemTypes(context.getHttpClient(), esAddress, indexPrefix,
"*", bundleContext)
context.printMessage("Found " + allItemTypes.size() + " item types")
- // Get all system items from the systemitems index
+ // Get all system items from the systemitems index (store lower-case for
painless checks)
Set<String> systemItems =
MigrationUtils.getAllItemTypes(context.getHttpClient(), esAddress, indexPrefix,
"systemitems", bundleContext)
+ .collect { it.toLowerCase(java.util.Locale.ROOT) } as Set
context.printMessage("Found " + systemItems.size() + " system items")
+ Map<String, Set<String>> shippedDefinitionIds =
MigrationUtils.loadShippedDefinitionIds(bundleContext)
+ context.printMessage("Loaded " + shippedDefinitionIds.values().sum {
it.size() } + " shipped definition ids")
+
// Create base parameters
Map<String, Object> baseParams = new HashMap<>()
baseParams.put("date", isoDate)
baseParams.put("tenantId", tenantId)
baseParams.put("systemTenantId", systemTenantId)
baseParams.put("systemItems", systemItems)
+ baseParams.put("shippedDefinitionIds", shippedDefinitionIds)
context.printMessage("Using tenant ID: " + tenantId)
diff --git
a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-4.0.0-05-fixSystemItemIds.groovy
b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-4.0.0-05-fixSystemItemIds.groovy
index c92606ddc..6a5065fa1 100644
---
a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-4.0.0-05-fixSystemItemIds.groovy
+++
b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-4.0.0-05-fixSystemItemIds.groovy
@@ -1,7 +1,6 @@
import org.apache.unomi.shell.migration.service.MigrationContext
import org.apache.unomi.shell.migration.utils.MigrationUtils
import org.apache.unomi.shell.migration.utils.HttpUtils
-import org.json.JSONObject
import static org.apache.unomi.shell.migration.service.MigrationConfig.*
/*
@@ -25,7 +24,7 @@ MigrationContext context = migrationContext
String esAddress = context.getConfigString(CONFIG_ES_ADDRESS)
String indexPrefix = context.getConfigString(INDEX_PREFIX)
-// Get all system item types
+// Get all system item types (keep the casing returned by the index for
keyword queries)
Set<String> systemItems =
MigrationUtils.getAllItemTypes(context.getHttpClient(), esAddress, indexPrefix,
"systemitems", bundleContext)
context.printMessage("Found " + systemItems.size() + " system item types")
@@ -38,48 +37,39 @@ context.printMessage("Found " + systemItems.size() + "
system item types")
// incorrectly processed the baseId, we need to fix the itemId in the source
to match the document ID.
context.performMigrationStep("4.0.0-fix-system-item-ids", () -> {
String systemItemsIndex = "${indexPrefix}-systemitems"
-
+
if (MigrationUtils.indexExists(context.getHttpClient(), esAddress,
systemItemsIndex)) {
context.printMessage("Fixing itemIds in systemitems index that end
with itemType suffix")
-
- // Process each system item type
+
+ long totalUpdated = 0L
+ List<String> failedItemTypes = []
+
+ // Process each system item type. The query matches camelCase values
such as
+ // conditionType / actionType / propertyType (UNOMI-997).
systemItems.each { itemType ->
context.printMessage("Fixing items of type: ${itemType}")
-
- // Get the Painless script from file
+
String fixScript =
MigrationUtils.getFileWithoutComments(bundleContext,
"requestBody/4.0.0/fix_system_item_ids.painless")
-
- // Build the update request using JSONObject to properly escape
the script
- // This is the same approach used in
MigrationUtils.getScriptPart() and other migrations
- JSONObject scriptObj = new JSONObject()
- scriptObj.put("source", fixScript)
- scriptObj.put("lang", "painless")
-
- JSONObject queryObj = new JSONObject()
- JSONObject termObj = new JSONObject()
- termObj.put("itemType", itemType)
- queryObj.put("term", termObj)
-
- JSONObject updateRequestObj = new JSONObject()
- updateRequestObj.put("script", scriptObj)
- updateRequestObj.put("query", queryObj)
-
- String updateRequest = updateRequestObj.toString()
-
+
+ String updateRequest =
MigrationUtils.buildItemTypeUpdateRequest(fixScript, itemType)
+
try {
- MigrationUtils.updateByQuery(context.getHttpClient(),
esAddress, systemItemsIndex, updateRequest)
- context.printMessage("Fixed itemIds for item type:
${itemType}")
+ long updated =
MigrationUtils.updateByQueryAndCount(context.getHttpClient(), esAddress,
systemItemsIndex, updateRequest)
+ totalUpdated += updated
+ context.printMessage("Fixed ${updated} itemId(s) for item
type: ${itemType}")
} catch (Exception e) {
context.printMessage("Warning: Could not fix itemIds for item
type ${itemType}: ${e.getMessage()}")
- // Continue with other item types even if one fails
+ failedItemTypes.add(itemType)
}
}
-
- // Refresh the index to make changes visible
+
HttpUtils.executePostRequest(context.getHttpClient(), esAddress +
"/${systemItemsIndex}/_refresh", null, null)
- context.printMessage("Fixed itemIds in systemitems index")
+ context.printMessage("Fixed ${totalUpdated} itemId(s) in systemitems
index")
+ // Fail the step so it is retried on the next run instead of being
recorded as completed
+ if (!failedItemTypes.isEmpty()) {
+ throw new IllegalStateException("Could not fix itemIds for item
types: " + failedItemTypes)
+ }
} else {
context.printMessage("Systemitems index does not exist, skipping
itemId fix")
}
})
-
diff --git
a/tools/shell-commands/src/main/resources/requestBody/4.0.0/fix_system_item_ids.painless
b/tools/shell-commands/src/main/resources/requestBody/4.0.0/fix_system_item_ids.painless
index 9d309d1cc..fdab21556 100644
---
a/tools/shell-commands/src/main/resources/requestBody/4.0.0/fix_system_item_ids.painless
+++
b/tools/shell-commands/src/main/resources/requestBody/4.0.0/fix_system_item_ids.painless
@@ -38,34 +38,15 @@ if (ctx._source.itemId != null && ctx._source.itemType !=
null && ctx._id != nul
documentIdWithoutTenant =
documentIdWithoutTenant.substring(firstUnderscoreIndex + 1);
}
- // For system items, the expected format after 2.2.0 migration is itemId =
documentId (with suffix)
- // However, new items created after migrations may have itemId = original
(without suffix)
- // Both work because setMetadata() extracts from document ID, but we
normalize to the migrated format
- // for consistency and to fix items affected by the 4.0.0-00 bug
- def itemTypeSuffix = '_' + ctx._source.itemType.toLowerCase();
- if (documentIdWithoutTenant.endsWith(itemTypeSuffix)) {
- // Document ID has the expected suffix format - itemId should match it
- if (ctx._source.itemId != documentIdWithoutTenant) {
- ctx._source.itemId = documentIdWithoutTenant;
- }
+ // Every case above ends the same way: itemId must equal the document ID
minus the tenant prefix.
+ // When the document ID lost its suffix (4.0.0-00 bug) it cannot be
repaired by update_by_query
+ // (that needs a reindex), but aligning itemId on it still lets
setMetadata() work.
+ // Documents that are already consistent are left untouched so they are
not counted as updated.
+ if (ctx._source.itemId != documentIdWithoutTenant) {
+ ctx._source.itemId = documentIdWithoutTenant;
} else {
- // Document ID doesn't have suffix (pre-2.2.0 format or incorrectly
processed by 4.0.0-00)
- // Check if source itemId has the suffix - this indicates the document
ID is wrong (from buggy 4.0.0-00)
- if (ctx._source.itemId != null &&
ctx._source.itemId.endsWith(itemTypeSuffix)) {
- // Source itemId has suffix but document ID doesn't - document ID
was incorrectly processed
- // We can't fix the document ID with update_by_query (would need
reindexing),
- // but we can at least ensure itemId matches the document ID so
setMetadata() can work
- // Note: This item will need to be reindexed to fix the document
ID properly
- if (ctx._source.itemId != documentIdWithoutTenant) {
- ctx._source.itemId = documentIdWithoutTenant;
- }
- } else {
- // Document ID doesn't have suffix and source itemId doesn't either
- // This is either pre-2.2.0 format or a legitimate case - ensure
they match
- if (ctx._source.itemId != documentIdWithoutTenant) {
- ctx._source.itemId = documentIdWithoutTenant;
- }
- }
+ ctx.op = 'noop';
}
+} else {
+ ctx.op = 'noop';
}
-
diff --git
a/tools/shell-commands/src/main/resources/requestBody/4.0.0/initialize_tenant_and_audit_fields.painless
b/tools/shell-commands/src/main/resources/requestBody/4.0.0/initialize_tenant_and_audit_fields.painless
index 7b089252d..97dcd9b38 100644
---
a/tools/shell-commands/src/main/resources/requestBody/4.0.0/initialize_tenant_and_audit_fields.painless
+++
b/tools/shell-commands/src/main/resources/requestBody/4.0.0/initialize_tenant_and_audit_fields.painless
@@ -14,37 +14,63 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
+// Decide the tenant for this document (UNOMI-997).
+// Shipped definitions and geonames go to system. User-created
rules/segments/scorings/goals
+// stay in the configured tenant even when their 3.0 scope is "systemscope".
+// A shipped definition is recognised by item type and id only, so a 3.0 item
sharing both
+// (such as a built-in rule edited in place) also goes to system.
+boolean useSystemTenant = false;
+if (params.forceDefaultTenant != true) {
+ def itemTypeLower = ctx._source.itemType != null ?
ctx._source.itemType.toLowerCase(Locale.ROOT) : '';
+ // Shipped ids are grouped by item type, so other types (profiles,
events...) skip the lookup
+ def shippedIds = params.shippedDefinitionIds != null ?
params.shippedDefinitionIds.get(itemTypeLower) : null;
+ if (itemTypeLower == 'geonameentry' || (ctx._index != null &&
ctx._index.toLowerCase(Locale.ROOT).endsWith('-geonameentry'))) {
+ useSystemTenant = true;
+ } else if (shippedIds != null) {
+ def rawId = ctx._source.itemId != null ? ctx._source.itemId : ctx._id;
+ if (rawId != null) {
+ def shippedId = rawId.toLowerCase(Locale.ROOT);
+ if (shippedId.endsWith('_' + itemTypeLower)) {
+ shippedId = shippedId.substring(0, shippedId.length() -
itemTypeLower.length() - 1);
+ }
+ useSystemTenant = shippedIds.contains(shippedId);
+ }
+ }
+ // Literal "system" only — not the 3.0 default "systemscope"
+ if (!useSystemTenant && ctx._source.scope == 'system') {
+ useSystemTenant = true;
+ }
+}
+def assignedTenantId = useSystemTenant ? params.systemTenantId :
params.tenantId;
+
// Update document ID with tenant prefix
if (ctx._id != null) {
// Skip if already has tenant prefix
if (!ctx._id.startsWith(params.tenantId + '_') &&
!ctx._id.startsWith(params.systemTenantId + '_')) {
- // Cluster nodes and unknown indices always stay in the configured
(default) tenant
- if (params.forceDefaultTenant == true) {
- ctx._id = params.tenantId + '_' + ctx._id;
- } else if (ctx._index.endsWith('-geonames') || (ctx._source.scope !=
null && ctx._source.scope == 'system')) {
- // Preserve the entire original document ID (it may contain
underscores, e.g., "dummy_scope")
- // Do NOT split on underscores - this was a bug in the original
migration that caused itemIds
- // like "dummy_scope" to become "dummy" when the document ID was
"dummy_scope_scope"
- def baseId = ctx._id;
- if (ctx._index.endsWith('-systemitems')) {
- // For system items, ensure itemType is lowercase and matches
the format in ElasticSearchPersistenceServiceImpl
- def itemType = ctx._source.itemType != null ?
ctx._source.itemType.toLowerCase() : null;
- if (itemType != null && params.systemItems.contains(itemType))
{
- ctx._id = ctx._source.scope == 'system' ?
params.systemTenantId + '_' + baseId + '_' + itemType : params.tenantId + '_' +
baseId + '_' + itemType;
- } else {
- ctx._id = ctx._source.scope == 'system' ?
params.systemTenantId + '_' + baseId : params.tenantId + '_' + baseId;
- }
+ def baseId = ctx._id;
+ if (ctx._index != null &&
ctx._index.toLowerCase(Locale.ROOT).endsWith('-systemitems')) {
+ // For system items the document ID is tenantId_itemId_itemtype
(see getDocumentIDForItemType).
+ // 3.0 document IDs already carry the lower-case _itemtype suffix,
so only add it when missing.
+ def itemType = ctx._source.itemType != null ?
ctx._source.itemType.toLowerCase(Locale.ROOT) : null;
+ if (itemType != null && params.systemItems != null &&
params.systemItems.contains(itemType)
+ && !baseId.toLowerCase(Locale.ROOT).endsWith('_' +
itemType)) {
+ ctx._id = assignedTenantId + '_' + baseId + '_' + itemType;
} else {
- ctx._id = params.systemTenantId + '_' + baseId;
+ ctx._id = assignedTenantId + '_' + baseId;
}
} else {
- ctx._id = params.tenantId + '_' + ctx._id;
+ ctx._id = assignedTenantId + '_' + baseId;
}
}
}
-// Update audit fields
-if (!ctx._index.endsWith('-systemitems') && !ctx._index.endsWith('-geonames'))
{
+// Update audit fields (skip geonames — not asserted and high volume).
+// systemitems must get audit here: UNOMI-997 puts shipped defs under the
system
+// tenant before Unomi starts, so bundle deploy treats them as updates and
would
+// otherwise leave createdBy unset on docs that never had it.
+def indexLower = ctx._index != null ? ctx._index.toLowerCase(Locale.ROOT) : '';
+if (!indexLower.endsWith('-geonameentry') &&
!indexLower.endsWith('-geonames')) {
// Handle creation date based on item type
if (ctx._source.creationDate == null) {
if (params.itemType == 'profile' && ctx._source.properties != null) {
@@ -59,7 +85,7 @@ if (!ctx._index.endsWith('-systemitems') &&
!ctx._index.endsWith('-geonames')) {
ctx._source.creationDate = params.date;
}
}
-
+
// Handle last modification date based on item type
if (ctx._source.lastModificationDate == null) {
if (params.itemType == 'profile' && ctx._source.properties != null) {
@@ -74,7 +100,7 @@ if (!ctx._index.endsWith('-systemitems') &&
!ctx._index.endsWith('-geonames')) {
ctx._source.lastModificationDate = ctx._source.creationDate;
}
}
-
+
// Set creator fields
if (ctx._source.createdBy == null) {
ctx._source.createdBy = 'system-migration-4.0.0';
@@ -82,7 +108,7 @@ if (!ctx._index.endsWith('-systemitems') &&
!ctx._index.endsWith('-geonames')) {
if (ctx._source.lastModifiedBy == null) {
ctx._source.lastModifiedBy = 'system-migration-4.0.0';
}
-
+
// Initialize source tracking fields
if (ctx._source.sourceInstanceId == null) {
ctx._source.sourceInstanceId = null;
@@ -92,13 +118,7 @@ if (!ctx._index.endsWith('-systemitems') &&
!ctx._index.endsWith('-geonames')) {
}
}
-// Set tenant ID in the source document based on scope for ALL items
+// Set tenant ID in the source document
if (ctx._source.tenantId == null) {
- if (params.forceDefaultTenant == true) {
- ctx._source.tenantId = params.tenantId;
- } else if (ctx._index.endsWith('-geonames') || (ctx._source.scope != null
&& ctx._source.scope == 'system')) {
- ctx._source.tenantId = params.systemTenantId;
- } else {
- ctx._source.tenantId = params.tenantId;
- }
-}
\ No newline at end of file
+ ctx._source.tenantId = assignedTenantId;
+}
diff --git
a/tools/shell-commands/src/main/resources/requestBody/4.0.0/shipped_system_definitions.json
b/tools/shell-commands/src/main/resources/requestBody/4.0.0/shipped_system_definitions.json
new file mode 100644
index 000000000..3d8cd07ad
--- /dev/null
+++
b/tools/shell-commands/src/main/resources/requestBody/4.0.0/shipped_system_definitions.json
@@ -0,0 +1,127 @@
+[
+ "actiontype:addtolistsaction",
+ "actiontype:alleventtoprofilepropertiesaction",
+ "actiontype:cdpsessioneventaction",
+ "actiontype:copypropertiesaction",
+ "actiontype:evaluateprofileageaction",
+ "actiontype:evaluateprofilesegmentsaction",
+ "actiontype:evaluatevisitpropertiesaction",
+ "actiontype:eventtoprofilepropertyaction",
+ "actiontype:incrementpropertyaction",
+ "actiontype:incrementtweetnumberaction",
+ "actiontype:mergeprofilesonpropertyaction",
+ "actiontype:modifyconsentaction",
+ "actiontype:requestheadertoprofilepropertyaction",
+ "actiontype:requestparametertoprofilepropertyaction",
+ "actiontype:sendeventaction",
+ "actiontype:sendmailaction",
+ "actiontype:seteventoccurencecountaction",
+ "actiontype:setpropertyaction",
+ "actiontype:setremotehostinfoaction",
+ "actiontype:sfdccreateorupdateleadaction",
+ "actiontype:sfdcupdateprofilefromleadaction",
+ "actiontype:testgroovyaction",
+ "actiontype:trainingnotifaction",
+ "actiontype:updateconsenteventaction",
+ "actiontype:updatelistseventaction",
+ "actiontype:updatepropertiesaction",
+ "actiontype:weatherupdateaction",
+ "conditiontype:booleancondition",
+ "conditiontype:cdpsessioneventcondition",
+ "conditiontype:eventpropertycondition",
+ "conditiontype:eventtypecondition",
+ "conditiontype:formeventcondition",
+ "conditiontype:geolocationbypointsessioncondition",
+ "conditiontype:geolocationsessioncondition",
+ "conditiontype:goalmatchcondition",
+ "conditiontype:hovereventcondition",
+ "conditiontype:idscondition",
+ "conditiontype:matchallcondition",
+ "conditiontype:modifyanyconsenteventcondition",
+ "conditiontype:modifyconsenteventcondition",
+ "conditiontype:nestedcondition",
+ "conditiontype:newvisitorcondition",
+ "conditiontype:notcondition",
+ "conditiontype:pasteventcondition",
+ "conditiontype:profilealiasespropertycondition",
+ "conditiontype:profilepropertycondition",
+ "conditiontype:profilesegmentcondition",
+ "conditiontype:profileupdatedeventcondition",
+ "conditiontype:profileuserlistcondition",
+ "conditiontype:returningvisitorcondition",
+ "conditiontype:scoringcondition",
+ "conditiontype:sessioncreatedeventcondition",
+ "conditiontype:sessiondurationcondition",
+ "conditiontype:sessionpropertycondition",
+ "conditiontype:sourceeventpropertycondition",
+ "conditiontype:topicpropertycondition",
+ "conditiontype:updateconsenteventcondition",
+ "conditiontype:updatelistseventcondition",
+ "conditiontype:updatepropertieseventcondition",
+ "conditiontype:userlistpropertycondition",
+ "conditiontype:videovieweventcondition",
+ "persona:europeanvisitor",
+ "persona:usvisitor",
+ "personasession:51c8e9a8-3db6-41f8-9569-057c339e3017",
+ "personasession:aa3b04bd-8f4d-4a07-8e96-d33ffa04d3d9",
+ "propertytype:address",
+ "propertytype:age",
+ "propertytype:birthdate",
+ "propertytype:city",
+ "propertytype:company",
+ "propertytype:countryname",
+ "propertytype:devicecategory",
+ "propertytype:email",
+ "propertytype:facebookid",
+ "propertytype:firstname",
+ "propertytype:firstvisit",
+ "propertytype:gender",
+ "propertytype:googleid",
+ "propertytype:income",
+ "propertytype:jobtitle",
+ "propertytype:kids",
+ "propertytype:lastname",
+ "propertytype:lastvisit",
+ "propertytype:latitude",
+ "propertytype:leadassignedto",
+ "propertytype:linkedinid",
+ "propertytype:longitude",
+ "propertytype:maritalstatus",
+ "propertytype:nationality",
+ "propertytype:nbofvisits",
+ "propertytype:operatingsystemfamily",
+ "propertytype:operatingsystemname",
+ "propertytype:pagereferringurl",
+ "propertytype:phonenumber",
+ "propertytype:previousvisit",
+ "propertytype:sessionadminsubdiv1",
+ "propertytype:sessionadminsubdiv2",
+ "propertytype:sessioncity",
+ "propertytype:sessioncountrycode",
+ "propertytype:sessioncountryname",
+ "propertytype:totalnbofvisits",
+ "propertytype:twitterid",
+ "propertytype:useragentname",
+ "propertytype:useragentversion",
+ "propertytype:weatherlike",
+ "propertytype:weathertemperature",
+ "propertytype:weatherwinddirection",
+ "propertytype:weatherwindspeed",
+ "propertytype:zipcode",
+ "rule:_8s3odnl0n_sessionreferrer",
+ "rule:_ajhg9u2s5_sessionassigned",
+ "rule:_ed2tvbd9p_evaluateprofileage.json",
+ "rule:_i6iqbc3x8_sessionpagereferrer",
+ "rule:_t7caz8npj_evaluateprofilesegments",
+ "rule:_z21qiv8b1_newsession",
+ "rule:cdp_sessionevent",
+ "rule:examplelogin",
+ "rule:modifyanyconsent",
+ "rule:optimizationtest",
+ "rule:smp:incrementtweetnumber",
+ "rule:smp:trainedrule",
+ "rule:testgroovyactionrule",
+ "rule:updateconsent",
+ "rule:updatelists",
+ "rule:updateproperties"
+]
diff --git
a/tools/shell-commands/src/test/java/org/apache/unomi/shell/migration/utils/MigrationUtilsTest.java
b/tools/shell-commands/src/test/java/org/apache/unomi/shell/migration/utils/MigrationUtilsTest.java
index 92b48578c..f2c1ea1a1 100644
---
a/tools/shell-commands/src/test/java/org/apache/unomi/shell/migration/utils/MigrationUtilsTest.java
+++
b/tools/shell-commands/src/test/java/org/apache/unomi/shell/migration/utils/MigrationUtilsTest.java
@@ -16,23 +16,36 @@
*/
package org.apache.unomi.shell.migration.utils;
+import org.json.JSONArray;
+import org.json.JSONObject;
import org.junit.Before;
import org.junit.Test;
import org.osgi.framework.Bundle;
import org.osgi.framework.BundleContext;
import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.nio.charset.StandardCharsets;
+import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.*;
public class MigrationUtilsTest {
@@ -302,4 +315,98 @@ public class MigrationUtilsTest {
assertEquals("sfdcconfiguration",
MigrationUtils.resolveItemType("context-sfdcconfiguration", "context", types));
assertEquals("generic",
MigrationUtils.resolveItemType("other-profile", "context", types));
}
+
+ private static final String SHIPPED_DEFINITIONS =
"requestBody/4.0.0/shipped_system_definitions.json";
+
+ @Test
+ public void loadShippedDefinitionIdsGroupsLowerCaseIdsByType() throws
Exception {
+ stubShippedDefinitions("[\"conditionType:booleanCondition\",
\"rule:_abc_myRule.json\", \"rule:other\"]");
+ Map<String, Set<String>> shipped =
MigrationUtils.loadShippedDefinitionIds(bundleContext);
+ assertEquals(Set.of("conditiontype", "rule"), shipped.keySet());
+ assertEquals(Set.of("booleancondition"), shipped.get("conditiontype"));
+ assertEquals(Set.of("_abc_myrule.json", "other"), shipped.get("rule"));
+ }
+
+ @Test
+ public void loadShippedDefinitionIdsRejectsEmptyOrMalformedLists() throws
Exception {
+ for (String content : Arrays.asList("[]", "[\"booleanCondition\"]",
"[\":booleanCondition\"]", "[\"rule:\"]", "[42]")) {
+ stubShippedDefinitions(content);
+ assertThrows(content, IllegalStateException.class, () ->
MigrationUtils.loadShippedDefinitionIds(bundleContext));
+ }
+ }
+
+ /**
+ * The shipped list is a snapshot: it must be updated whenever a
definition is added to or
+ * removed from the bundles, or migrated copies land in the wrong tenant.
+ */
+ @Test
+ public void shippedDefinitionsListMatchesTheDefinitionsInTheSourceTree()
throws Exception {
+ Path repositoryRoot =
Path.of("").toAbsolutePath().getParent().getParent();
+ assertTrue("Unexpected source layout: " + repositoryRoot,
Files.isDirectory(repositoryRoot.resolve("plugins")));
+
+ Map<String, String> typeByDirectory = Map.of("conditions",
"conditiontype", "actions", "actiontype",
+ "properties", "propertytype", "rules", "rule", "personas",
"persona");
+ Map<String, Set<String>> expected = new TreeMap<>();
+ Files.walkFileTree(repositoryRoot, new SimpleFileVisitor<Path>() {
+ @Override
+ public FileVisitResult preVisitDirectory(Path dir,
BasicFileAttributes attrs) {
+ String name = dir.getFileName().toString();
+ boolean skip = name.startsWith(".") || name.equals("target")
|| name.equals("node_modules") || name.equals("test");
+ return skip ? FileVisitResult.SKIP_SUBTREE :
FileVisitResult.CONTINUE;
+ }
+
+ @Override
+ public FileVisitResult visitFile(Path file, BasicFileAttributes
attrs) throws IOException {
+ String path =
repositoryRoot.relativize(file).toString().replace(File.separatorChar, '/');
+ int cxs = path.indexOf("/src/main/resources/META-INF/cxs/");
+ if (cxs < 0 || !path.endsWith(".json")) {
+ return FileVisitResult.CONTINUE;
+ }
+ String directory = path.substring(cxs +
"/src/main/resources/META-INF/cxs/".length()).split("/")[0];
+ String type = typeByDirectory.get(directory);
+ if (type == null) {
+ return FileVisitResult.CONTINUE;
+ }
+ JSONObject definition = new JSONObject(Files.readString(file));
+ if ("persona".equals(type)) {
+ addExpected(expected, type,
definition.getJSONObject("persona").getString("itemId"));
+ JSONArray sessions = definition.getJSONArray("sessions");
+ for (int i = 0; i < sessions.length(); i++) {
+ addExpected(expected, "personasession",
sessions.getJSONObject(i).getString("itemId"));
+ }
+ } else {
+ addExpected(expected, type,
definition.getJSONObject("metadata").getString("id"));
+ }
+ return FileVisitResult.CONTINUE;
+ }
+ });
+
+
when(bundle.getResource(SHIPPED_DEFINITIONS)).thenReturn(getClass().getResource("/"
+ SHIPPED_DEFINITIONS));
+ assertEquals("Update " + SHIPPED_DEFINITIONS + " to match the
definitions shipped under META-INF/cxs",
+ expected,
MigrationUtils.loadShippedDefinitionIds(bundleContext));
+ }
+
+ private static void addExpected(Map<String, Set<String>> expected, String
type, String id) {
+ expected.computeIfAbsent(type, key -> new
TreeSet<>()).add(id.toLowerCase(Locale.ROOT));
+ }
+
+ private void stubShippedDefinitions(String content) throws IOException {
+ Path file = Files.createTempFile("shipped-definitions", ".json");
+ file.toFile().deleteOnExit();
+ Files.writeString(file, content);
+
when(bundle.getResource(SHIPPED_DEFINITIONS)).thenReturn(file.toUri().toURL());
+ }
+
+ @Test
+ public void buildItemTypeUpdateRequestMatchesKeywordAndFoldedItemType() {
+ JSONObject request = new
JSONObject(MigrationUtils.buildItemTypeUpdateRequest("ctx.op = 'noop'",
"conditionType"));
+ assertEquals("ctx.op = 'noop'",
request.getJSONObject("script").getString("source"));
+ assertEquals("painless",
request.getJSONObject("script").getString("lang"));
+ JSONObject bool = request.getJSONObject("query").getJSONObject("bool");
+ assertEquals(1, bool.getInt("minimum_should_match"));
+ JSONArray should = bool.getJSONArray("should");
+ assertEquals(2, should.length());
+ assertEquals("conditionType",
should.getJSONObject(0).getJSONObject("term").getString("itemType.keyword"));
+ assertEquals("conditiontype",
should.getJSONObject(1).getJSONObject("term").getString("itemType"));
+ }
}
