This is an automated email from the ASF dual-hosted git repository.
sergehuber pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/unomi.git
The following commit(s) were added to refs/heads/master by this push:
new 670319cce UNOMI-998: Cover a 3.0 to 4.0 migration in the integration
tests (#910)
670319cce is described below
commit 670319cce67abc20c3b95a78e9f7c3eaa4279ace
Author: Serge Huber <[email protected]>
AuthorDate: Thu Oct 8 12:29:59 2026 +0200
UNOMI-998: Cover a 3.0 to 4.0 migration in the integration tests (#910)
Co-authored-by: Claude Opus 5.5 <[email protected]>
---
.../test/java/org/apache/unomi/itests/AllITs.java | 2 +
.../apache/unomi/itests/CorePersistenceITs.java | 2 +
.../migration/Migrate16xToCurrentVersionIT.java | 11 +
.../migration/Migrate30ToCurrentVersionIT.java | 375 +++++++++++++++++++++
.../itests/persistence/OpenSearchITBackend.java | 12 +-
.../migrate-4.0.0-10-tenantInitialization.groovy | 4 +-
.../main/resources/org.apache.unomi.migration.cfg | 4 +-
.../initialize_tenant_and_audit_fields.painless | 11 +-
8 files changed, 414 insertions(+), 7 deletions(-)
diff --git a/itests/src/test/java/org/apache/unomi/itests/AllITs.java
b/itests/src/test/java/org/apache/unomi/itests/AllITs.java
index b072a31fd..76c1ad054 100644
--- a/itests/src/test/java/org/apache/unomi/itests/AllITs.java
+++ b/itests/src/test/java/org/apache/unomi/itests/AllITs.java
@@ -18,6 +18,7 @@ package org.apache.unomi.itests;
import org.apache.unomi.itests.migration.Migrate16xToCurrentVersionIT;
import org.apache.unomi.itests.graphql.*;
+import org.apache.unomi.itests.migration.Migrate30ToCurrentVersionIT;
import org.apache.unomi.itests.migration.Migrate40IndexMappingIT;
import org.apache.unomi.itests.migration.Migrate40TenantAssignmentIT;
import org.apache.unomi.itests.migration.MigrationIT;
@@ -34,6 +35,7 @@ import org.junit.runners.Suite.SuiteClasses;
*/
@RunWith(ProgressSuite.class)
@SuiteClasses({
+ Migrate30ToCurrentVersionIT.class,
Migrate16xToCurrentVersionIT.class,
MigrationIT.class,
Migrate40IndexMappingIT.class,
diff --git
a/itests/src/test/java/org/apache/unomi/itests/CorePersistenceITs.java
b/itests/src/test/java/org/apache/unomi/itests/CorePersistenceITs.java
index 6f3c510ef..38d88b17a 100644
--- a/itests/src/test/java/org/apache/unomi/itests/CorePersistenceITs.java
+++ b/itests/src/test/java/org/apache/unomi/itests/CorePersistenceITs.java
@@ -18,6 +18,7 @@ package org.apache.unomi.itests;
import org.apache.unomi.itests.migration.Migrate16xToCurrentVersionIT;
import org.apache.unomi.itests.graphql.*;
+import org.apache.unomi.itests.migration.Migrate30ToCurrentVersionIT;
import org.apache.unomi.itests.migration.Migrate40IndexMappingIT;
import org.apache.unomi.itests.migration.Migrate40TenantAssignmentIT;
import org.apache.unomi.itests.migration.MigrationIT;
@@ -35,6 +36,7 @@ import org.junit.runners.Suite.SuiteClasses;
*/
@RunWith(ProgressSuite.class)
@SuiteClasses({
+ Migrate30ToCurrentVersionIT.class,
Migrate16xToCurrentVersionIT.class,
MigrationIT.class,
Migrate40IndexMappingIT.class,
diff --git
a/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xToCurrentVersionIT.java
b/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xToCurrentVersionIT.java
index 63e0755ef..aa325edf4 100644
---
a/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xToCurrentVersionIT.java
+++
b/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xToCurrentVersionIT.java
@@ -40,6 +40,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
import java.util.*;
@Category(SearchBackendIT.class)
@@ -125,6 +127,15 @@ public class Migrate16xToCurrentVersionIT extends BaseIT {
// Restore snapshot from 1.6.x
try (CloseableHttpClient httpClient = createSearchEngineHttpClient()) {
+ // A prior migration IT (e.g. Migrate30) may have left open
context-* indices.
+ // Snapshot restore refuses to overwrite open indices, so clear
them first.
+ for (String index :
MigrationUtils.getIndexesPrefixedBy(httpClient, getEsBaseUrl(),
INDEX_PREFIX_CONTEXT)) {
+ LOGGER.info("Deleting existing index before 1.6 snapshot
restore: {}", index);
+ MigrationUtils.deleteIndex(httpClient, getEsBaseUrl(), index);
+ }
+ // Its completed history would make this migration skip every
4.0.0 step.
+ Files.deleteIfExists(Paths.get(karafData(), "migration",
"history.json"));
+
// Create snapshot repo
HttpUtils.executePutRequest(httpClient, getEsSnapshotRepo(),
resourceAsString(RESOURCE_CREATE_SNAPSHOTS_REPO), null);
// Get snapshot, insure it exists
diff --git
a/itests/src/test/java/org/apache/unomi/itests/migration/Migrate30ToCurrentVersionIT.java
b/itests/src/test/java/org/apache/unomi/itests/migration/Migrate30ToCurrentVersionIT.java
new file mode 100644
index 000000000..6137b73a6
--- /dev/null
+++
b/itests/src/test/java/org/apache/unomi/itests/migration/Migrate30ToCurrentVersionIT.java
@@ -0,0 +1,375 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License
+ */
+package org.apache.unomi.itests.migration;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.unomi.api.conditions.ConditionType;
+import org.apache.unomi.api.tenants.Tenant;
+import org.apache.unomi.itests.BaseIT;
+import org.apache.unomi.itests.persistence.SearchBackendIT;
+import org.apache.unomi.shell.migration.utils.HttpRequestException;
+import org.apache.unomi.shell.migration.utils.HttpUtils;
+import org.apache.unomi.shell.migration.utils.MigrationUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.osgi.framework.Bundle;
+import org.osgi.framework.FrameworkUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.charset.StandardCharsets;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Arrays;
+
+/**
+ * UNOMI-998: migrate a synthetic Unomi 3.0 data set with {@code unomi:migrate
3.0.0}
+ * before Unomi starts, then assert the post-migration shape that 996/997
fixed.
+ * <p>
+ * Must run before {@link Migrate16xToCurrentVersionIT}, which deletes this
fixture and the
+ * migration history before restoring its 1.6 snapshot. Where that snapshot
migration runs
+ * (Elasticsearch), this test leaves Unomi stopped and only checks the
migrated indices. Uses HTTP seeding so both Elasticsearch and OpenSearch
+ * can run it ({@code httpAdminApi}), unlike the 1.6 snapshot path.
+ * <p>
+ * One {@code @Test} method only: {@code @Before} must not re-seed on every
method, or the
+ * second migrate is skipped by migration history and leaves documents without
tenants.
+ * <p>
+ * Pax Exam still installs the full feature set for the shared suite. The
boot-feature
+ * mapping path is guarded by asserting that the migration command bundle
itself ships
+ * the engine mappings (UNOMI-996).
+ */
+@Category(SearchBackendIT.class)
+public class Migrate30ToCurrentVersionIT extends BaseIT {
+
+ private static final Logger LOGGER =
LoggerFactory.getLogger(Migrate30ToCurrentVersionIT.class);
+
+ private static final String MIGRATION_COMMAND = "unomi:migrate 3.0.0 true";
+ private static final long MIGRATION_TIMEOUT = 900000L;
+
+ private static final String INDEX_PREFIX = "context-";
+ private static final String INDEX_SYSTEMITEMS = INDEX_PREFIX +
"systemitems";
+ private static final String INDEX_PROFILE = INDEX_PREFIX + "profile";
+ private static final String INDEX_GEONAME = INDEX_PREFIX + "geonameentry";
+ private static final String INDEX_CLUSTERNODE = INDEX_PREFIX +
"clusternode";
+ private static final String INDEX_SFDC = INDEX_PREFIX +
"sfdcconfiguration";
+ private static final String INDEX_TENANT = INDEX_PREFIX + "tenant";
+
+ private static final String USER_RULE_ID = "unomi998-user-rule";
+ private static final String USER_SEGMENT_ID = "unomi998-user-segment";
+ private static final String GEO_ID = "unomi998-paris";
+ private static final String NODE_ID = "unomi998-node";
+ private static final String SFDC_ID = "unomi998-sfdc";
+ private static final String PROFILE_ID = "unomi998-profile";
+ private static final String OLD_CONDITION_MARKER =
"unomi998-3.0-seed-booleanCondition";
+ // Post-2.2 document id for system condition types:
system_<id>_conditiontype
+ private static final String SHIPPED_CONDITION_DOC_ID =
"system_booleanCondition_conditiontype";
+
+ private static final String SIMPLE_MAPPING = "{"
+ + "\"settings\":{\"number_of_shards\":1,\"number_of_replicas\":0},"
+ + "\"mappings\":{\"properties\":{"
+ + "\"itemId\":{\"type\":\"keyword\"},"
+ +
"\"itemType\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\"}}},"
+ + "\"scope\":{\"type\":\"keyword\"},"
+ + "\"tenantId\":{\"type\":\"keyword\"},"
+ + "\"metadata\":{\"type\":\"object\",\"enabled\":true},"
+ + "\"name\":{\"type\":\"keyword\"},"
+ + "\"properties\":{\"type\":\"object\",\"enabled\":true}"
+ + "}}}";
+
+ private CloseableHttpClient httpClient;
+
+ @Override
+ @Before
+ public void waitForStartup() throws InterruptedException {
+ checkSearchEngine();
+
+ if (!persistenceCapabilities().httpAdminApi()) {
+ System.out.println("httpAdminApi not supported for provider "
+ + getPersistenceBackend().providerId() + " — starting
Unomi without 3.0 migrate seed");
+ super.waitForStartup();
+ return;
+ }
+
+ try {
+ httpClient = createSearchEngineHttpClient();
+ // A completed history from an earlier run would make this migrate
skip every step
+ Path history = Paths.get(karafData(), "migration", "history.json");
+ Files.deleteIfExists(history);
+ seedUnomi30Dataset(httpClient);
+ assertMigrationBundleShipsEngineMappings();
+
+ System.out.println("Launching migration from 3.0.0...");
+ LOGGER.info("Launching migration from 3.0.0...");
+ String migrationOutput = executeCommand(MIGRATION_COMMAND,
MIGRATION_TIMEOUT, false);
+ System.out.println("Migration command output results:");
+ System.out.println(migrationOutput);
+ Assert.assertNotNull(migrationOutput);
+ Assert.assertTrue("migration must run 4.0.0 tenant scripts",
+ migrationOutput.contains("tenantDocumentIds"));
+ // Only written once every script has finished without error
+ Assert.assertEquals("migration must finish", "COMPLETED",
+
getObjectMapper().readTree(history.toFile()).path("migrationStatus").asText());
+
+ assertMigrationWroteSystemDefinitionAndTenant();
+ prepareSearchEngineAfterMigration();
+ } catch (Throwable t) {
+ LOGGER.error("Error during 3.0 migration setup", t);
+ throw new RuntimeException("Error during 3.0 migration setup", t);
+ }
+
+ // Where the 1.6 snapshot migration runs next, it needs Unomi still
stopped: started here,
+ // Unomi would keep running on indices that test deletes and never
redeploy its definitions.
+ if (!persistenceCapabilities().snapshotRestoreMigration()) {
+ super.waitForStartup();
+ }
+ }
+
+ @After
+ public void closeHttpClient() throws Exception {
+ if (httpClient != null) {
+ httpClient.close();
+ }
+ }
+
+ @Test
+ public void checkMigratedDataFrom30() throws Exception {
+ Assume.assumeTrue(
+ "HTTP admin API required (provider=" +
getPersistenceBackend().providerId() + ")",
+ persistenceCapabilities().httpAdminApi());
+
+ assertUserContentStaysOutOfSystemTenant();
+ assertGeonamesClusterNodeAndSfdc();
+ assertEverySeededDocumentHasATenant();
+ assertApiKeyFileNamesCorrectHeaders();
+ if (unomiStarted) {
+ assertDefaultTenantExists();
+ assertShippedDefinitionsAreFourZero();
+ }
+ }
+
+ /**
+ * Checked before Unomi starts: startup deploys the bundled definitions
and BaseIT creates the
+ * test tenant when it is missing, so afterwards both would exist whatever
the migration did.
+ */
+ private void assertMigrationWroteSystemDefinitionAndTenant() throws
Exception {
+ HttpUtils.executePostRequest(httpClient, getSearchEngineBaseUrl() +
"/_refresh", null, null);
+
+ JsonNode shipped = getByDocumentId(INDEX_SYSTEMITEMS,
SHIPPED_CONDITION_DOC_ID);
+ Assert.assertNotNull("migration must write " +
SHIPPED_CONDITION_DOC_ID, shipped);
+ Assert.assertEquals("shipped definitions go to system", "system",
+ shipped.path("_source").path("tenantId").asText());
+
+ JsonNode tenants =
getObjectMapper().readTree(HttpUtils.executeGetRequest(httpClient,
+ getSearchEngineBaseUrl() + "/" + INDEX_TENANT + "/_search",
null)).path("hits").path("hits");
+ Assert.assertEquals("migration must write the default tenant", 1,
tenants.size());
+ Assert.assertEquals(TEST_TENANT_ID,
tenants.get(0).path("_source").path("itemId").asText());
+ }
+
+ private void assertDefaultTenantExists() throws Exception {
+ Assert.assertTrue("tenant index must exist",
+ MigrationUtils.indexExists(httpClient,
getSearchEngineBaseUrl(), INDEX_TENANT));
+
+ Tenant tenant = tenantService.getTenant(TEST_TENANT_ID);
+ Assert.assertNotNull("default tenant must exist after migration
(UNOMI-998)", tenant);
+ Assert.assertEquals(TEST_TENANT_ID, tenant.getItemId());
+ Assert.assertNotNull("tenant must have a public API key",
tenant.getPublicApiKey());
+ Assert.assertNotNull("tenant must have a private API key",
tenant.getPrivateApiKey());
+ }
+
+ private void assertUserContentStaysOutOfSystemTenant() throws Exception {
+ JsonNode rule = findByItemId(INDEX_SYSTEMITEMS, USER_RULE_ID);
+ Assert.assertNotNull("user rule must still exist", rule);
+ Assert.assertEquals(TEST_TENANT_ID,
rule.path("_source").path("tenantId").asText());
+ Assert.assertTrue(rule.path("_id").asText().startsWith(TEST_TENANT_ID
+ "_"));
+
+ JsonNode segment = findByItemId(INDEX_SYSTEMITEMS, USER_SEGMENT_ID);
+ Assert.assertNotNull("user segment must still exist", segment);
+ Assert.assertEquals(TEST_TENANT_ID,
segment.path("_source").path("tenantId").asText());
+ Assert.assertFalse("user segment must not use the system document id",
+ segment.path("_id").asText().startsWith("system_"));
+ }
+
+ private void assertGeonamesClusterNodeAndSfdc() throws Exception {
+ JsonNode geo = findByItemId(INDEX_GEONAME, GEO_ID);
+ Assert.assertNotNull(geo);
+ Assert.assertEquals("system",
geo.path("_source").path("tenantId").asText());
+
+ JsonNode node = findByItemId(INDEX_CLUSTERNODE, NODE_ID);
+ Assert.assertNotNull(node);
+ Assert.assertEquals("cluster nodes stay in the configured tenant",
TEST_TENANT_ID,
+ node.path("_source").path("tenantId").asText());
+
+ // Unknown index type uses forceDefaultTenant — literal scope "system"
does not win
+ JsonNode sfdc = findByItemId(INDEX_SFDC, SFDC_ID);
+ Assert.assertNotNull(sfdc);
+ Assert.assertEquals(TEST_TENANT_ID,
sfdc.path("_source").path("tenantId").asText());
+
+ JsonNode profile = findByItemId(INDEX_PROFILE, PROFILE_ID);
+ Assert.assertNotNull(profile);
+ Assert.assertEquals(TEST_TENANT_ID,
profile.path("_source").path("tenantId").asText());
+ }
+
+ private void assertEverySeededDocumentHasATenant() throws Exception {
+ for (String index : Arrays.asList(INDEX_SYSTEMITEMS, INDEX_PROFILE,
INDEX_GEONAME, INDEX_CLUSTERNODE, INDEX_SFDC, INDEX_TENANT)) {
+ if (!MigrationUtils.indexExists(httpClient,
getSearchEngineBaseUrl(), index)) {
+ continue;
+ }
+ JsonNode hits =
getObjectMapper().readTree(HttpUtils.executePostRequest(httpClient,
+ getSearchEngineBaseUrl() + "/" + index + "/_search",
+
"{\"size\":10,\"_source\":false,\"query\":{\"bool\":{\"must_not\":{\"exists\":{\"field\":\"tenantId\"}}}}}",
null))
+ .path("hits").path("hits");
+ Assert.assertEquals(index + " has documents without tenantId: " +
hits, 0, hits.size());
+ }
+ }
+
+ private void assertShippedDefinitionsAreFourZero() throws Exception {
+ ConditionType booleanCondition =
definitionsService.getConditionType("booleanCondition");
+ Assert.assertNotNull("booleanCondition must be available after
migration", booleanCondition);
+ String description = booleanCondition.getMetadata() != null
+ ? booleanCondition.getMetadata().getDescription() : null;
+ Assert.assertNotEquals("3.0 seeded marker must not win over the 4.0
bundle definition",
+ OLD_CONDITION_MARKER, description);
+ }
+
+ private void assertApiKeyFileNamesCorrectHeaders() throws Exception {
+ Path secretsDir = Paths.get(karafData(), "migration", "secrets");
+ Assert.assertTrue("migration secrets directory must exist: " +
secretsDir, Files.isDirectory(secretsDir));
+
+ Path keyFile = null;
+ try (DirectoryStream<Path> stream =
Files.newDirectoryStream(secretsDir, "tenant-api-keys-" + TEST_TENANT_ID +
"-*.txt")) {
+ for (Path path : stream) {
+ keyFile = path;
+ break;
+ }
+ }
+ Assert.assertNotNull("migration must write a tenant API key file",
keyFile);
+
+ String content = Files.readString(keyFile, StandardCharsets.UTF_8);
+ Assert.assertTrue("public key must name X-Unomi-Api-Key\n" + content,
+ content.contains("X-Unomi-Api-Key"));
+ Assert.assertTrue("private key must mention Basic authentication\n" +
content,
+ content.toLowerCase().contains("basic"));
+ Assert.assertFalse("must not advertise the non-existent
X-Unomi-Public-Key header\n" + content,
+ content.contains("X-Unomi-Public-Key"));
+ Assert.assertFalse("must not advertise the non-existent X-Unomi-Key
header\n" + content,
+ content.contains("X-Unomi-Key header"));
+ }
+
+ private void seedUnomi30Dataset(CloseableHttpClient client) throws
Exception {
+ String es = getSearchEngineBaseUrl();
+ System.out.println("Seeding synthetic Unomi 3.0 dataset...");
+ LOGGER.info("Seeding synthetic Unomi 3.0 dataset...");
+
+ // The migration only creates the tenant (and its key file) when the
tenant index is missing
+ MigrationUtils.deleteIndex(client, es, INDEX_TENANT);
+ for (String index : Arrays.asList(INDEX_SYSTEMITEMS, INDEX_PROFILE,
INDEX_GEONAME, INDEX_CLUSTERNODE, INDEX_SFDC)) {
+ MigrationUtils.deleteIndex(client, es, index);
+ MigrationUtils.createIndex(client, es, index, SIMPLE_MAPPING);
+ }
+
+ putDoc(client, INDEX_SYSTEMITEMS, "booleanCondition",
+
"{\"itemId\":\"booleanCondition\",\"itemType\":\"conditionType\","
+ +
"\"metadata\":{\"id\":\"booleanCondition\",\"description\":\"" +
OLD_CONDITION_MARKER + "\"}}");
+ putDoc(client, INDEX_SYSTEMITEMS, USER_RULE_ID,
+ "{\"itemId\":\"" + USER_RULE_ID +
"\",\"itemType\":\"rule\",\"scope\":\"systemscope\","
+ + "\"metadata\":{\"id\":\"" + USER_RULE_ID +
"\",\"name\":\"UNOMI-998 user rule\"}}");
+ putDoc(client, INDEX_SYSTEMITEMS, USER_SEGMENT_ID,
+ "{\"itemId\":\"" + USER_SEGMENT_ID +
"\",\"itemType\":\"segment\",\"scope\":\"systemscope\","
+ + "\"metadata\":{\"id\":\"" + USER_SEGMENT_ID +
"\",\"name\":\"UNOMI-998 user segment\"}}");
+
+ putDoc(client, INDEX_GEONAME, GEO_ID,
+ "{\"itemId\":\"" + GEO_ID +
"\",\"itemType\":\"geonameEntry\",\"name\":\"Paris\"}");
+ putDoc(client, INDEX_CLUSTERNODE, NODE_ID,
+ "{\"itemId\":\"" + NODE_ID +
"\",\"itemType\":\"clusterNode\",\"scope\":\"system\"}");
+ putDoc(client, INDEX_SFDC, SFDC_ID,
+ "{\"itemId\":\"" + SFDC_ID +
"\",\"itemType\":\"sfdcConfiguration\",\"scope\":\"system\"}");
+ putDoc(client, INDEX_PROFILE, PROFILE_ID,
+ "{\"itemId\":\"" + PROFILE_ID +
"\",\"itemType\":\"profile\",\"properties\":{\"firstName\":\"Ada\"}}");
+
+ HttpUtils.executePostRequest(client, es + "/_refresh", null, null);
+ }
+
+ private void assertMigrationBundleShipsEngineMappings() {
+ Bundle migrationBundle = FrameworkUtil.getBundle(MigrationUtils.class);
+ Assert.assertNotNull(migrationBundle);
+ for (String engine : Arrays.asList("elasticsearch", "opensearch")) {
+ for (String mappingFile : Arrays.asList("clusterNode.json",
"event.json", "geonameEntry.json",
+ "personaSession.json", "profile.json",
"profileAlias.json", "session.json", "systemItems.json", "tenant.json")) {
+ String path = "META-INF/cxs/migration-mappings/" + engine +
"/" + mappingFile;
+ Assert.assertNotNull("boot migrate needs " + path + " in
shell-commands",
+ migrationBundle.getEntry(path));
+ }
+ }
+ }
+
+ private void putDoc(CloseableHttpClient client, String index, String id,
String source) throws Exception {
+ HttpUtils.executePutRequest(client,
+ getSearchEngineBaseUrl() + "/" + index + "/_doc/" + id +
"?refresh=true", source, null);
+ }
+
+ private JsonNode getByDocumentId(String index, String documentId) throws
Exception {
+ try {
+ String response = HttpUtils.executeGetRequest(httpClient,
+ getSearchEngineBaseUrl() + "/" + index + "/_doc/" +
documentId, null);
+ if (response == null || response.isBlank()) {
+ return null;
+ }
+ JsonNode node = getObjectMapper().readTree(response);
+ if (!node.path("found").asBoolean(false)) {
+ return null;
+ }
+ return node;
+ } catch (HttpRequestException e) {
+ // HttpUtils throws on HTTP 404 when the document is missing
+ if (e.getCode() != 404) {
+ throw e;
+ }
+ return null;
+ }
+ }
+
+ private JsonNode findByItemId(String index, String itemId) throws
Exception {
+ // After migrate reindex, string fields use the folding text mapping
with a .keyword subfield.
+ // Term/prefix on the analyzed field miss camelCase ids such as
booleanCondition.
+ String body = "{\"size\":20,\"query\":{\"bool\":{\"should\":["
+ + "{\"term\":{\"itemId.keyword\":\"" + itemId + "\"}},"
+ + "{\"prefix\":{\"itemId.keyword\":\"" + itemId + "_\"}},"
+ + "{\"term\":{\"itemId\":\"" + itemId + "\"}},"
+ + "{\"prefix\":{\"itemId\":\"" + itemId + "_\"}}"
+ + "],\"minimum_should_match\":1}}}";
+ JsonNode hits =
getObjectMapper().readTree(HttpUtils.executePostRequest(httpClient,
+ getSearchEngineBaseUrl() + "/" + index + "/_search", body,
null))
+ .path("hits").path("hits");
+ for (JsonNode hit : hits) {
+ String found = hit.path("_source").path("itemId").asText();
+ String docId = hit.path("_id").asText();
+ if (found.equals(itemId) || found.startsWith(itemId + "_")
+ || docId.equals(itemId) || docId.contains("_" + itemId +
"_")
+ || docId.endsWith("_" + itemId)) {
+ return hit;
+ }
+ }
+ return null;
+ }
+}
diff --git
a/itests/src/test/java/org/apache/unomi/itests/persistence/OpenSearchITBackend.java
b/itests/src/test/java/org/apache/unomi/itests/persistence/OpenSearchITBackend.java
index 04e182264..4c225602f 100644
---
a/itests/src/test/java/org/apache/unomi/itests/persistence/OpenSearchITBackend.java
+++
b/itests/src/test/java/org/apache/unomi/itests/persistence/OpenSearchITBackend.java
@@ -82,15 +82,25 @@ public class OpenSearchITBackend implements
PersistenceITBackend {
@Override
public Option[] configurationOptions() {
String port = searchPort();
+ String address = "localhost:" + port;
return new Option[]{
editConfigurationFilePut("etc/custom.system.properties",
"org.apache.unomi.opensearch.cluster.name", "contextElasticSearchITests"),
- editConfigurationFilePut("etc/custom.system.properties",
"org.apache.unomi.opensearch.addresses", "localhost:" + port),
+ editConfigurationFilePut("etc/custom.system.properties",
"org.apache.unomi.opensearch.addresses", address),
editConfigurationFilePut("etc/custom.system.properties",
"org.apache.unomi.opensearch.username", OS_USER),
editConfigurationFilePut("etc/custom.system.properties",
"org.apache.unomi.opensearch.password", OS_PASSWORD),
editConfigurationFilePut("etc/custom.system.properties",
"org.apache.unomi.opensearch.sslEnable", "false"),
editConfigurationFilePut("etc/custom.system.properties",
"org.apache.unomi.opensearch.sslTrustAllCertificates", "true"),
editConfigurationFilePut("etc/custom.system.properties",
"org.apache.unomi.opensearch.rollover.maxDocs", "300"),
editConfigurationFilePut("etc/custom.system.properties",
"org.apache.unomi.opensearch.minimalClusterState", "YELLOW"),
+ // org.apache.unomi.migration.cfg still resolves connection
settings from the
+ // elasticsearch.* placeholders. Mirror OpenSearch here so
unomi:migrate (e.g.
+ // Migrate30ToCurrentVersionIT) does not fall back to
localhost:9200.
+ editConfigurationFilePut("etc/custom.system.properties",
"org.apache.unomi.elasticsearch.addresses", address),
+ editConfigurationFilePut("etc/custom.system.properties",
"org.apache.unomi.elasticsearch.username", OS_USER),
+ editConfigurationFilePut("etc/custom.system.properties",
"org.apache.unomi.elasticsearch.password", OS_PASSWORD),
+ editConfigurationFilePut("etc/custom.system.properties",
"org.apache.unomi.elasticsearch.sslEnable", "false"),
+ editConfigurationFilePut("etc/custom.system.properties",
"org.apache.unomi.elasticsearch.sslTrustAllCertificates", "true"),
+ editConfigurationFilePut("etc/custom.system.properties",
"org.apache.unomi.elasticsearch.rollover.maxDocs", "300"),
};
}
diff --git
a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-4.0.0-10-tenantInitialization.groovy
b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-4.0.0-10-tenantInitialization.groovy
index dada0f0f4..2cb53fdac 100644
---
a/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-4.0.0-10-tenantInitialization.groovy
+++
b/tools/shell-commands/src/main/resources/META-INF/cxs/migration/migrate-4.0.0-10-tenantInitialization.groovy
@@ -107,10 +107,10 @@ ${sep}
Generated : ${isoDate}
Tenant ID : ${tenantId}
-PUBLIC KEY (X-Unomi-Public-Key header -- context.json / event collector):
+PUBLIC KEY (X-Unomi-Api-Key header -- context.json / event collector):
${generatedPublicKey}
-PRIVATE KEY (X-Unomi-Key header -- protected / admin endpoints):
+PRIVATE KEY (HTTP Basic authentication as tenantId:privateKey -- protected /
admin endpoints):
${generatedPrivateKey}
IMPORTANT: These keys cannot be recovered after this file is deleted.
diff --git
a/tools/shell-commands/src/main/resources/org.apache.unomi.migration.cfg
b/tools/shell-commands/src/main/resources/org.apache.unomi.migration.cfg
index ef347295a..62423a6af 100644
--- a/tools/shell-commands/src/main/resources/org.apache.unomi.migration.cfg
+++ b/tools/shell-commands/src/main/resources/org.apache.unomi.migration.cfg
@@ -17,7 +17,9 @@
# Migration config used for silent migration
-# Various configuration related to ElasticSearch communication to be able to
connect and migrate the data
+# Various configuration related to search-engine communication to be able to
connect and migrate the data.
+# These keys read the org.apache.unomi.elasticsearch.* system properties for
historical reasons, also when
+# the search engine is OpenSearch: set them to your OpenSearch connection
settings before running unomi:migrate.
esAddresses = ${org.apache.unomi.elasticsearch.addresses:-localhost:9200}
esSSLEnabled = ${org.apache.unomi.elasticsearch.sslEnable:-false}
esLogin = ${org.apache.unomi.elasticsearch.username:-}
diff --git
a/tools/shell-commands/src/main/resources/requestBody/4.0.0/initialize_tenant_and_audit_fields.painless
b/tools/shell-commands/src/main/resources/requestBody/4.0.0/initialize_tenant_and_audit_fields.painless
index 97dcd9b38..455735b03 100644
---
a/tools/shell-commands/src/main/resources/requestBody/4.0.0/initialize_tenant_and_audit_fields.painless
+++
b/tools/shell-commands/src/main/resources/requestBody/4.0.0/initialize_tenant_and_audit_fields.painless
@@ -20,12 +20,18 @@
// stay in the configured tenant even when their 3.0 scope is "systemscope".
// A shipped definition is recognised by item type and id only, so a 3.0 item
sharing both
// (such as a built-in rule edited in place) also goes to system.
+// The reindex reads from the "-cloned" backup and ctx._index is that source
index, so drop the suffix.
+def indexLower = ctx._index != null ? ctx._index.toLowerCase(Locale.ROOT) : '';
+if (indexLower.endsWith('-cloned')) {
+ indexLower = indexLower.substring(0, indexLower.length() -
'-cloned'.length());
+}
+
boolean useSystemTenant = false;
if (params.forceDefaultTenant != true) {
def itemTypeLower = ctx._source.itemType != null ?
ctx._source.itemType.toLowerCase(Locale.ROOT) : '';
// Shipped ids are grouped by item type, so other types (profiles,
events...) skip the lookup
def shippedIds = params.shippedDefinitionIds != null ?
params.shippedDefinitionIds.get(itemTypeLower) : null;
- if (itemTypeLower == 'geonameentry' || (ctx._index != null &&
ctx._index.toLowerCase(Locale.ROOT).endsWith('-geonameentry'))) {
+ if (itemTypeLower == 'geonameentry' ||
indexLower.endsWith('-geonameentry')) {
useSystemTenant = true;
} else if (shippedIds != null) {
def rawId = ctx._source.itemId != null ? ctx._source.itemId : ctx._id;
@@ -49,7 +55,7 @@ if (ctx._id != null) {
// Skip if already has tenant prefix
if (!ctx._id.startsWith(params.tenantId + '_') &&
!ctx._id.startsWith(params.systemTenantId + '_')) {
def baseId = ctx._id;
- if (ctx._index != null &&
ctx._index.toLowerCase(Locale.ROOT).endsWith('-systemitems')) {
+ if (indexLower.endsWith('-systemitems')) {
// For system items the document ID is tenantId_itemId_itemtype
(see getDocumentIDForItemType).
// 3.0 document IDs already carry the lower-case _itemtype suffix,
so only add it when missing.
def itemType = ctx._source.itemType != null ?
ctx._source.itemType.toLowerCase(Locale.ROOT) : null;
@@ -69,7 +75,6 @@ if (ctx._id != null) {
// systemitems must get audit here: UNOMI-997 puts shipped defs under the
system
// tenant before Unomi starts, so bundle deploy treats them as updates and
would
// otherwise leave createdBy unset on docs that never had it.
-def indexLower = ctx._index != null ? ctx._index.toLowerCase(Locale.ROOT) : '';
if (!indexLower.endsWith('-geonameentry') &&
!indexLower.endsWith('-geonames')) {
// Handle creation date based on item type
if (ctx._source.creationDate == null) {
