This is an automated email from the ASF dual-hosted git repository.
shuber pushed a commit to branch opensearch-persistence
in repository https://gitbox.apache.org/repos/asf/unomi.git
The following commit(s) were added to refs/heads/opensearch-persistence by this
push:
new 71368be21 Refactor monthly index properties to rollover properties in
configuration files and update related test cases. Rename test method for
clarity and adjust logging messages to reflect changes in index template
creation.
71368be21 is described below
commit 71368be2196cc979c77459a4aa74d3560d116948
Author: Serge Huber <[email protected]>
AuthorDate: Tue Oct 14 08:47:47 2025 +0200
Refactor monthly index properties to rollover properties in configuration
files and update related test cases. Rename test method for clarity and adjust
logging messages to reflect changes in index template creation.
---
.../org/apache/unomi/itests/ProfileServiceIT.java | 4 +--
.../main/resources/etc/custom.system.properties | 18 ++++------
.../ElasticSearchPersistenceServiceImpl.java | 2 +-
.../OpenSearchPersistenceServiceImpl.java | 42 ++++------------------
.../resources/OSGI-INF/blueprint/blueprint.xml | 20 +++--------
.../org.apache.unomi.persistence.opensearch.cfg | 16 +++------
6 files changed, 26 insertions(+), 76 deletions(-)
diff --git a/itests/src/test/java/org/apache/unomi/itests/ProfileServiceIT.java
b/itests/src/test/java/org/apache/unomi/itests/ProfileServiceIT.java
index 55eebd57b..2ff55fc12 100644
--- a/itests/src/test/java/org/apache/unomi/itests/ProfileServiceIT.java
+++ b/itests/src/test/java/org/apache/unomi/itests/ProfileServiceIT.java
@@ -354,7 +354,7 @@ public class ProfileServiceIT extends BaseIT {
}
@Test
- public void testMonthlyIndicesPurge() throws Exception {
+ public void testOldItemsPurge() throws Exception {
Date currentDate = new Date();
LocalDateTime minus10Months =
LocalDateTime.ofInstant(currentDate.toInstant(),
ZoneId.systemDefault()).minusMonths(10);
LocalDateTime minus30Months =
LocalDateTime.ofInstant(currentDate.toInstant(),
ZoneId.systemDefault()).minusMonths(30);
@@ -364,7 +364,7 @@ public class ProfileServiceIT extends BaseIT {
long originalSessionsCount =
persistenceService.getAllItemsCount(Session.ITEM_TYPE);
long originalEventsCount =
persistenceService.getAllItemsCount(Event.ITEM_TYPE);
- Profile profile = new Profile("dummy-profile-monthly-purge-test");
+ Profile profile = new Profile("dummy-profile-old-items-purge-test");
persistenceService.save(profile);
// create 10 months old items
diff --git a/package/src/main/resources/etc/custom.system.properties
b/package/src/main/resources/etc/custom.system.properties
index 681643fdf..7d57069c7 100644
--- a/package/src/main/resources/etc/custom.system.properties
+++ b/package/src/main/resources/etc/custom.system.properties
@@ -166,18 +166,12 @@
org.apache.unomi.opensearch.itemTypeToRefreshPolicy=${env:UNOMI_OPENSEARCH_REFRE
org.apache.unomi.opensearch.fatalIllegalStateErrors=${env:UNOMI_OPENSEARCH_FATAL_STATE_ERRORS:-}
org.apache.unomi.opensearch.index.prefix=${env:UNOMI_OPENSEARCH_INDEXPREFIX:-context}
-# These monthlyIndex properties are now deprecated, please use rollover
equivalent.
-org.apache.unomi.opensearch.monthlyIndex.nbShards=${env:UNOMI_OPENSEARCH_MONTHLYINDEX_SHARDS:-5}
-org.apache.unomi.opensearch.monthlyIndex.nbReplicas=${env:UNOMI_OPENSEARCH_MONTHLYINDEX_REPLICAS:-0}
-org.apache.unomi.opensearch.monthlyIndex.indexMappingTotalFieldsLimit=${env:UNOMI_OPENSEARCH_MONTHLYINDEX_MAPPINGTOTALFIELDSLIMIT:-1000}
-org.apache.unomi.opensearch.monthlyIndex.indexMaxDocValueFieldsSearch=${env:UNOMI_OPENSEARCH_MONTHLYINDEX_MAXDOCVALUEFIELDSSEARCH:-1000}
-org.apache.unomi.opensearch.monthlyIndex.itemsMonthlyIndexedOverride=${env:UNOMI_OPENSEARCH_MONTHLYINDEX_ITEMSMONTHLYINDEXED:-event,session}
-# New rollover properties (it overrides monthlyIndex values)
-org.apache.unomi.opensearch.rollover.nbShards=${env:UNOMI_OPENSEARCH_ROLLOVER_SHARDS}
-org.apache.unomi.opensearch.rollover.nbReplicas=${env:UNOMI_OPENSEARCH_ROLLOVER_REPLICAS}
-org.apache.unomi.opensearch.rollover.indexMappingTotalFieldsLimit=${env:UNOMI_OPENSEARCH_ROLLOVER_MAPPINGTOTALFIELDSLIMIT}
-org.apache.unomi.opensearch.rollover.indexMaxDocValueFieldsSearch=${env:UNOMI_OPENSEARCH_ROLLOVER_MAXDOCVALUEFIELDSSEARCH}
-org.apache.unomi.opensearch.rollover.indices=${env:UNOMI_OPENSEARCH_ROLLOVER_INDICES}
+# New rollover properties
+org.apache.unomi.opensearch.rollover.nbShards=${env:UNOMI_OPENSEARCH_ROLLOVER_SHARDS:-5}
+org.apache.unomi.opensearch.rollover.nbReplicas=${env:UNOMI_OPENSEARCH_ROLLOVER_REPLICAS:-0}
+org.apache.unomi.opensearch.rollover.indexMappingTotalFieldsLimit=${env:UNOMI_OPENSEARCH_ROLLOVER_MAPPINGTOTALFIELDSLIMIT:-1000}
+org.apache.unomi.opensearch.rollover.indexMaxDocValueFieldsSearch=${env:UNOMI_OPENSEARCH_ROLLOVER_MAXDOCVALUEFIELDSSEARCH:-1000}
+org.apache.unomi.opensearch.rollover.indices=${env:UNOMI_OPENSEARCH_ROLLOVER_INDICES:-event,session}
# Rollover configuration
org.apache.unomi.opensearch.rollover.maxSize=${env:UNOMI_OPENSEARCH_ROLLOVER_MAXSIZE:-30gb}
diff --git
a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
index 20eb85a5d..ed3b744f0 100644
---
a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
+++
b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
@@ -1412,7 +1412,7 @@ public class ElasticSearchPersistenceServiceImpl
implements PersistenceService,
private void internalCreateRolloverTemplate(String itemName) throws
IOException {
if (!mappings.containsKey(itemName)) {
- LOGGER.warn("Couldn't find mapping for item {}, won't create
monthly index template", itemName);
+ LOGGER.warn("Couldn't find mapping for item {}, won't create
rollover index template", itemName);
return;
}
diff --git
a/persistence-opensearch/core/src/main/java/org/apache/unomi/persistence/opensearch/OpenSearchPersistenceServiceImpl.java
b/persistence-opensearch/core/src/main/java/org/apache/unomi/persistence/opensearch/OpenSearchPersistenceServiceImpl.java
index a068c732b..afff43ee2 100644
---
a/persistence-opensearch/core/src/main/java/org/apache/unomi/persistence/opensearch/OpenSearchPersistenceServiceImpl.java
+++
b/persistence-opensearch/core/src/main/java/org/apache/unomi/persistence/opensearch/OpenSearchPersistenceServiceImpl.java
@@ -101,10 +101,6 @@ public class OpenSearchPersistenceServiceImpl implements
PersistenceService, Syn
private final List<String> openSearchAddressList = new ArrayList<>();
private String clusterName;
private String indexPrefix;
- private String monthlyIndexNumberOfShards;
- private String monthlyIndexNumberOfReplicas;
- private String monthlyIndexMappingTotalFieldsLimit;
- private String monthlyIndexMaxDocValueFieldsSearch;
private String numberOfShards;
private String numberOfReplicas;
private String indexMappingTotalFieldsLimit;
@@ -114,7 +110,6 @@ public class OpenSearchPersistenceServiceImpl implements
PersistenceService, Syn
private final Map<String, String> mappings = new HashMap<>();
private ConditionEvaluatorDispatcher conditionEvaluatorDispatcher;
private ConditionOSQueryBuilderDispatcher
conditionOSQueryBuilderDispatcher;
- private List<String> itemsMonthlyIndexed;
private Map<String, String> routingByType;
private Integer defaultQueryLimit = 10;
@@ -219,30 +214,7 @@ public class OpenSearchPersistenceServiceImpl implements
PersistenceService, Syn
this.indexPrefix = indexPrefix;
}
- @Deprecated
- public void setMonthlyIndexNumberOfShards(String
monthlyIndexNumberOfShards) {
- this.monthlyIndexNumberOfShards = monthlyIndexNumberOfShards;
- }
- @Deprecated
- public void setMonthlyIndexNumberOfReplicas(String
monthlyIndexNumberOfReplicas) {
- this.monthlyIndexNumberOfReplicas = monthlyIndexNumberOfReplicas;
- }
-
- @Deprecated
- public void setMonthlyIndexMappingTotalFieldsLimit(String
monthlyIndexMappingTotalFieldsLimit) {
- this.monthlyIndexMappingTotalFieldsLimit =
monthlyIndexMappingTotalFieldsLimit;
- }
-
- @Deprecated
- public void setMonthlyIndexMaxDocValueFieldsSearch(String
monthlyIndexMaxDocValueFieldsSearch) {
- this.monthlyIndexMaxDocValueFieldsSearch =
monthlyIndexMaxDocValueFieldsSearch;
- }
-
- @Deprecated
- public void setItemsMonthlyIndexedOverride(String
itemsMonthlyIndexedOverride) {
- this.itemsMonthlyIndexed =
StringUtils.isNotEmpty(itemsMonthlyIndexedOverride) ?
Arrays.asList(itemsMonthlyIndexedOverride.split(",").clone()) :
Collections.emptyList();
- }
public void setNumberOfShards(String numberOfShards) {
this.numberOfShards = numberOfShards;
@@ -1243,7 +1215,7 @@ public class OpenSearchPersistenceServiceImpl implements
PersistenceService, Syn
}
public boolean registerRolloverLifecyclePolicy() {
- Boolean result = new InClassLoaderExecute<Boolean>(metricsService,
this.getClass().getName() + ".createMonthlyIndexLifecyclePolicy",
this.bundleContext, this.fatalIllegalStateErrors, throwExceptions) {
+ Boolean result = new InClassLoaderExecute<Boolean>(metricsService,
this.getClass().getName() + ".createRolloverLifecyclePolicy",
this.bundleContext, this.fatalIllegalStateErrors, throwExceptions) {
protected Boolean execute(Object... args) throws IOException {
try {
String policyName = indexPrefix +
"-rollover-lifecycle-policy";
@@ -1394,15 +1366,15 @@ public class OpenSearchPersistenceServiceImpl
implements PersistenceService, Syn
private void internalCreateRolloverTemplate(String itemName) throws
IOException {
String rolloverAlias = indexPrefix + "-" + itemName;
if (mappings.get(itemName) == null) {
- LOGGER.warn("Couldn't find mapping for item {}, won't create
monthly index template", itemName);
+ LOGGER.warn("Couldn't find mapping for item {}, won't create
rollover index template", itemName);
return;
}
String indexSource =
" {" +
- " \"number_of_shards\" : " +
StringUtils.defaultIfEmpty(rolloverIndexNumberOfShards,
monthlyIndexNumberOfShards) + "," +
- " \"number_of_replicas\" : " +
StringUtils.defaultIfEmpty(rolloverIndexNumberOfReplicas,
monthlyIndexNumberOfReplicas) + "," +
- " \"mapping.total_fields.limit\" : " +
StringUtils.defaultIfEmpty(rolloverIndexMappingTotalFieldsLimit,
monthlyIndexMappingTotalFieldsLimit) + "," +
- " \"max_docvalue_fields_search\" : " +
StringUtils.defaultIfEmpty(rolloverIndexMaxDocValueFieldsSearch,
monthlyIndexMaxDocValueFieldsSearch) + "," +
+ " \"number_of_shards\" : " +
rolloverIndexNumberOfShards + "," +
+ " \"number_of_replicas\" : " +
rolloverIndexNumberOfReplicas + "," +
+ " \"mapping.total_fields.limit\" : " +
rolloverIndexMappingTotalFieldsLimit + "," +
+ " \"max_docvalue_fields_search\" : " +
rolloverIndexMaxDocValueFieldsSearch + "," +
" \"plugins.index_state_management.rollover_alias\":
\"" + rolloverAlias + "\"" +
" },";
String analysisSource =
@@ -2646,7 +2618,7 @@ public class OpenSearchPersistenceServiceImpl implements
PersistenceService, Syn
}
private boolean isItemTypeRollingOver(String itemType) {
- return (rolloverIndices != null ? rolloverIndices :
itemsMonthlyIndexed).contains(itemType);
+ return (rolloverIndices != null ? rolloverIndices.contains(itemType) :
false);
}
private Refresh getRefreshPolicy(String itemType) {
diff --git
a/persistence-opensearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml
b/persistence-opensearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml
index a8bcbcd54..ccfb8e73e 100644
---
a/persistence-opensearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml
+++
b/persistence-opensearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml
@@ -37,11 +37,6 @@
<cm:property name="numberOfReplicas" value="0"/>
<cm:property name="indexMappingTotalFieldsLimit" value="1000"/>
<cm:property name="indexMaxDocValueFieldsSearch" value="1000"/>
- <cm:property name="monthlyIndex.numberOfShards" value="3"/>
- <cm:property name="monthlyIndex.numberOfReplicas" value="0"/>
- <cm:property name="monthlyIndex.indexMappingTotalFieldsLimit"
value="1000"/>
- <cm:property name="monthlyIndex.indexMaxDocValueFieldsSearch"
value="1000"/>
- <cm:property name="monthlyIndex.itemsMonthlyIndexedOverride"
value="event,session"/>
<cm:property name="defaultQueryLimit" value="10"/>
<cm:property name="bulkProcessor.concurrentRequests" value="1" />
@@ -50,14 +45,14 @@
<cm:property name="bulkProcessor.flushInterval" value="5s" />
<cm:property name="bulkProcessor.backoffPolicy"
value="exponential" />
- <cm:property name="rollover.indices" value="" />
+ <cm:property name="rollover.indices" value="event,session" />
<cm:property name="rollover.maxSize" value="30gb" />
<cm:property name="rollover.maxAge" value="" />
<cm:property name="rollover.maxDocs" value="" />
- <cm:property name="rollover.numberOfShards" value=""/>
- <cm:property name="rollover.numberOfReplicas" value=""/>
- <cm:property name="rollover.indexMappingTotalFieldsLimit"
value=""/>
- <cm:property name="rollover.indexMaxDocValueFieldsSearch"
value=""/>
+ <cm:property name="rollover.numberOfShards" value="5"/>
+ <cm:property name="rollover.numberOfReplicas" value="0"/>
+ <cm:property name="rollover.indexMappingTotalFieldsLimit"
value="1000"/>
+ <cm:property name="rollover.indexMaxDocValueFieldsSearch"
value="1000"/>
<cm:property name="minimalOpenSearchVersion" value="2.1.0" />
<cm:property name="maximalOpenSearchVersion" value="4.0.0" />
@@ -111,10 +106,6 @@
<property name="conditionOSQueryBuilderDispatcher"
ref="conditionOSQueryBuilderDispatcher"/>
<property name="clusterName" value="${os.cluster.name}"/>
<property name="indexPrefix" value="${os.index.prefix}"/>
- <property name="monthlyIndexNumberOfShards"
value="${os.monthlyIndex.numberOfShards}"/>
- <property name="monthlyIndexNumberOfReplicas"
value="${os.monthlyIndex.numberOfReplicas}"/>
- <property name="monthlyIndexMappingTotalFieldsLimit"
value="${os.monthlyIndex.indexMappingTotalFieldsLimit}"/>
- <property name="monthlyIndexMaxDocValueFieldsSearch"
value="${os.monthlyIndex.indexMaxDocValueFieldsSearch}"/>
<property name="numberOfShards" value="${os.numberOfShards}"/>
<property name="numberOfReplicas" value="${os.numberOfReplicas}"/>
<property name="indexMappingTotalFieldsLimit"
value="${os.indexMappingTotalFieldsLimit}"/>
@@ -122,7 +113,6 @@
<property name="openSearchAddresses"
value="${os.openSearchAddresses}"/>
<property name="fatalIllegalStateErrors"
value="${os.fatalIllegalStateErrors}"/>
<property name="defaultQueryLimit" value="${os.defaultQueryLimit}"/>
- <property name="itemsMonthlyIndexedOverride"
value="${os.monthlyIndex.itemsMonthlyIndexedOverride}" />
<property name="routingByType">
<map>
</map>
diff --git
a/persistence-opensearch/core/src/main/resources/org.apache.unomi.persistence.opensearch.cfg
b/persistence-opensearch/core/src/main/resources/org.apache.unomi.persistence.opensearch.cfg
index 729d40a21..55d708459 100644
---
a/persistence-opensearch/core/src/main/resources/org.apache.unomi.persistence.opensearch.cfg
+++
b/persistence-opensearch/core/src/main/resources/org.apache.unomi.persistence.opensearch.cfg
@@ -23,18 +23,12 @@
openSearchAddresses=${org.apache.unomi.opensearch.addresses:-localhost:9200}
fatalIllegalStateErrors=${org.apache.unomi.opensearch.fatalIllegalStateErrors:-}
index.prefix=${org.apache.unomi.opensearch.index.prefix:-context}
-# Deprecated properties. Please use rollover corresponding properties
-monthlyIndex.numberOfShards=${org.apache.unomi.opensearch.monthlyIndex.nbShards:-5}
-monthlyIndex.numberOfReplicas=${org.apache.unomi.opensearch.monthlyIndex.nbReplicas:-0}
-monthlyIndex.indexMappingTotalFieldsLimit=${org.apache.unomi.opensearch.monthlyIndex.indexMappingTotalFieldsLimit:-1000}
-monthlyIndex.indexMaxDocValueFieldsSearch=${org.apache.unomi.opensearch.monthlyIndex.indexMaxDocValueFieldsSearch:-1000}
-monthlyIndex.itemsMonthlyIndexedOverride=${org.apache.unomi.opensearch.monthlyIndex.itemsMonthlyIndexedOverride:-event,session}
# New properties for index rotation:
-rollover.numberOfShards=${org.apache.unomi.opensearch.rollover.nbShards}
-rollover.numberOfReplicas=${org.apache.unomi.opensearch.rollover.nbReplicas}
-rollover.indexMappingTotalFieldsLimit=${org.apache.unomi.opensearch.rollover.indexMappingTotalFieldsLimit}
-rollover.indexMaxDocValueFieldsSearch=${org.apache.unomi.opensearch.rollover.indexMaxDocValueFieldsSearch}
-rollover.indices=${org.apache.unomi.opensearch.rollover.indices}
+rollover.numberOfShards=${org.apache.unomi.opensearch.rollover.nbShards:-5}
+rollover.numberOfReplicas=${org.apache.unomi.opensearch.rollover.nbReplicas:-0}
+rollover.indexMappingTotalFieldsLimit=${org.apache.unomi.opensearch.rollover.indexMappingTotalFieldsLimit:-1000}
+rollover.indexMaxDocValueFieldsSearch=${org.apache.unomi.opensearch.rollover.indexMaxDocValueFieldsSearch:-1000}
+rollover.indices=${org.apache.unomi.opensearch.rollover.indices:-event,session}
numberOfShards=${org.apache.unomi.opensearch.defaultIndex.nbShards:-5}
numberOfReplicas=${org.apache.unomi.opensearch.defaultIndex.nbReplicas:-0}
