This is an automated email from the ASF dual-hosted git repository. shuber pushed a commit to branch unomi-3-dev in repository https://gitbox.apache.org/repos/asf/unomi.git
commit d22dce16c54021cb54ba8f7ade28231a6a1d056c Merge: 7891e7e30 0c44c9bd2 Author: Serge Huber <[email protected]> AuthorDate: Sat Jan 3 19:47:38 2026 +0100 Merge remote-tracking branch 'origin/master' into sync-master-2026Q1 .gitignore | 3 +- .vscode/settings.json | 3 + .../apache/unomi/api/services/EventService.java | 8 - docker/pom.xml | 1 + .../unomi/healthcheck/HealthCheckService.java | 26 +- .../resources/OSGI-INF/blueprint/blueprint.xml | 2 +- generate-manual-config.sh | 2 +- generate-manual.sh | 0 .../SegmentProfilePropertiesConditionParser.java | 2 +- .../types/output/CDPConsentUpdateEvent.java | 6 +- .../org/apache/unomi/graphql/utils/DateUtils.java | 24 - itests/README.md | 2 +- .../org/apache/unomi/itests/ContextServletIT.java | 2 +- .../apache/unomi/itests/graphql/BaseGraphQLIT.java | 1 + .../src/main/asciidoc/builtin-condition-types.adoc | 2 +- migration_es7-es9.sh | 0 .../main/resources/etc/custom.system.properties | 68 + .../resources/OSGI-INF/blueprint/blueprint.xml | 4 +- .../src/main/resources/mapping-FoldToASCII.txt | 3813 -------------------- persistence-elasticsearch/pom.xml | 4 +- .../unomi/persistence/spi/PersistenceService.java | 18 - .../persistence/spi/aggregate/DateAggregate.java | 2 +- .../persistence/spi/aggregate/TermsAggregate.java | 6 + .../SourceEventPropertyConditionEvaluator.java | 2 +- .../conditions/PropertyConditionEvaluatorTest.java | 1 - .../unomi/rest/endpoints/ScopeServiceEndPoint.java | 3 +- .../swagger-ui/5.27.1/swagger-initializer.js | 30 +- .../unomi/scripting/ExpressionFilterFactory.java | 2 +- .../internal/ExpressionFilterFactoryImpl.java | 2 +- .../services/impl/events/EventServiceImpl.java | 69 - .../main/resources/org.apache.unomi.cluster.cfg | 2 +- shell-utils.sh | 0 32 files changed, 123 insertions(+), 3987 deletions(-) diff --cc .gitignore index 85c793ac4,4de759d76..5749f0914 --- a/.gitignore +++ b/.gitignore @@@ -16,15 -16,6 +16,14 @@@ allCountries.zi rest/.miredot-offline.json /extensions/salesforce-connector/test.properties **/*.versionsBackup --itests/src/main --dependency_tree.txt ++itests/src/maindependency_tree.txt .mvn/.develocity/develocity-workspace-id +/.cursor/rules/always-read-existing-code.mdc +/.cursor/rules/avoid-code-duplication.mdc +/.cursor/rules/osgi-injection-frameworks.mdc +/services/data/ +/extensions/json-schema/services/data/ +# Ignore all data directories used by InMemoryPersistenceServiceImpl and integration tests +**/data/persistence/ +itests/snapshots_repository/ +.env.local diff --cc api/src/main/java/org/apache/unomi/api/services/EventService.java index d68a2c703,fb6d60d70..d662529c0 --- a/api/src/main/java/org/apache/unomi/api/services/EventService.java +++ b/api/src/main/java/org/apache/unomi/api/services/EventService.java @@@ -61,23 -61,23 +61,15 @@@ public interface EventService int send(Event event); /** - * Check if the sender is allowed to sent the speecified event. Restricted event must be explicitely allowed for a sender. + * Check if the tenant is allowed to send the specified event. Restricted events must be explicitly allowed for a tenant. * - * @param event event to test - * @param thirdPartyId third party id - * @return true if the event is allowed + * @param event event to test + * @param tenantId the ID of the tenant + * @param sourceIP the IP address from which the event was sent (not persisted for privacy) + * @return true if the event is allowed for the tenant */ - boolean isEventAllowed(Event event, String thirdPartyId); - - /** - * Get the third party server name, if the request is originated from a known peer - * - * @param key the key - * @param ip the ip - * @return server name - */ - String authenticateThirdPartyServer(String key, String ip); + boolean isEventAllowedForTenant(Event event, String tenantId, String sourceIP); - /** - * Retrieves the list of available event properties. - * - * @return a list of available event properties - * @deprecated use event types instead - */ - List<EventProperty> getEventProperties(); - /** * Retrieves the set of known event type identifiers. * diff --cc extensions/router/router-core/src/main/resources/OSGI-INF/blueprint/blueprint.xml index e1084a5d6,aae3abbe2..3afc58e5a --- a/extensions/router/router-core/src/main/resources/OSGI-INF/blueprint/blueprint.xml +++ b/extensions/router/router-core/src/main/resources/OSGI-INF/blueprint/blueprint.xml @@@ -136,9 -115,18 +136,9 @@@ <bean id="collectProfileBean" class="org.apache.unomi.router.core.bean.CollectProfileBean"> <property name="persistenceService" ref="persistenceService"/> + <property name="executionContextManager" ref="contextManager"/> </bean> - <reference id="configSharingService" interface="org.apache.unomi.api.services.ConfigSharingService" /> - <reference id="profileImportService" interface="org.apache.unomi.router.api.services.ProfileImportService"/> - <reference id="profileExportService" interface="org.apache.unomi.router.api.services.ProfileExportService"/> - <reference id="profileService" interface="org.apache.unomi.api.services.ProfileService"/> - <reference id="persistenceService" interface="org.apache.unomi.persistence.spi.PersistenceService"/> - <reference id="segmentService" interface="org.apache.unomi.api.services.SegmentService"/> - <reference id="importConfigurationService" interface="org.apache.unomi.router.api.services.ImportExportConfigurationService" filter="(configDiscriminator=IMPORT)"/> - <reference id="exportConfigurationService" interface="org.apache.unomi.router.api.services.ImportExportConfigurationService" filter="(configDiscriminator=EXPORT)"/> - <reference id="clusterService" interface="org.apache.unomi.api.services.ClusterService" /> - <reference id="osgiConfigurationAdmin" interface="org.osgi.service.cm.ConfigurationAdmin"/> + <service id="camelContextOSGI" ref="camelContext" interface="org.apache.unomi.router.api.IRouterCamelContext"/> --</blueprint> ++</blueprint> diff --cc generate-manual-config.sh index e02e5ef5e,37edfdd8b..171e9ffcb mode 100644,100755..100644 --- a/generate-manual-config.sh +++ b/generate-manual-config.sh diff --cc itests/README.md index 97f4c5745,3c4216660..4f77d3669 --- a/itests/README.md +++ b/itests/README.md @@@ -129,16 -98,8 +129,16 @@@ https://maven.apache.org/surefire/maven Here's an example: - mvn clean install -Dit.karaf.debug=hold:true -Dit.test=org.apache.unomi.itests.graphql.GraphQLEventIT + mvn clean install -Dit.karaf.debug=hold:true -Dit.test=org.apache.unomi.itests.BasicIT +To run a specific test method within a test class, you can use the # symbol followed by the method name: + + mvn clean install -Dit.test=org.apache.unomi.itests.ContextServletIT#testContextEndpointAuthentication + +You can also use patterns to run multiple methods that match a pattern: + + mvn clean install -Dit.test=org.apache.unomi.itests.ContextServletIT#test*Authentication* + ## Migration tests Migration can now be tested, by reusing an ElasticSearch snapshot. diff --cc itests/src/test/java/org/apache/unomi/itests/ContextServletIT.java index a3f10f79d,7014ec66e..16c112c89 --- a/itests/src/test/java/org/apache/unomi/itests/ContextServletIT.java +++ b/itests/src/test/java/org/apache/unomi/itests/ContextServletIT.java @@@ -960,10 -843,13 +960,10 @@@ public class ContextServletIT extends B HttpPost request = new HttpPost(getFullUrl(CONTEXT_URL)); request.setEntity(new StringEntity(objectMapper.writeValueAsString(contextRequest), ContentType.APPLICATION_JSON)); assertEquals(TestUtils.executeContextJSONRequest(request, sessionId).getContextResponse().getProfileProperties().get("customProperty"), ("concealedValue")); - // set the property as concealed + // set the property as concealed customPropertyType.getMetadata().getSystemTags().add("concealed"); profileService.deletePropertyType(customPropertyType.getItemId()); - persistenceService.refreshIndex(PropertyType.class); - Thread.sleep(2000); profileService.setPropertyType(customPropertyType); - // Not in all properties request.setEntity(new StringEntity(objectMapper.writeValueAsString(contextRequest), ContentType.APPLICATION_JSON)); assertNull(TestUtils.executeContextJSONRequest(request, sessionId).getContextResponse().getProfileProperties().get("customProperty")); diff --cc manual/src/main/asciidoc/builtin-condition-types.adoc index d108150ec,fc0352fe0..64a24b87f --- a/manual/src/main/asciidoc/builtin-condition-types.adoc +++ b/manual/src/main/asciidoc/builtin-condition-types.adoc @@@ -78,7 -78,7 +78,7 @@@ When implementing a new condition type <!-- OpenSearch Query Builder --> <service interface="org.apache.unomi.persistence.opensearch.ConditionOSQueryBuilder"> <service-properties> -- <entry key="queryBuilderId" value="booleanConditionOSQueryBuilder"/> ++ <entry key="queryBuilderId" value="booleanConditionQueryBuilder"/> </service-properties> <bean class="org.apache.unomi.persistence.opensearch.BooleanConditionOSQueryBuilder"/> </service> diff --cc package/src/main/resources/etc/custom.system.properties index 747d9dd79,7ca2c55bc..e4af942d5 --- a/package/src/main/resources/etc/custom.system.properties +++ b/package/src/main/resources/etc/custom.system.properties @@@ -151,26 -151,23 +151,94 @@@ org.apache.unomi.elasticsearch.password org.apache.unomi.elasticsearch.sslEnable=${env:UNOMI_ELASTICSEARCH_SSL_ENABLE:-false} org.apache.unomi.elasticsearch.sslTrustAllCertificates=${env:UNOMI_ELASTICSEARCH_SSL_TRUST_ALL_CERTIFICATES:-false} +# ES logging +org.apache.unomi.elasticsearch.logLevelRestClient=${env:UNOMI_ELASTICSEARCH_LOG_LEVEL_REST_CLIENT:-ERROR} + +####################################################################################################################### +## OpenSearch settings ## +####################################################################################################################### +org.apache.unomi.opensearch.cluster.name=${env:UNOMI_OPENSEARCH_CLUSTERNAME:-opensearch-cluster} +# The opensearch.addresses may be a comma seperated list of host names and ports such as +# hostA:9200,hostB:9200 +# Note: the port number must be repeated for each host. +org.apache.unomi.opensearch.addresses=${env:UNOMI_OPENSEARCH_ADDRESSES:-localhost:9200} +# refresh policy per item type in Json. +# Valid values are False/WaitFor/True (corresponding to NONE/WAIT_UNTIL/IMMEDIATE). The default refresh policy is False (NONE). +# Example: "{"event":"WaitFor","rule":"False"} +org.apache.unomi.opensearch.itemTypeToRefreshPolicy=${env:UNOMI_OPENSEARCH_REFRESH_POLICY_PER_ITEM_TYPE:-{"scheduledTask":"WaitFor"}} +org.apache.unomi.opensearch.fatalIllegalStateErrors=${env:UNOMI_OPENSEARCH_FATAL_STATE_ERRORS:-} +org.apache.unomi.opensearch.index.prefix=${env:UNOMI_OPENSEARCH_INDEXPREFIX:-context} + +# Rollover properties +org.apache.unomi.opensearch.rollover.nbShards=${env:UNOMI_OPENSEARCH_ROLLOVER_SHARDS:-5} ++org.apache.unomi.opensearch.rollover.nbReplicas=${env:UNOMI_OPENSEARCH_ROLLOVER_REPLICAS:-0} ++org.apache.unomi.opensearch.rollover.indexMappingTotalFieldsLimit=${env:UNOMI_OPENSEARCH_ROLLOVER_MAPPINGTOTALFIELDSLIMIT:-1000} ++org.apache.unomi.opensearch.rollover.indexMaxDocValueFieldsSearch=${env:UNOMI_OPENSEARCH_ROLLOVER_MAXDOCVALUEFIELDSSEARCH:-1000} ++org.apache.unomi.opensearch.rollover.indices=${env:UNOMI_OPENSEARCH_ROLLOVER_INDICES:-event,session} ++ ++# Rollover configuration ++org.apache.unomi.opensearch.rollover.maxSize=${env:UNOMI_OPENSEARCH_ROLLOVER_MAXSIZE:-30gb} ++org.apache.unomi.opensearch.rollover.maxAge=${env:UNOMI_OPENSEARCH_ROLLOVER_MAXAGE:-} ++org.apache.unomi.opensearch.rollover.maxDocs=${env:UNOMI_OPENSEARCH_ROLLOVER_MAXDOCS:-} ++ ++org.apache.unomi.opensearch.defaultIndex.nbShards=${env:UNOMI_OPENSEARCH_DEFAULTINDEX_SHARDS:-5} ++org.apache.unomi.opensearch.defaultIndex.nbReplicas=${env:UNOMI_OPENSEARCH_DEFAULTINDEX_REPLICAS:-0} ++org.apache.unomi.opensearch.defaultIndex.indexMappingTotalFieldsLimit=${env:UNOMI_OPENSEARCH_DEFAULTINDEX_MAPPINGTOTALFIELDSLIMIT:-1000} ++org.apache.unomi.opensearch.defaultIndex.indexMaxDocValueFieldsSearch=${env:UNOMI_OPENSEARCH_DEFAULTINDEX_MAXDOCVALUEFIELDSSEARCH:-1000} ++org.apache.unomi.opensearch.defaultQueryLimit=${env:UNOMI_OPENSEARCH_DEFAULTQUERYLIMIT:-10} ++org.apache.unomi.opensearch.aggregateQueryBucketSize=${env:UNOMI_OPENSEARCH_AGGREGATEBUCKETSIZE:-5000} ++org.apache.unomi.opensearch.maximumIdsQueryCount=${env:UNOMI_OPENSEARCH_MAXIMUMIDSQUERYCOUNT:-5000} ++# Defines the socket timeout (SO_TIMEOUT) in milliseconds, which is the timeout for waiting for data or, put differently, a maximum period inactivity between two consecutive data packets). ++# A timeout value of zero is interpreted as an infinite timeout. A negative value is interpreted as undefined (system default). ++# Default: -1 (System default) ++org.apache.unomi.opensearch.clientSocketTimeout=${env:UNOMI_OPENSEARCH_CLIENT_SOCKET_TIMEOUT:--1} ++# Defines the waiting for task completion timeout in milliseconds. ++# Some operations like update_by_query and delete_by_query are delegated to ElasticSearch using tasks ++# For consistency the thread that trigger one of those operations will wait for the task to be completed on ElasticSearch side. ++# This timeout configuration is here to ensure not blocking the thread infinitely, in case of very long running tasks. ++# A timeout value of zero or negative is interpreted as an infinite timeout. ++# Default: 3600000 (one hour) ++org.apache.unomi.opensearch.taskWaitingTimeout=${env:UNOMI_OPENSEARCH_TASK_WAITING_TIMEOUT:-3600000} ++# Defines the polling interval in milliseconds, which is used to check if task is completed on ElasticSearch side ++# Default: 1000 (1 second) ++org.apache.unomi.opensearch.taskWaitingPollingInterval=${env:UNOMI_OPENSEARCH_TASK_WAITING_POLLING_INTERVAL:-1000} ++org.apache.unomi.opensearch.pastEventsDisablePartitions=${env:UNOMI_OPENSEARCH_PAST_EVENTS_DISABLE_PARTITIONS:-false} ++org.apache.unomi.opensearch.aggQueryThrowOnMissingDocs=${env:UNOMI_OPENSEARCH_AGG_QUERY_THROW_ON_MISSING_DOCS:-false} ++org.apache.unomi.opensearch.aggQueryMaxResponseSizeHttp=${env:UNOMI_OPENSEARCH_AGG_QUERY_MAX_RESPONSE_SIZE_HTTP:-} ++# The following settings control the behavior of the BulkProcessor API. You can find more information about these ++# settings and their behavior here : https://www.elastic.co/guide/en/elasticsearch/client/java-api/2.4/java-docs-bulk-processor.html ++# The values used here are the default values of the API ++org.apache.unomi.opensearch.bulkProcessor.concurrentRequests=${env:UNOMI_OPENSEARCH_BULK_CONCURRENTREQUESTS:-1} ++org.apache.unomi.opensearch.bulkProcessor.bulkActions=${env:UNOMI_OPENSEARCH_BULK_ACTIONS:-1000} ++org.apache.unomi.opensearch.bulkProcessor.bulkSize=${env:UNOMI_OPENSEARCH_BULK_SIZE:-5MB} ++org.apache.unomi.opensearch.bulkProcessor.flushInterval=${env:UNOMI_OPENSEARCH_BULK_FLUSHINTERVAL:-5s} ++org.apache.unomi.opensearch.bulkProcessor.backoffPolicy=${env:UNOMI_OPENSEARCH_BULK_BACKOFFPOLICY:-exponential} ++# Errors ++org.apache.unomi.opensearch.throwExceptions=${env:UNOMI_OPENSEARCH_THROW_EXCEPTIONS:-false} ++# Authentication ++org.apache.unomi.opensearch.username=${env:UNOMI_OPENSEARCH_USERNAME:-admin} ++org.apache.unomi.opensearch.password=${env:UNOMI_OPENSEARCH_PASSWORD:-} ++org.apache.unomi.opensearch.sslEnable=${env:UNOMI_OPENSEARCH_SSL_ENABLE:-true} ++org.apache.unomi.opensearch.sslTrustAllCertificates=${env:UNOMI_OPENSEARCH_SSL_TRUST_ALL_CERTIFICATES:-true} ++org.apache.unomi.opensearch.minimalClusterState=${env:UNOMI_OPENSEARCH_MINIMAL_CLUSTER_STATE:-GREEN} ++ + ####################################################################################################################### + ## OpenSearch settings ## + ####################################################################################################################### + org.apache.unomi.opensearch.cluster.name=${env:UNOMI_OPENSEARCH_CLUSTERNAME:-opensearch-cluster} + # The opensearch.addresses may be a comma seperated list of host names and ports such as + # hostA:9200,hostB:9200 + # Note: the port number must be repeated for each host. + org.apache.unomi.opensearch.addresses=${env:UNOMI_OPENSEARCH_ADDRESSES:-localhost:9200} + # refresh policy per item type in Json. + # Valid values are WAIT_UNTIL/IMMEDIATE/NONE. The default refresh policy is NONE. + # Example: "{"event":"WAIT_UNTIL","rule":"NONE"} + org.apache.unomi.opensearch.itemTypeToRefreshPolicy=${env:UNOMI_OPENSEARCH_REFRESH_POLICY_PER_ITEM_TYPE:-} + org.apache.unomi.opensearch.fatalIllegalStateErrors=${env:UNOMI_OPENSEARCH_FATAL_STATE_ERRORS:-} + org.apache.unomi.opensearch.index.prefix=${env:UNOMI_OPENSEARCH_INDEXPREFIX:-context} + + # Rollover properties + org.apache.unomi.opensearch.rollover.nbShards=${env:UNOMI_OPENSEARCH_ROLLOVER_SHARDS:-5} org.apache.unomi.opensearch.rollover.nbReplicas=${env:UNOMI_OPENSEARCH_ROLLOVER_REPLICAS:-0} org.apache.unomi.opensearch.rollover.indexMappingTotalFieldsLimit=${env:UNOMI_OPENSEARCH_ROLLOVER_MAPPINGTOTALFIELDSLIMIT:-1000} org.apache.unomi.opensearch.rollover.indexMaxDocValueFieldsSearch=${env:UNOMI_OPENSEARCH_ROLLOVER_MAXDOCVALUEFIELDSSEARCH:-1000} diff --cc persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml index c645b0774,77ebd264b..e575411c6 --- a/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml +++ b/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml @@@ -73,8 -73,10 +73,8 @@@ </cm:default-properties> </cm:property-placeholder> - <reference id="metricsService" interface="org.apache.unomi.metrics.MetricsService" /> + <reference id="metricsService" interface="org.apache.unomi.metrics.MetricsService"/> - <reference id="scriptExecutor" interface="org.apache.unomi.scripting.ScriptExecutor"/> - <reference id="conditionEvaluatorDispatcherService" - interface="org.apache.unomi.persistence.spi.conditions.evaluator.ConditionEvaluatorDispatcher"/> + <reference id="scriptExecutor" interface="org.apache.unomi.scripting.ScriptExecutor" /> <service id="elasticSearchPersistenceService" ref="elasticSearchPersistenceServiceImpl"> <interfaces> @@@ -89,21 -87,9 +89,21 @@@ <bean id="conditionESQueryBuilderDispatcher" class="org.apache.unomi.persistence.elasticsearch.ConditionESQueryBuilderDispatcher"> - <property name="scriptExecutor" ref="scriptExecutor" /> + <property name="scriptExecutor" ref="scriptExecutor"/> </bean> + <!-- DefinitionsService Reference with listener to avoid blueprint context restart --> + <reference id="definitionsService" + interface="org.apache.unomi.api.services.DefinitionsService" + availability="optional"> + <reference-listener bind-method="bindDefinitionsService" + unbind-method="unbindDefinitionsService"> + <ref component-id="conditionESQueryBuilderDispatcher"/> + </reference-listener> + </reference> + + <reference id="conditionEvaluatorDispatcher" interface="org.apache.unomi.persistence.spi.conditions.evaluator.ConditionEvaluatorDispatcher" /> + <bean id="elasticSearchPersistenceServiceImpl" class="org.apache.unomi.persistence.elasticsearch.ElasticSearchPersistenceServiceImpl" init-method="start" diff --cc plugins/advanced-conditions/src/main/java/org/apache/unomi/plugins/advancedconditions/conditions/SourceEventPropertyConditionEvaluator.java index f0cb5f360,aeeebb78c..56618dc89 --- a/plugins/advanced-conditions/src/main/java/org/apache/unomi/plugins/advancedconditions/conditions/SourceEventPropertyConditionEvaluator.java +++ b/plugins/advanced-conditions/src/main/java/org/apache/unomi/plugins/advancedconditions/conditions/SourceEventPropertyConditionEvaluator.java @@@ -89,4 -89,4 +89,4 @@@ public class SourceEventPropertyConditi public void setDefinitionsService(DefinitionsService definitionsService) { this.definitionsService = definitionsService; } --} ++} diff --cc services/src/main/resources/org.apache.unomi.cluster.cfg index 44fd720db,eecb7e1de..744d383fe --- a/services/src/main/resources/org.apache.unomi.cluster.cfg +++ b/services/src/main/resources/org.apache.unomi.cluster.cfg @@@ -28,4 -28,4 +28,4 @@@ nodeId=${org.apache.unomi.cluster.nodeI # system load average and uptime. This value is set in milliseconds and is set to 10 seconds by default. Each node # will retrieve the local values and broadcast them through a cluster event to all the other nodes to update # the global cluster statistics. --nodeStatisticsUpdateFrequency=${org.apache.unomi.cluster.nodeStatisticsUpdateFrequency:-10000} ++nodeStatisticsUpdateFrequency=${org.apache.unomi.cluster.nodeStatisticsUpdateFrequency:-10000}
