This is an automated email from the ASF dual-hosted git repository.
sergehuber pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/unomi.git
The following commit(s) were added to refs/heads/master by this push:
new 148702fec UNOMI-1017: Sum interest points by key when profiles merge
(#963)
148702fec is described below
commit 148702fece5cb26a6b33d5851b46535adb4e4e44
Author: Serge Huber <[email protected]>
AuthorDate: Sat Nov 14 14:56:53 2026 +0100
UNOMI-1017: Sum interest points by key when profiles merge (#963)
Co-authored-by: Claude Opus 5.5 <[email protected]>
---
.../unomi/graphql/schema/PropertyFilterUtils.java | 2 +
.../org/apache/unomi/itests/ProfileMergeIT.java | 28 ++++
.../asciidoc/migrations/migrate-3.0-to-4.0.adoc | 2 +
manual/src/main/asciidoc/property-types.adoc | 3 +
.../SumByKeyPropertyMergeStrategyExecutor.java | 173 ++++++++++++++++++++
.../cxs/mergers/sumByKeyPropertyMergeStrategy.json | 4 +
.../cxs/properties/profiles/system/interests.json | 17 ++
.../resources/OSGI-INF/blueprint/blueprint.xml | 8 +
.../SumByKeyPropertyMergeStrategyExecutorTest.java | 178 +++++++++++++++++++++
.../4.0.0/shipped_system_definitions.json | 1 +
10 files changed, 416 insertions(+)
diff --git
a/graphql/cxs-impl/src/main/java/org/apache/unomi/graphql/schema/PropertyFilterUtils.java
b/graphql/cxs-impl/src/main/java/org/apache/unomi/graphql/schema/PropertyFilterUtils.java
index 49e0741c4..a3bd362b3 100644
---
a/graphql/cxs-impl/src/main/java/org/apache/unomi/graphql/schema/PropertyFilterUtils.java
+++
b/graphql/cxs-impl/src/main/java/org/apache/unomi/graphql/schema/PropertyFilterUtils.java
@@ -173,6 +173,8 @@ public class PropertyFilterUtils {
.type(inputObjectType)
.build());
}
+ } else if ("json".equals(propertyType.getTypeId())) {
+ // String filters cannot match a JSON value (e.g. the nested
properties.interests list), so none are generated.
} else if ("geopoint".equals(propertyType.getTypeId())) {
fieldDefinitions.add(GraphQLInputObjectField.newInputObjectField()
diff --git a/itests/src/test/java/org/apache/unomi/itests/ProfileMergeIT.java
b/itests/src/test/java/org/apache/unomi/itests/ProfileMergeIT.java
index f6952e67f..98703b176 100644
--- a/itests/src/test/java/org/apache/unomi/itests/ProfileMergeIT.java
+++ b/itests/src/test/java/org/apache/unomi/itests/ProfileMergeIT.java
@@ -535,6 +535,34 @@ public class ProfileMergeIT extends BaseIT {
assertPersonalizationStrategyStatus(profileMaster,
Arrays.asList(perso1true, perso2false, perso3true));
}
+ /**
+ * Goes through the real merge strategy lookup, so a mismatch between
interests.json, the merger
+ * definition and the blueprint service property would fall back to the
default strategy and fail here.
+ */
+ @Test
+ public void testInterestsAreSummedByKeyOnMerge() throws
InterruptedException {
+ keepTrying("interests property type should use sumByKeyMergeStrategy",
+ () -> profileService.getPropertyType("interests"),
+ propertyType -> propertyType != null &&
"sumByKeyMergeStrategy".equals(propertyType.getMergeStrategy()),
+ DEFAULT_TRYING_TIMEOUT, DEFAULT_TRYING_TRIES);
+
+ Profile master = new Profile("interests-master-profile");
+ master.setProperty("interests", new
ArrayList<>(Collections.singletonList(buildInterest("funds", 30))));
+ Profile other = new Profile("interests-other-profile");
+ other.setProperty("interests", new
ArrayList<>(Arrays.asList(buildInterest("funds", 5), buildInterest("pension",
7))));
+
+ master = profileService.mergeProfiles(master,
Collections.singletonList(other));
+
+ Assert.assertEquals(Arrays.asList(buildInterest("funds", 35),
buildInterest("pension", 7)), master.getProperty("interests"));
+ }
+
+ private Map<String, Object> buildInterest(String key, int value) {
+ Map<String, Object> interest = new LinkedHashMap<>();
+ interest.put("key", key);
+ interest.put("value", value);
+ return interest;
+ }
+
private Map<String, Object> buildPersonalizationStrategyStatus(String
persoId, boolean inControlGroup) {
Map<String, Object> personalizationStrategyStatus = new HashMap<>();
personalizationStrategyStatus.put(PERSONALIZATION_STRATEGY_STATUS_ID,
persoId);
diff --git a/manual/src/main/asciidoc/migrations/migrate-3.0-to-4.0.adoc
b/manual/src/main/asciidoc/migrations/migrate-3.0-to-4.0.adoc
index 40885bdc5..0e8cb3f16 100644
--- a/manual/src/main/asciidoc/migrations/migrate-3.0-to-4.0.adoc
+++ b/manual/src/main/asciidoc/migrations/migrate-3.0-to-4.0.adoc
@@ -564,6 +564,8 @@ The sections above cover authentication and client-facing
hardening. The lists b
==== Integrators / REST clients
* Creating a profile without an `itemId` is refused.
+* Merging profiles adds the points of `properties.interests` by key.
Previously the last profile's list replaced the others. Values that are not
numbers are kept unchanged. See <<_property_types,Property Types>>.
+* `interests` is now a built-in profile property type of type `json`, so
GraphQL lists it as an `interests` field on `CDP_Profile`, `CDP_Persona`,
`CDP_PersonaInput` and `CDP_ProfileUpdateEventInput`. GraphQL no longer
generates string filters (`_equals`, `_startsWith`, ...) for `json` properties,
since they could not match a JSON value.
* Condition validation failures on REST return a generic `badRequest`
response. Detailed messages stay in the server log. See
<<_condition_validation,Condition validation>>.
* Public endpoints need a tenant public API key unless
<<_v2_compatibility_mode,single-tenant compatibility mode>> is on (for clients
written for Unomi 2 or Unomi 3). See
<<_updating_applications_consuming_unomi,Updating applications consuming
Unomi>> and the client-facing hardening notes above.
* The Unomi web tracker 1.x cannot call a multi-tenant server. Use tracker
2.0.0 or later and set `apiKey`, or keep compatibility mode on. The script
served at `/tracker/` is updated with the server. An npm install or a copied
file is not. See <<_upgrading_the_web_tracker,Upgrading the tracker from 1.x to
2.0>>.
diff --git a/manual/src/main/asciidoc/property-types.adoc
b/manual/src/main/asciidoc/property-types.adoc
index 7d8316433..39910ac05 100644
--- a/manual/src/main/asciidoc/property-types.adoc
+++ b/manual/src/main/asciidoc/property-types.adoc
@@ -463,6 +463,9 @@ Common merge strategies include:
* `oldestMergeStrategy` - Use the oldest value (for dates and timestamps)
* `addMergeStrategy` - Add values together (for numeric types)
* `nonEmptyMergeStrategy` - Prefer non-empty values
+* `sumByKeyMergeStrategy` - For a list of `{key, value}` objects, add `value`
for entries that share a `key`. A `value` may be a number or a numeric string;
entries that cannot be summed are kept unchanged
+
+The built-in profile property `interests` uses `sumByKeyMergeStrategy`.
NOTE: The merge strategy identifier in property type JSON files should match
the `id` field from the merge strategy definition JSON files located in the
`META-INF/cxs/mergers/` directory (this is a file system path within plugin
bundles, not an API endpoint).
diff --git
a/services/src/main/java/org/apache/unomi/services/mergers/SumByKeyPropertyMergeStrategyExecutor.java
b/services/src/main/java/org/apache/unomi/services/mergers/SumByKeyPropertyMergeStrategyExecutor.java
new file mode 100644
index 000000000..5e972a4a4
--- /dev/null
+++
b/services/src/main/java/org/apache/unomi/services/mergers/SumByKeyPropertyMergeStrategyExecutor.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.unomi.services.mergers;
+
+import org.apache.unomi.api.Profile;
+import org.apache.unomi.api.PropertyMergeStrategyExecutor;
+import org.apache.unomi.api.PropertyType;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Adds {@code value} for entries that share a {@code key}.
+ * Used for {@code properties.interests}, a list of {@code {key, value}}
objects.
+ * A {@code value} may be a number or a numeric string. Entries that cannot be
summed are kept as they are.
+ * If the target already holds something other than a list, it is left
unchanged.
+ * If the target has no value and no list is merged in, the first other
non-null value is copied.
+ */
+public class SumByKeyPropertyMergeStrategyExecutor implements
PropertyMergeStrategyExecutor {
+
+ @Override
+ public boolean mergeProperty(String propertyName, PropertyType
propertyType, List<Profile> profilesToMerge, Profile targetProfile) {
+ Object current = targetProfile.getProperty(propertyName);
+ if (current != null && !(current instanceof List)) {
+ return false;
+ }
+
+ List<Object> merged = entries(current);
+ Object fallback = null;
+ boolean changed = false;
+ for (Profile profileToMerge : profilesToMerge) {
+ Object incoming = profileToMerge.getProperty(propertyName);
+ if (incoming == null) {
+ continue;
+ }
+ if (!(incoming instanceof List)) {
+ if (fallback == null) {
+ fallback = incoming;
+ }
+ continue;
+ }
+ changed |= addInto(merged, (List<?>) incoming);
+ }
+ if (!changed) {
+ if (current == null && fallback != null) {
+ targetProfile.setProperty(propertyName, fallback);
+ return true;
+ }
+ return false;
+ }
+ targetProfile.setProperty(propertyName, merged);
+ return true;
+ }
+
+ private static List<Object> entries(Object value) {
+ List<Object> entries = new ArrayList<>();
+ if (value instanceof List) {
+ addInto(entries, (List<?>) value);
+ }
+ return entries;
+ }
+
+ private static boolean addInto(List<Object> target, List<?> incoming) {
+ boolean changed = false;
+ for (Object item : incoming) {
+ Map<String, Object> entry = summable(item);
+ if (entry == null) {
+ if (item != null && !target.contains(item)) {
+ target.add(item);
+ changed = true;
+ }
+ continue;
+ }
+ Map<String, Object> existing = find(target, entry.get("key"));
+ if (existing == null) {
+ target.add(entry);
+ changed = true;
+ } else {
+ Number sum = add((Number) existing.get("value"), (Number)
entry.get("value"));
+ if (!sum.equals(existing.get("value"))) {
+ existing.put("value", sum);
+ changed = true;
+ }
+ }
+ }
+ return changed;
+ }
+
+ @SuppressWarnings("unchecked")
+ private static Map<String, Object> find(List<Object> target, Object key) {
+ for (Object candidate : target) {
+ if (candidate instanceof Map && key.equals(((Map<String, Object>)
candidate).get("key"))
+ && ((Map<String, Object>) candidate).get("value")
instanceof Number) {
+ return (Map<String, Object>) candidate;
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Returns a copy of {@code item} with a string key and a numeric value,
or null if it cannot be summed.
+ */
+ private static Map<String, Object> summable(Object item) {
+ if (!(item instanceof Map)) {
+ return null;
+ }
+ Map<?, ?> source = (Map<?, ?>) item;
+ Object key = source.get("key");
+ Number number = toNumber(source.get("value"));
+ if (key == null || number == null) {
+ return null;
+ }
+ Map<String, Object> copy = new LinkedHashMap<>();
+ for (Map.Entry<?, ?> field : source.entrySet()) {
+ if (field.getKey() != null) {
+ copy.put(field.getKey().toString(), field.getValue());
+ }
+ }
+ copy.put("key", key.toString());
+ copy.put("value", number);
+ return copy;
+ }
+
+ private static Number toNumber(Object value) {
+ if (value instanceof Number) {
+ return (Number) value;
+ }
+ if (!(value instanceof String)) {
+ return null;
+ }
+ String text = ((String) value).trim();
+ try {
+ long parsed = Long.parseLong(text);
+ return parsed >= Integer.MIN_VALUE && parsed <= Integer.MAX_VALUE
? (Number) (int) parsed : (Number) parsed;
+ } catch (NumberFormatException e) {
+ // not an integer, try a decimal below
+ }
+ try {
+ double parsed = Double.parseDouble(text);
+ return Double.isFinite(parsed) ? parsed : null;
+ } catch (NumberFormatException e) {
+ return null;
+ }
+ }
+
+ private static Number add(Number left, Number right) {
+ if (left instanceof Double || left instanceof Float || right
instanceof Double || right instanceof Float) {
+ return left.doubleValue() + right.doubleValue();
+ }
+ long sum = left.longValue() + right.longValue();
+ if (sum >= Integer.MIN_VALUE && sum <= Integer.MAX_VALUE) {
+ return (int) sum;
+ }
+ return sum;
+ }
+}
diff --git
a/services/src/main/resources/META-INF/cxs/mergers/sumByKeyPropertyMergeStrategy.json
b/services/src/main/resources/META-INF/cxs/mergers/sumByKeyPropertyMergeStrategy.json
new file mode 100644
index 000000000..f502ebf87
--- /dev/null
+++
b/services/src/main/resources/META-INF/cxs/mergers/sumByKeyPropertyMergeStrategy.json
@@ -0,0 +1,4 @@
+{
+ "id": "sumByKeyMergeStrategy",
+ "filter":
"(propertyMergeStrategyExecutorId=sumByKeyPropertyMergeStrategyExecutor)"
+}
diff --git
a/services/src/main/resources/META-INF/cxs/properties/profiles/system/interests.json
b/services/src/main/resources/META-INF/cxs/properties/profiles/system/interests.json
new file mode 100644
index 000000000..d53eeeda0
--- /dev/null
+++
b/services/src/main/resources/META-INF/cxs/properties/profiles/system/interests.json
@@ -0,0 +1,17 @@
+{
+ "metadata": {
+ "id": "interests",
+ "name": "Interests",
+ "systemTags": [
+ "properties",
+ "profileProperties",
+ "systemProfileProperties"
+ ]
+ },
+ "type": "json",
+ "defaultValue": "",
+ "automaticMappingsFrom": [ ],
+ "mergeStrategy": "sumByKeyMergeStrategy",
+ "multivalued": true,
+ "rank": "106.0"
+}
diff --git a/services/src/main/resources/OSGI-INF/blueprint/blueprint.xml
b/services/src/main/resources/OSGI-INF/blueprint/blueprint.xml
index e5d416fc2..76a75e49d 100644
--- a/services/src/main/resources/OSGI-INF/blueprint/blueprint.xml
+++ b/services/src/main/resources/OSGI-INF/blueprint/blueprint.xml
@@ -628,6 +628,14 @@
</bean>
</service>
+ <service interface="org.apache.unomi.api.PropertyMergeStrategyExecutor">
+ <service-properties>
+ <entry key="propertyMergeStrategyExecutorId"
value="sumByKeyPropertyMergeStrategyExecutor"/>
+ </service-properties>
+ <bean
class="org.apache.unomi.services.mergers.SumByKeyPropertyMergeStrategyExecutor">
+ </bean>
+ </service>
+
<service interface="org.apache.unomi.api.PersonalizationStrategy">
<service-properties>
<entry key="personalizationStrategyId" value="matching-first"/>
diff --git
a/services/src/test/java/org/apache/unomi/services/mergers/SumByKeyPropertyMergeStrategyExecutorTest.java
b/services/src/test/java/org/apache/unomi/services/mergers/SumByKeyPropertyMergeStrategyExecutorTest.java
new file mode 100644
index 000000000..04ae4b536
--- /dev/null
+++
b/services/src/test/java/org/apache/unomi/services/mergers/SumByKeyPropertyMergeStrategyExecutorTest.java
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.unomi.services.mergers;
+
+import org.apache.unomi.api.Profile;
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class SumByKeyPropertyMergeStrategyExecutorTest {
+
+ private final SumByKeyPropertyMergeStrategyExecutor executor = new
SumByKeyPropertyMergeStrategyExecutor();
+
+ @Test
+ public void
sumsPointsThatShareAKeyAndKeepsKeysThatAppearOnOnlyOneProfile() {
+ Profile master = profile(interests(entry("funds", 30)));
+ Profile other = profile(interests(entry("funds", 5), entry("pension",
7)));
+
+ assertTrue(executor.mergeProperty("interests", null,
Collections.singletonList(other), master));
+
+ assertEquals(interests(entry("funds", 35), entry("pension", 7)),
master.getProperty("interests"));
+ assertEquals(interests(entry("funds", 5), entry("pension", 7)),
other.getProperty("interests"));
+ }
+
+ @Test
+ public void copiesInterestsOntoAProfileThatHasNone() {
+ Profile master = profile(null);
+ Profile other = profile(interests(entry("funds", 5)));
+
+ assertTrue(executor.mergeProperty("interests", null,
Collections.singletonList(other), master));
+
+ assertEquals(interests(entry("funds", 5)),
master.getProperty("interests"));
+ }
+
+ @Test
+ public void leavesTheMasterUntouchedWhenNothingIsAdded() {
+ List<Map<String, Object>> original = interests(entry("funds", 30));
+ Profile master = profile(original);
+ Profile other = profile(null);
+
+ assertFalse(executor.mergeProperty("interests", null,
Collections.singletonList(other), master));
+ assertSame(original, master.getProperty("interests"));
+ }
+
+ @Test
+ public void sumsTheSameKeyAcrossSeveralProfiles() {
+ Profile master = profile(interests(entry("funds", 1)));
+ Profile first = profile(interests(entry("funds", 2)));
+ Profile second = profile(interests(entry("funds", 3)));
+
+ assertTrue(executor.mergeProperty("interests", null,
Arrays.asList(first, second), master));
+
+ assertEquals(interests(entry("funds", 6)),
master.getProperty("interests"));
+ }
+
+ @Test
+ public void sumsDuplicateKeysInsideOneList() {
+ Profile master = profile(interests(entry("funds", 1), entry("funds",
2)));
+ Profile other = profile(interests(entry("funds", 3)));
+
+ assertTrue(executor.mergeProperty("interests", null,
Collections.singletonList(other), master));
+
+ assertEquals(interests(entry("funds", 6)),
master.getProperty("interests"));
+ }
+
+ @Test
+ public void keepsAFractionalScoreAsADouble() {
+ Profile master = profile(interests(entry("funds", 1.5d)));
+ Profile other = profile(interests(entry("funds", 2)));
+
+ assertTrue(executor.mergeProperty("interests", null,
Collections.singletonList(other), master));
+
+ assertEquals(interests(entry("funds", 3.5d)),
master.getProperty("interests"));
+ }
+
+ @Test
+ public void leavesAValueThatIsNotAListUnchanged() {
+ Profile master = profile("funds");
+ Profile other = profile(interests(entry("funds", 5)));
+
+ assertFalse(executor.mergeProperty("interests", null,
Collections.singletonList(other), master));
+ assertEquals("funds", master.getProperty("interests"));
+ }
+
+ @Test
+ public void keepsEntriesThatCannotBeSummed() {
+ Map<String, Object> missingKey = new LinkedHashMap<>();
+ missingKey.put("value", 9);
+ Map<String, Object> missingNumber = new LinkedHashMap<>();
+ missingNumber.put("key", "pension");
+ missingNumber.put("value", "high");
+ Profile master = profile(interests(entry("funds", 1), missingNumber));
+ Profile other = profile(new ArrayList<>(Arrays.asList(missingKey,
missingNumber, entry("funds", 4))));
+
+ assertTrue(executor.mergeProperty("interests", null,
Collections.singletonList(other), master));
+
+ assertEquals(Arrays.asList(entry("funds", 5), missingNumber,
missingKey), master.getProperty("interests"));
+ }
+
+ @Test
+ public void sumsNumericStringValues() {
+ Map<String, Object> textValue = new LinkedHashMap<>();
+ textValue.put("key", "funds");
+ textValue.put("value", " 10 ");
+ Map<String, Object> decimalText = new LinkedHashMap<>();
+ decimalText.put("key", "pension");
+ decimalText.put("value", "0.5");
+ Profile master = profile(new ArrayList<>(Arrays.asList(textValue)));
+ Profile other = profile(new ArrayList<>(Arrays.asList(entry("funds",
5), decimalText)));
+
+ assertTrue(executor.mergeProperty("interests", null,
Collections.singletonList(other), master));
+
+ assertEquals(interests(entry("funds", 15), entry("pension", 0.5d)),
master.getProperty("interests"));
+ }
+
+ @Test
+ public void copiesANonListValueOntoAProfileThatHasNone() {
+ Profile master = profile(null);
+ Profile other = profile("funds");
+
+ assertTrue(executor.mergeProperty("interests", null,
Collections.singletonList(other), master));
+ assertEquals("funds", master.getProperty("interests"));
+ }
+
+ @Test
+ public void prefersAMergedListOverANonListValue() {
+ Profile master = profile(null);
+ Profile first = profile("funds");
+ Profile second = profile(interests(entry("pension", 7)));
+
+ assertTrue(executor.mergeProperty("interests", null,
Arrays.asList(first, second), master));
+ assertEquals(interests(entry("pension", 7)),
master.getProperty("interests"));
+ }
+
+ private static Profile profile(Object interests) {
+ Profile profile = new Profile("profile-" + System.nanoTime());
+ if (interests != null) {
+ profile.setProperty("interests", interests);
+ }
+ return profile;
+ }
+
+ @SafeVarargs
+ private static List<Map<String, Object>> interests(Map<String, Object>...
entries) {
+ return new ArrayList<>(Arrays.asList(entries));
+ }
+
+ private static Map<String, Object> entry(String key, Number value) {
+ Map<String, Object> entry = new LinkedHashMap<>();
+ entry.put("key", key);
+ entry.put("value", value);
+ return entry;
+ }
+}
diff --git
a/tools/shell-commands/src/main/resources/requestBody/4.0.0/shipped_system_definitions.json
b/tools/shell-commands/src/main/resources/requestBody/4.0.0/shipped_system_definitions.json
index 3d8cd07ad..e689a50c0 100644
---
a/tools/shell-commands/src/main/resources/requestBody/4.0.0/shipped_system_definitions.json
+++
b/tools/shell-commands/src/main/resources/requestBody/4.0.0/shipped_system_definitions.json
@@ -78,6 +78,7 @@
"propertytype:gender",
"propertytype:googleid",
"propertytype:income",
+ "propertytype:interests",
"propertytype:jobtitle",
"propertytype:kids",
"propertytype:lastname",