This is an automated email from the ASF dual-hosted git repository. dgriffon pushed a commit to branch fix-login-event-scope-migration in repository https://gitbox.apache.org/repos/asf/unomi.git
commit fc72f0fd350b580e3feb59ccf89f89672e04ef73 Author: david.griffon <[email protected]> AuthorDate: Mon Jun 3 08:17:38 2024 +0200 UNOMI-816 : ensure empty scope are filled properly at migration time --- .../unomi/itests/migration/Migrate16xTo220IT.java | 123 +++++++++++++-------- .../migration/match_all_login_event_request.json | 10 ++ .../resources/migration/snapshots_repository.zip | Bin 7807889 -> 35637487 bytes .../requestBody/2.0.0/event_migrate.painless | 2 +- 4 files changed, 88 insertions(+), 47 deletions(-) diff --git a/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo220IT.java b/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo220IT.java index 725f0fa4c..ed27daea6 100644 --- a/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo220IT.java +++ b/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xTo220IT.java @@ -19,6 +19,7 @@ package org.apache.unomi.itests.migration; import com.fasterxml.jackson.databind.JsonNode; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.unomi.api.*; +import org.apache.unomi.api.conditions.Condition; import org.apache.unomi.itests.BaseIT; import org.apache.unomi.persistence.spi.aggregate.TermsAggregate; import org.apache.unomi.shell.migration.utils.HttpUtils; @@ -29,16 +30,15 @@ import org.junit.Before; import org.junit.Test; import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Objects; +import java.util.*; public class Migrate16xTo220IT extends BaseIT { private int eventCount = 0; private int sessionCount = 0; + private Set<String[]> initialScopes = new HashSet<>(); + private static final String SCOPE_NOT_EXIST = "SCOPE_NOT_EXIST"; private static final int NUMBER_DUPLICATE_SESSIONS = 3; private static final List<String> oldSystemItemsIndices = Arrays.asList("context-actiontype", "context-campaign", "context-campaignevent", "context-goal", "context-userlist", "context-propertytype", "context-scope", "context-conditiontype", "context-rule", "context-scoring", "context-segment", "context-groovyaction", "context-topic", @@ -100,6 +100,7 @@ public class Migrate16xTo220IT extends BaseIT { checkIndexReductions2_2_0(); checkPagePathForEventView(); checkPastEvents(); + checkScopeEventHaveBeenUpdated(); } /** @@ -155,23 +156,23 @@ public class Migrate16xTo220IT extends BaseIT { /** * Data set contains a form event (id: 7b55b4fd-5ff0-4a85-9dc4-ffde322a1de6) with this data: * { - * "properties": { - * "pets": "cat", - * "firstname": "foo", - * "sports": [ - * "football", - * "tennis" - * ], - * "city": "Berlin", - * "age": "15", - * "email": "[email protected]", - * "drone": "dewey", - * "lastname": "bar", - * "contactMethod": [ - * "postalMethod", - * "phoneMethod" - * ] - * } + * "properties": { + * "pets": "cat", + * "firstname": "foo", + * "sports": [ + * "football", + * "tennis" + * ], + * "city": "Berlin", + * "age": "15", + * "email": "[email protected]", + * "drone": "dewey", + * "lastname": "bar", + * "contactMethod": [ + * "postalMethod", + * "phoneMethod" + * ] + * } * } */ private void checkFormEventRestructured() { @@ -269,6 +270,17 @@ public class Migrate16xTo220IT extends BaseIT { } } + private void checkScopeEventHaveBeenUpdated() { + for (String[] loginEvent : initialScopes) { + Event event = eventService.getEvent(loginEvent[0]); + if ("digitall".equals(loginEvent[1])) { + Assert.assertEquals(event.getScope(), "digitall"); + } else { + Assert.assertEquals(event.getScope(), "systemsite"); + } + } + } + /** * Data set contains a profile (id: e67ecc69-a7b3-47f1-b91f-5d6e7b90276e) with two interests: football:50 and basketball:30 * Also it's first name is test_profile @@ -316,6 +328,7 @@ public class Migrate16xTo220IT extends BaseIT { private void initCounts(CloseableHttpClient httpClient) { try { for (String eventIndex : MigrationUtils.getIndexesPrefixedBy(httpClient, "http://localhost:9400";, "context-event-date")) { + getScopeFromEvents(httpClient, eventIndex); eventCount += countItems(httpClient, eventIndex, resourceAsString("migration/must_not_match_some_eventype_body.json")); } @@ -327,33 +340,51 @@ public class Migrate16xTo220IT extends BaseIT { } } - private int countItems(CloseableHttpClient httpClient, String index, String requestBody) throws IOException { - if (requestBody == null) { - requestBody = resourceAsString("migration/must_not_match_some_eventype_body.json"); - } - JsonNode jsonNode = objectMapper.readTree(HttpUtils.executePostRequest(httpClient, "http://localhost:9400"; + "/" + index + "/_count", requestBody, null)); - return jsonNode.get("count").asInt(); + private void getScopeFromEvents(CloseableHttpClient httpClient, String eventIndex) throws IOException { + JsonNode jsonNode = objectMapper.readTree(HttpUtils.executeGetRequest(httpClient, "http://localhost:9400"; + "/" + eventIndex + "/_search", null)); + jsonNode.get("hits").get("hits").forEach(event -> { + if (event.get("_source").has("scope")) { + if (event.get("_source").get("scope") == null) { + String[] initialScope = {event.get("itemId").asText(), null}; + initialScopes.add(initialScope); + } else { + String[] initialScope = {event.get("itemId").asText(), event.get("_source").get("scope").asText()}; + initialScopes.add(initialScope); + } + } else { + String[] initialScope = {event.get("itemId").asText(), SCOPE_NOT_EXIST}; + initialScopes.add(initialScope); + } + }); } - /** - * Data set contains 2 events that had a value in properties.path: - * The properties.path should have been moved to properties.pageInfo.pagePath - */ - private void checkPagePathForEventView() { - Assert.assertEquals(2, persistenceService.query("target.properties.pageInfo.pagePath", "/path/to/migrate/to/pageInfo", null, Event.class).size()); - Assert.assertEquals(0, persistenceService.query("properties.path", "/path/to/migrate/to/pageInfo", null, Event.class).size()); - } + private int countItems (CloseableHttpClient httpClient, String index, String requestBody) throws IOException { + if (requestBody == null) { + requestBody = resourceAsString("migration/must_not_match_some_eventype_body.json"); + } + JsonNode jsonNode = objectMapper.readTree(HttpUtils.executePostRequest(httpClient, "http://localhost:9400"; + "/" + index + "/_count", requestBody, null)); + return jsonNode.get("count").asInt(); + } + + /** + * Data set contains 2 events that had a value in properties.path: + * The properties.path should have been moved to properties.pageInfo.pagePath + */ + private void checkPagePathForEventView () { + Assert.assertEquals(2, persistenceService.query("target.properties.pageInfo.pagePath", "/path/to/migrate/to/pageInfo", null, Event.class).size()); + Assert.assertEquals(0, persistenceService.query("properties.path", "/path/to/migrate/to/pageInfo", null, Event.class).size()); + } - /** - * Data set contains a profile (id: 164adad8-6885-45b6-8e9d-512bf4a7d10d) with a system property pastEvents that contains 5 events with key eventTriggeredabcdefgh - * This test ensures that the pastEvents have been migrated to the new data structure - */ - private void checkPastEvents() { - Profile profile = persistenceService.load("164adad8-6885-45b6-8e9d-512bf4a7d10d", Profile.class); - List<Map<String, Object>> pastEvents = ((List<Map<String, Object>>)profile.getSystemProperties().get("pastEvents")); - Assert.assertEquals(1, pastEvents.size()); - Assert.assertEquals("eventTriggeredabcdefgh", pastEvents.get(0).get("key")); - Assert.assertEquals(5, (int) pastEvents.get(0).get("count")); + /** + * Data set contains a profile (id: 164adad8-6885-45b6-8e9d-512bf4a7d10d) with a system property pastEvents that contains 5 events with key eventTriggeredabcdefgh + * This test ensures that the pastEvents have been migrated to the new data structure + */ + private void checkPastEvents () { + Profile profile = persistenceService.load("164adad8-6885-45b6-8e9d-512bf4a7d10d", Profile.class); + List<Map<String, Object>> pastEvents = ((List<Map<String, Object>>) profile.getSystemProperties().get("pastEvents")); + Assert.assertEquals(1, pastEvents.size()); + Assert.assertEquals("eventTriggeredabcdefgh", pastEvents.get(0).get("key")); + Assert.assertEquals(5, (int) pastEvents.get(0).get("count")); + } } -} diff --git a/itests/src/test/resources/migration/match_all_login_event_request.json b/itests/src/test/resources/migration/match_all_login_event_request.json new file mode 100644 index 000000000..24fb0bb38 --- /dev/null +++ b/itests/src/test/resources/migration/match_all_login_event_request.json @@ -0,0 +1,10 @@ +{ + "query": { + "term": { + "eventType": { + "value": "login", + "boost": 1.0 + } + } + } +} \ No newline at end of file diff --git a/itests/src/test/resources/migration/snapshots_repository.zip b/itests/src/test/resources/migration/snapshots_repository.zip index 675e710cc..9c3ee6146 100644 Binary files a/itests/src/test/resources/migration/snapshots_repository.zip and b/itests/src/test/resources/migration/snapshots_repository.zip differ diff --git a/tools/shell-commands/src/main/resources/requestBody/2.0.0/event_migrate.painless b/tools/shell-commands/src/main/resources/requestBody/2.0.0/event_migrate.painless index 9fd7e968a..04f4a58c0 100644 --- a/tools/shell-commands/src/main/resources/requestBody/2.0.0/event_migrate.painless +++ b/tools/shell-commands/src/main/resources/requestBody/2.0.0/event_migrate.painless @@ -18,7 +18,7 @@ /* Handle login events */ if ('login' == ctx._source.eventType) { /* Look for empty scope */ - if (ctx._source.scope == '') { + if (ctx._source.scope == '' || ctx._source.scope == null) { ctx._source.put('scope', 'systemsite'); if (ctx._source.source != null) { ctx._source.source.put('scope', 'systemsite');
