This is an automated email from the ASF dual-hosted git repository. jkevan pushed a commit to branch eventFlattenedProperties in repository https://gitbox.apache.org/repos/asf/unomi.git
commit 4ba96755ac120a7fc2f9efe735bc93cdeb9c9070 Author: Kevan <[email protected]> AuthorDate: Tue Jun 7 17:11:32 2022 +0200 UNOMI-584: introduce new flattenedProperties for events, to allow mapping free event data to be stored in ElasticSearch --- api/src/main/java/org/apache/unomi/api/Event.java | 20 ++++- itests/pom.xml | 9 +- .../java/org/apache/unomi/itests/JSONSchemaIT.java | 96 ++++++++++++++++++++-- .../schemas/event-flattened-invalid-1.json | 14 ++++ .../schemas/event-flattened-invalid-2.json | 16 ++++ .../schemas/event-flattened-invalid-3.json | 13 +++ .../resources/schemas/event-flattened-valid.json | 13 +++ .../src/test/resources/schemas/event-request.json | 6 ++ ...ma-flattened-flattenedProperties-interests.json | 19 +++++ .../schema-flattened-flattenedProperties.json | 18 ++++ .../schemas/schema-flattened-properties.json | 18 ++++ .../test/resources/schemas/schema-flattened.json | 25 ++++++ persistence-elasticsearch/core/pom.xml | 7 ++ .../resources/META-INF/cxs/mappings/event.json | 3 + pom.xml | 1 + .../rest/service/impl/RestServiceUtilsImpl.java | 2 + 16 files changed, 269 insertions(+), 11 deletions(-) diff --git a/api/src/main/java/org/apache/unomi/api/Event.java b/api/src/main/java/org/apache/unomi/api/Event.java index 45a129609..3d418257c 100644 --- a/api/src/main/java/org/apache/unomi/api/Event.java +++ b/api/src/main/java/org/apache/unomi/api/Event.java @@ -58,6 +58,7 @@ public class Event extends Item implements TimestampedItem { private String profileId = null; private Date timeStamp; private Map<String, Object> properties; + private Map<String, Object> flattenedProperties; private transient Profile profile; private transient Session session; @@ -166,7 +167,8 @@ public class Event extends Item implements TimestampedItem { } this.timeStamp = timestamp; - this.properties = new HashMap<String, Object>(); + this.properties = new HashMap<>(); + this.flattenedProperties = new HashMap<>(); actionPostExecutors = new ArrayList<>(); } @@ -376,6 +378,22 @@ public class Event extends Item implements TimestampedItem { this.properties = properties; } + /** + * Retrieves the flattened properties + * @return the flattened properties. + */ + public Map<String, Object> getFlattenedProperties() { + return flattenedProperties; + } + + /** + * Set the flattened properties for current event + * @param flattenedProperties the properties + */ + public void setFlattenedProperties(Map<String, Object> flattenedProperties) { + this.flattenedProperties = flattenedProperties; + } + /** * @return the scope */ diff --git a/itests/pom.xml b/itests/pom.xml index 1de0f6384..b65bce832 100644 --- a/itests/pom.xml +++ b/itests/pom.xml @@ -200,12 +200,12 @@ <groupId>com.github.alexcojocaru</groupId> <artifactId>elasticsearch-maven-plugin</artifactId> <!-- REPLACE THE FOLLOWING WITH THE PLUGIN VERSION YOU NEED --> - <version>6.19</version> + <version>6.23</version> <configuration> <clusterName>contextElasticSearchITests</clusterName> <transportPort>9500</transportPort> <httpPort>9400</httpPort> - <version>${elasticsearch.version}</version> + <version>${elasticsearch.test.version}</version> <autoCreateIndex>true</autoCreateIndex> <timeout>120</timeout> <environmentVariables> @@ -213,9 +213,10 @@ </environmentVariables> <instanceSettings> <properties> - <cluster.routing.allocation.disk.threshold_enabled>false - </cluster.routing.allocation.disk.threshold_enabled> + <cluster.routing.allocation.disk.threshold_enabled>false</cluster.routing.allocation.disk.threshold_enabled> <http.cors.allow-origin>*</http.cors.allow-origin> + <http.cors.allow-methods>OPTIONS,HEAD,GET,POST,PUT,DELETE</http.cors.allow-methods> + <http.cors.allow-headers>Authorization,X-Requested-With,X-Auth-Token,Content-Type,Content-Length</http.cors.allow-headers> </properties> </instanceSettings> </configuration> diff --git a/itests/src/test/java/org/apache/unomi/itests/JSONSchemaIT.java b/itests/src/test/java/org/apache/unomi/itests/JSONSchemaIT.java index 2600a8912..5640f211e 100644 --- a/itests/src/test/java/org/apache/unomi/itests/JSONSchemaIT.java +++ b/itests/src/test/java/org/apache/unomi/itests/JSONSchemaIT.java @@ -23,6 +23,8 @@ import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; import org.apache.http.util.EntityUtils; import org.apache.unomi.api.Event; +import org.apache.unomi.api.conditions.Condition; +import org.apache.unomi.itests.tools.httpclient.HttpClientThatWaitsForUnomi; import org.apache.unomi.schema.api.JsonSchemaWrapper; import org.apache.unomi.schema.api.SchemaService; import org.junit.After; @@ -38,13 +40,9 @@ import org.slf4j.LoggerFactory; import javax.inject.Inject; import java.io.IOException; -import java.util.List; -import java.util.Objects; +import java.util.*; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; /** * Class to tests the JSON schema features @@ -54,6 +52,7 @@ import static org.junit.Assert.assertTrue; @ExamReactorStrategy(PerSuite.class) public class JSONSchemaIT extends BaseIT { private final static Logger LOGGER = LoggerFactory.getLogger(JSONSchemaIT.class); + private final static String EVENT_COLLECTOR_URL = "/cxs/eventcollector"; private final static String JSONSCHEMA_URL = "/cxs/jsonSchema"; private static final int DEFAULT_TRYING_TIMEOUT = 2000; private static final int DEFAULT_TRYING_TRIES = 30; @@ -246,6 +245,53 @@ public class JSONSchemaIT extends BaseIT { DEFAULT_TRYING_TRIES); } + @Test + public void testFlattenedProperties() throws Exception { + assertNull(schemaService.getSchema("https://vendor.test.com/schemas/json/events/flattened/1-0-0")); + assertNull(schemaService.getSchema("https://vendor.test.com/schemas/json/events/flattened/properties/1-0-0")); + assertNull(schemaService.getSchema("https://vendor.test.com/schemas/json/events/flattened/properties/interests/1-0-0")); + + // Test that at first the flattened event is not valid. + assertFalse(schemaService.isEventValid(resourceAsString("schemas/event-flattened-valid.json"), "flattened")); + + // save schemas and check the event pass the validation + schemaService.saveSchema(resourceAsString("schemas/schema-flattened.json")); + schemaService.saveSchema(resourceAsString("schemas/schema-flattened-flattenedProperties.json")); + schemaService.saveSchema(resourceAsString("schemas/schema-flattened-flattenedProperties-interests.json")); + schemaService.saveSchema(resourceAsString("schemas/schema-flattened-properties.json")); + keepTrying("Event should be valid now", + () -> schemaService.isEventValid(resourceAsString("schemas/event-flattened-valid.json"), "flattened"), + isValid -> isValid, DEFAULT_TRYING_TIMEOUT, DEFAULT_TRYING_TRIES); + + // insure event is correctly indexed when send to /cxs/eventCollector + Event event = sendEventAndWaitItsIndexed("schemas/event-flattened-valid.json"); + Map<String, Integer> interests = (Map<String, Integer>) event.getFlattenedProperties().get("interests"); + assertEquals(15, interests.get("cars").intValue()); + assertEquals(59, interests.get("football").intValue()); + assertEquals(2, interests.size()); + + // check that range query is not working on flattened props: + Condition condition = new Condition(definitionsService.getConditionType("eventPropertyCondition")); + condition.setParameter("propertyName","flattenedProperties.interests.cars"); + condition.setParameter("comparisonOperator","greaterThan"); + condition.setParameter("propertyValueInteger", 2); + assertNull(persistenceService.query(condition, null, Event.class, 0, -1)); + + // check that term query is working on flattened props: + condition = new Condition(definitionsService.getConditionType("eventPropertyCondition")); + condition.setParameter("propertyName","flattenedProperties.interests.cars"); + condition.setParameter("comparisonOperator","equals"); + condition.setParameter("propertyValueInteger", 15); + List<Event> events = persistenceService.query(condition, null, Event.class, 0, -1).getList(); + assertEquals(1, events.size()); + assertEquals(events.get(0).getItemId(), event.getItemId()); + + // Bonus: Check that other invalid flattened events are correctly rejected by schema service: + assertFalse(schemaService.isEventValid(resourceAsString("schemas/event-flattened-invalid-1.json"), "flattened")); + assertFalse(schemaService.isEventValid(resourceAsString("schemas/event-flattened-invalid-2.json"), "flattened")); + assertFalse(schemaService.isEventValid(resourceAsString("schemas/event-flattened-invalid-3.json"), "flattened")); + } + @Test public void testSaveFail_PredefinedJSONSchema() throws IOException { try (CloseableHttpResponse response = post(JSONSCHEMA_URL, "schemas/schema-predefined.json", ContentType.TEXT_PLAIN)) { @@ -266,4 +312,42 @@ public class JSONSchemaIT extends BaseIT { assertEquals("Unable to save schema", 400, response.getStatusLine().getStatusCode()); } } + + private Event sendEventAndWaitItsIndexed(String eventResourcePath) throws IOException, InterruptedException { + // build event collector request + String eventMarker = UUID.randomUUID().toString(); + HashMap<String, String> eventReplacements = new HashMap<>(); + eventReplacements.put("EVENT_MARKER", eventMarker); + String event = getValidatedBundleJSON(eventResourcePath, eventReplacements); + HashMap<String, String> eventRequestReplacements = new HashMap<>(); + eventRequestReplacements.put("EVENTS", event); + String eventRequest = getValidatedBundleJSON("schemas/event-request.json", eventRequestReplacements); + + // send event + eventCollectorPost(eventRequest); + + // wait for the event to be indexed + Condition condition = new Condition(definitionsService.getConditionType("eventPropertyCondition")); + condition.setParameter("propertyName","properties.marker.keyword"); + condition.setParameter("comparisonOperator","equals"); + condition.setParameter("propertyValue", eventMarker); + List<Event> events = keepTrying("The event should have been persisted", + () -> persistenceService.query(condition, null, Event.class), results -> results.size() == 1, + DEFAULT_TRYING_TIMEOUT, DEFAULT_TRYING_TRIES); + return events.get(0); + } + + private void eventCollectorPost(String eventCollectorRequest) { + HttpPost request = new HttpPost(URL + EVENT_COLLECTOR_URL); + request.addHeader("Content-Type", "application/json"); + request.setEntity(new StringEntity(eventCollectorRequest, ContentType.create("application/json"))); + CloseableHttpResponse response; + try { + response = HttpClientThatWaitsForUnomi.doRequest(request, 200); + } catch (Exception e) { + fail("Something went wrong with the request to Unomi that is unexpected: " + e.getMessage()); + return; + } + assertEquals("Invalid response code", 200, response.getStatusLine().getStatusCode()); + } } diff --git a/itests/src/test/resources/schemas/event-flattened-invalid-1.json b/itests/src/test/resources/schemas/event-flattened-invalid-1.json new file mode 100644 index 000000000..ffc6eaa8c --- /dev/null +++ b/itests/src/test/resources/schemas/event-flattened-invalid-1.json @@ -0,0 +1,14 @@ +{ + "eventType":"flattened", + "scope":"scope", + "flattenedProperties": { + "interests": { + "cars": 15, + "football": 59 + }, + "notAllowedProp": "notAllowedProp" + }, + "properties": { + "marker": "###EVENT_MARKER###" + } +} \ No newline at end of file diff --git a/itests/src/test/resources/schemas/event-flattened-invalid-2.json b/itests/src/test/resources/schemas/event-flattened-invalid-2.json new file mode 100644 index 000000000..e062a1de6 --- /dev/null +++ b/itests/src/test/resources/schemas/event-flattened-invalid-2.json @@ -0,0 +1,16 @@ +{ + "eventType":"flattened", + "scope":"scope", + "flattenedProperties": { + "interests": { + "cars": 15, + "football": 59, + "tennis": 42, + "fishing": 8, + "too_much_interests": 9 + } + }, + "properties": { + "marker": "###EVENT_MARKER###" + } +} \ No newline at end of file diff --git a/itests/src/test/resources/schemas/event-flattened-invalid-3.json b/itests/src/test/resources/schemas/event-flattened-invalid-3.json new file mode 100644 index 000000000..a49597982 --- /dev/null +++ b/itests/src/test/resources/schemas/event-flattened-invalid-3.json @@ -0,0 +1,13 @@ +{ + "eventType":"flattened", + "scope":"scope", + "flattenedProperties": { + "interests": { + "cars": 15, + "football": "not authorized value" + } + }, + "properties": { + "marker": "###EVENT_MARKER###" + } +} \ No newline at end of file diff --git a/itests/src/test/resources/schemas/event-flattened-valid.json b/itests/src/test/resources/schemas/event-flattened-valid.json new file mode 100644 index 000000000..0a9a80276 --- /dev/null +++ b/itests/src/test/resources/schemas/event-flattened-valid.json @@ -0,0 +1,13 @@ +{ + "eventType":"flattened", + "scope":"scope", + "flattenedProperties": { + "interests": { + "cars": 15, + "football": 59 + } + }, + "properties": { + "marker": "###EVENT_MARKER###" + } +} \ No newline at end of file diff --git a/itests/src/test/resources/schemas/event-request.json b/itests/src/test/resources/schemas/event-request.json new file mode 100644 index 000000000..d6a8c6f42 --- /dev/null +++ b/itests/src/test/resources/schemas/event-request.json @@ -0,0 +1,6 @@ +{ + "sessionId": "dummy-session-id", + "events":[ + ###EVENTS### + ] +} \ No newline at end of file diff --git a/itests/src/test/resources/schemas/schema-flattened-flattenedProperties-interests.json b/itests/src/test/resources/schemas/schema-flattened-flattenedProperties-interests.json new file mode 100644 index 000000000..a5fde38fc --- /dev/null +++ b/itests/src/test/resources/schemas/schema-flattened-flattenedProperties-interests.json @@ -0,0 +1,19 @@ +{ + "$id": "https://vendor.test.com/schemas/json/events/flattened/flattenedProperties/interests/1-0-0", + "$schema": "https://json-schema.org/draft/2019-09/schema", + "self": { + "vendor": "com.vendor.test", + "name": "flattenedPropertiesInterests", + "format": "jsonschema", + "version": "1-0-0" + }, + "title": "Flattened Properties interests", + "type": "object", + "patternProperties": { + ".": { + "type": "number" + } + }, + "maxProperties": 4, + "unevaluatedProperties": false +} \ No newline at end of file diff --git a/itests/src/test/resources/schemas/schema-flattened-flattenedProperties.json b/itests/src/test/resources/schemas/schema-flattened-flattenedProperties.json new file mode 100644 index 000000000..e50e883f2 --- /dev/null +++ b/itests/src/test/resources/schemas/schema-flattened-flattenedProperties.json @@ -0,0 +1,18 @@ +{ + "$id": "https://vendor.test.com/schemas/json/events/flattened/flattenedProperties/1-0-0", + "$schema": "https://json-schema.org/draft/2019-09/schema", + "self": { + "vendor": "com.vendor.test", + "name": "flattenedProperties", + "format": "jsonschema", + "version": "1-0-0" + }, + "title": "Flattened Properties", + "type": "object", + "properties": { + "interests": { + "$ref": "https://vendor.test.com/schemas/json/events/flattened/flattenedProperties/interests/1-0-0" + } + }, + "unevaluatedProperties": false +} \ No newline at end of file diff --git a/itests/src/test/resources/schemas/schema-flattened-properties.json b/itests/src/test/resources/schemas/schema-flattened-properties.json new file mode 100644 index 000000000..7d7fd7c0d --- /dev/null +++ b/itests/src/test/resources/schemas/schema-flattened-properties.json @@ -0,0 +1,18 @@ +{ + "$id": "https://vendor.test.com/schemas/json/events/flattened/properties/1-0-0", + "$schema": "https://json-schema.org/draft/2019-09/schema", + "self": { + "vendor": "com.vendor.test", + "name": "flattenedProperties", + "format": "jsonschema", + "version": "1-0-0" + }, + "title": "flattenedProperties", + "type": "object", + "properties": { + "marker": { + "type": "string" + } + }, + "unevaluatedProperties": false +} diff --git a/itests/src/test/resources/schemas/schema-flattened.json b/itests/src/test/resources/schemas/schema-flattened.json new file mode 100644 index 000000000..0102486d6 --- /dev/null +++ b/itests/src/test/resources/schemas/schema-flattened.json @@ -0,0 +1,25 @@ +{ + "$id": "https://vendor.test.com/schemas/json/events/flattened/1-0-0", + "$schema": "https://json-schema.org/draft/2019-09/schema", + "self":{ + "vendor":"com.vendor.test", + "name":"flattened", + "format":"jsonschema", + "target":"events", + "version":"1-0-0" + }, + "title": "Flattened Event test", + "type": "object", + "allOf": [ + { "$ref": "https://unomi.apache.org/schemas/json/event/1-0-0" } + ], + "properties": { + "flattenedProperties": { + "$ref": "https://vendor.test.com/schemas/json/events/flattened/flattenedProperties/1-0-0" + }, + "properties": { + "$ref": "https://vendor.test.com/schemas/json/events/flattened/properties/1-0-0" + } + }, + "unevaluatedProperties": false +} diff --git a/persistence-elasticsearch/core/pom.xml b/persistence-elasticsearch/core/pom.xml index beaf2de14..4b7791ad5 100644 --- a/persistence-elasticsearch/core/pom.xml +++ b/persistence-elasticsearch/core/pom.xml @@ -114,6 +114,13 @@ <artifactId>lucene-test-framework</artifactId> <version>8.2.0</version> <scope>test</scope> + <exclusions> + <exclusion> + <!-- internal dep that is scoped "compile", ending up to be embedded and conflicting with ES Rest client internal deps --> + <groupId>org.apache.lucene</groupId> + <artifactId>lucene-core</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>org.apache.logging.log4j</groupId> diff --git a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json index f3958e6ea..e7a8231b8 100644 --- a/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json +++ b/persistence-elasticsearch/core/src/main/resources/META-INF/cxs/mappings/event.json @@ -18,6 +18,9 @@ } ], "properties": { + "flattenedProperties": { + "type": "flattened" + }, "timeStamp": { "type": "date" }, diff --git a/pom.xml b/pom.xml index 89d6d038b..ee5abe5f9 100644 --- a/pom.xml +++ b/pom.xml @@ -72,6 +72,7 @@ <version.karaf.cellar>4.2.1</version.karaf.cellar> <version.pax.exam>4.13.1</version.pax.exam> <elasticsearch.version>7.4.2</elasticsearch.version> + <elasticsearch.test.version>7.11.0</elasticsearch.test.version> <groovy.version>3.0.3</groovy.version> <networknt.version>1.0.49</networknt.version> <bean.validation.version>1.1.0.Final</bean.validation.version> diff --git a/rest/src/main/java/org/apache/unomi/rest/service/impl/RestServiceUtilsImpl.java b/rest/src/main/java/org/apache/unomi/rest/service/impl/RestServiceUtilsImpl.java index ac617d3d7..44612f158 100644 --- a/rest/src/main/java/org/apache/unomi/rest/service/impl/RestServiceUtilsImpl.java +++ b/rest/src/main/java/org/apache/unomi/rest/service/impl/RestServiceUtilsImpl.java @@ -93,6 +93,7 @@ public class RestServiceUtilsImpl implements RestServiceUtils { if (event.getEventType() != null) { Event eventToSend = new Event(event.getEventType(), session, profile, event.getSourceId(), event.getSource(), event.getTarget(), event.getProperties(), timestamp, event.isPersistent()); + eventToSend.setFlattenedProperties(event.getFlattenedProperties()); if (!eventService.isEventAllowed(event, thirdPartyId)) { logger.warn("Event is not allowed : {}", event.getEventType()); continue; @@ -100,6 +101,7 @@ public class RestServiceUtilsImpl implements RestServiceUtils { if (thirdPartyId != null && event.getItemId() != null) { eventToSend = new Event(event.getItemId(), event.getEventType(), session, profile, event.getSourceId(), event.getSource(), event.getTarget(), event.getProperties(), timestamp, event.isPersistent()); + eventToSend.setFlattenedProperties(event.getFlattenedProperties()); } if (filteredEventTypes != null && filteredEventTypes.contains(event.getEventType())) { logger.debug("Profile is filtering event type {}", event.getEventType());
