This is an automated email from the ASF dual-hosted git repository. zehnder pushed a commit to branch SP-1065 in repository https://gitbox.apache.org/repos/asf/streampipes.git
commit 3123212a72894e32d56456fcd6dfcdbdddf8ed02 Author: Philipp Zehnder <[email protected]> AuthorDate: Mon Jan 9 11:25:19 2023 +0100 [#1065] Add e2e tests for all json formats --- .../adapter/format/geojson/GeoJsonParser.java | 58 ++++++--- ...onObjectParser.java => AbstractJsonParser.java} | 49 +------- .../adapter/format/json/arraykey/JsonParser.java | 32 +---- .../format/json/arraynokey/JsonArrayParser.java | 36 +----- .../format/json/object/JsonObjectParser.java | 47 +------ .../fixtures/connect/formatTests/geoJson.json | 16 +++ .../fixtures/connect/formatTests/jsonArray.json | 9 ++ .../connect/formatTests/jsonArrayField.json | 11 ++ .../fixtures/connect/formatTests/jsonObject.json | 7 ++ ui/cypress/support/utils/connect/ConnectBtns.ts | 43 +++++++ ui/cypress/support/utils/connect/ConnectUtils.ts | 12 +- .../adapter/formats/jsonObjectFormat.smoke.spec.ts | 139 +++++++++++++++++++++ .../event-schema-preview.component.html | 6 +- 13 files changed, 285 insertions(+), 180 deletions(-) diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/geojson/GeoJsonParser.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/geojson/GeoJsonParser.java index f09a8405d..632f93a00 100644 --- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/geojson/GeoJsonParser.java +++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/geojson/GeoJsonParser.java @@ -24,6 +24,8 @@ import org.apache.streampipes.extensions.api.connect.exception.ParseException; import org.apache.streampipes.extensions.management.connect.adapter.format.util.JsonEventProperty; import org.apache.streampipes.extensions.management.connect.adapter.model.generic.Parser; import org.apache.streampipes.model.connect.grounding.FormatDescription; +import org.apache.streampipes.model.connect.guess.AdapterGuessInfo; +import org.apache.streampipes.model.connect.guess.GuessTypeInfo; import org.apache.streampipes.model.schema.EventProperty; import org.apache.streampipes.model.schema.EventPropertyPrimitive; import org.apache.streampipes.model.schema.EventSchema; @@ -34,7 +36,6 @@ import com.google.common.base.Charsets; import com.google.common.io.CharStreams; import com.google.gson.Gson; import org.geojson.Feature; -import org.geojson.FeatureCollection; import org.geojson.LineString; import org.geojson.MultiLineString; import org.geojson.MultiPoint; @@ -66,7 +67,6 @@ public class GeoJsonParser extends Parser { @Override public void parse(InputStream data, EmitBinaryEvent emitBinaryEvent) throws ParseException { - FeatureCollection geoFeature; Gson gson = new Gson(); try { @@ -85,29 +85,33 @@ public class GeoJsonParser extends Parser { @Override public EventSchema getEventSchema(List<byte[]> oneEvent) { - EventSchema resultSchema = new EventSchema(); + return this.getSchemaAndSample(oneEvent).getEventSchema(); + } + + @Override + public boolean supportsPreview() { + return true; + } + + @Override + public AdapterGuessInfo getSchemaAndSample(List<byte[]> eventSample) throws ParseException { Feature geoFeature = null; try { - geoFeature = new ObjectMapper().readValue(oneEvent.get(0), Feature.class); + geoFeature = new ObjectMapper().readValue(eventSample.get(0), Feature.class); } catch (IOException e) { logger.error(e.toString()); } - for (Map.Entry<String, Object> entry : geoFeature.getProperties().entrySet()) { - EventProperty p = JsonEventProperty.getEventProperty(entry.getKey(), entry.getValue()); - resultSchema.addEventProperty(p); - } - - List<EventProperty> eventProperties = parseGeometryField(geoFeature); - eventProperties.forEach(eventProperty -> resultSchema.addEventProperty(eventProperty)); - - return resultSchema; + return this.parseGeometryField(geoFeature); } - private List<EventProperty> parseGeometryField(Feature geoFeature) { + private AdapterGuessInfo parseGeometryField(Feature geoFeature) { + + EventSchema resultSchema = new EventSchema(); List<EventProperty> eventProperties = new LinkedList<>(); + var sampleValues = new HashMap<String, GuessTypeInfo>(); if (geoFeature.getGeometry() instanceof Point) { Point point = (Point) geoFeature.getGeometry(); @@ -115,35 +119,51 @@ public class GeoJsonParser extends Parser { "http://www.w3.org/2003/01/geo/wgs84_pos#long";)); eventProperties.add(getEventPropertyGeoJson("latitude", point.getCoordinates().getLatitude(), "http://www.w3.org/2003/01/geo/wgs84_pos#lat";)); + + sampleValues.put("longitude", new GuessTypeInfo("java.lang.Double", point.getCoordinates().getLongitude())); + sampleValues.put("latitude", new GuessTypeInfo("java.lang.Double", point.getCoordinates().getLatitude())); if (point.getCoordinates().hasAltitude()) { eventProperties.add(getEventPropertyGeoJson("altitude", point.getCoordinates().getAltitude(), SO.ALTITUDE)); + sampleValues.put("altitude", new GuessTypeInfo("java.lang.Double", point.getCoordinates().getAltitude())); } } else if (geoFeature.getGeometry() instanceof LineString) { LineString lineString = (LineString) geoFeature.getGeometry(); eventProperties.add(JsonEventProperty.getEventProperty("coorindatesLineString", lineString.getCoordinates())); - + sampleValues.put("coorindatesLineString", new GuessTypeInfo("java.lang.Double", lineString.getCoordinates())); } else if (geoFeature.getGeometry() instanceof Polygon) { Polygon polygon = (Polygon) geoFeature.getGeometry(); eventProperties.add(JsonEventProperty.getEventProperty("coorindatesPolygon", polygon.getCoordinates())); - + sampleValues.put("coorindatesPolygon", new GuessTypeInfo("java.lang.Double", polygon.getCoordinates())); } else if (geoFeature.getGeometry() instanceof MultiPoint) { MultiPoint multiPoint = (MultiPoint) geoFeature.getGeometry(); eventProperties.add(JsonEventProperty.getEventProperty("coorindatesMultiPoint", multiPoint.getCoordinates())); - + sampleValues.put("coorindatesMultiPoint", new GuessTypeInfo("java.lang.Double", multiPoint.getCoordinates())); } else if (geoFeature.getGeometry() instanceof MultiLineString) { MultiLineString multiLineString = (MultiLineString) geoFeature.getGeometry(); eventProperties.add( JsonEventProperty.getEventProperty("coorindatesMultiLineString", multiLineString.getCoordinates())); - + sampleValues.put("coorindatesMultiLineString", + new GuessTypeInfo("java.lang.Double", multiLineString.getCoordinates())); } else if (geoFeature.getGeometry() instanceof MultiPolygon) { MultiPolygon multiPolygon = (MultiPolygon) geoFeature.getGeometry(); eventProperties.add(JsonEventProperty.getEventProperty("coorindatesMultiPolygon", multiPolygon.getCoordinates())); + sampleValues.put("coorindatesMultiPolygon", new GuessTypeInfo("java.lang.Double", multiPolygon.getCoordinates())); } else { logger.error("No geometry field found in geofeature: " + geoFeature.toString()); } - return eventProperties; + + for (Map.Entry<String, Object> entry : geoFeature.getProperties().entrySet()) { + EventProperty p = JsonEventProperty.getEventProperty(entry.getKey(), entry.getValue()); + resultSchema.addEventProperty(p); + sampleValues.put(p.getRuntimeName(), + new GuessTypeInfo(entry.getValue().getClass().getCanonicalName(), entry.getValue())); + } + + eventProperties.forEach(eventProperty -> resultSchema.addEventProperty(eventProperty)); + + return new AdapterGuessInfo(resultSchema, sampleValues); } diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/json/object/JsonObjectParser.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/json/AbstractJsonParser.java similarity index 67% copy from streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/json/object/JsonObjectParser.java copy to streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/json/AbstractJsonParser.java index 58efc8e83..3d04e882f 100644 --- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/json/object/JsonObjectParser.java +++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/json/AbstractJsonParser.java @@ -16,65 +16,23 @@ * */ -package org.apache.streampipes.extensions.management.connect.adapter.format.json.object; - +package org.apache.streampipes.extensions.management.connect.adapter.format.json; import org.apache.streampipes.commons.exceptions.SpRuntimeException; import org.apache.streampipes.dataformat.json.JsonDataFormatDefinition; -import org.apache.streampipes.extensions.api.connect.EmitBinaryEvent; import org.apache.streampipes.extensions.api.connect.exception.ParseException; import org.apache.streampipes.extensions.management.connect.adapter.format.util.JsonEventProperty; import org.apache.streampipes.extensions.management.connect.adapter.model.generic.Parser; -import org.apache.streampipes.model.connect.grounding.FormatDescription; import org.apache.streampipes.model.connect.guess.AdapterGuessInfo; import org.apache.streampipes.model.connect.guess.GuessTypeInfo; import org.apache.streampipes.model.schema.EventProperty; import org.apache.streampipes.model.schema.EventSchema; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.io.InputStream; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; -public class JsonObjectParser extends Parser { - - private static final Logger logger = LoggerFactory.getLogger(JsonObjectParser.class); - - private ObjectMapper mapper; - private JsonDataFormatDefinition jsonDefinition; - - @Override - public Parser getInstance(FormatDescription formatDescription) { - return new JsonObjectParser(); - } - - /** - * Use this constructor when just a specific key of the object should be parsed - */ - public JsonObjectParser() { - mapper = new ObjectMapper(); - jsonDefinition = new JsonDataFormatDefinition(); - } - - @Override - public void parse(InputStream data, EmitBinaryEvent emitBinaryEvent) throws ParseException { - - try { - Map<String, Object> map = mapper.readValue(data, HashMap.class); - emitBinaryEvent.emit(jsonDefinition.fromMap(map)); - } catch (IOException e) { - e.printStackTrace(); - } catch (SpRuntimeException e) { - e.printStackTrace(); - } - - } +public abstract class AbstractJsonParser extends Parser { @Override public EventSchema getEventSchema(List<byte[]> oneEvent) { @@ -93,7 +51,7 @@ public class JsonObjectParser extends Parser { JsonDataFormatDefinition jsonDefinition = new JsonDataFormatDefinition(); - Map<String, Object> exampleEvent = null; + Map<String, Object> exampleEvent; try { exampleEvent = jsonDefinition.toMap(eventSample.get(0)); @@ -114,4 +72,5 @@ public class JsonObjectParser extends Parser { throw new ParseException("Could not serialize event, did you choose the correct format?", e); } } + } diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/json/arraykey/JsonParser.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/json/arraykey/JsonParser.java index 5f1b31dcb..0193ab936 100644 --- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/json/arraykey/JsonParser.java +++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/json/arraykey/JsonParser.java @@ -18,17 +18,14 @@ package org.apache.streampipes.extensions.management.connect.adapter.format.json.arraykey; - import org.apache.streampipes.commons.exceptions.SpRuntimeException; import org.apache.streampipes.dataformat.json.JsonDataFormatDefinition; import org.apache.streampipes.extensions.api.connect.EmitBinaryEvent; import org.apache.streampipes.extensions.api.connect.exception.ParseException; -import org.apache.streampipes.extensions.management.connect.adapter.format.util.JsonEventProperty; +import org.apache.streampipes.extensions.management.connect.adapter.format.json.AbstractJsonParser; import org.apache.streampipes.extensions.management.connect.adapter.model.generic.Parser; import org.apache.streampipes.extensions.management.connect.adapter.sdk.ParameterExtractor; import org.apache.streampipes.model.connect.grounding.FormatDescription; -import org.apache.streampipes.model.schema.EventProperty; -import org.apache.streampipes.model.schema.EventSchema; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,7 +39,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -public class JsonParser extends Parser { +public class JsonParser extends AbstractJsonParser { Logger logger = LoggerFactory.getLogger(JsonParser.class); private boolean isArray; @@ -121,31 +118,6 @@ public class JsonParser extends Parser { } } - @Override - public EventSchema getEventSchema(List<byte[]> oneEvent) { - EventSchema resultSchema = new EventSchema(); - - JsonDataFormatDefinition jsonDefinition = new JsonDataFormatDefinition(); - - Map<String, Object> exampleEvent = null; - - try { - exampleEvent = jsonDefinition.toMap(oneEvent.get(0)); - } catch (SpRuntimeException e) { - e.printStackTrace(); - } - - for (Map.Entry<String, Object> entry : exampleEvent.entrySet()) { -// System.out.println(entry.getKey() + "/" + entry.getValue()); - EventProperty p = JsonEventProperty.getEventProperty(entry.getKey(), entry.getValue()); - - resultSchema.addEventProperty(p); - - } - - return resultSchema; - } - public Map<String, Object> parseObject(jakarta.json.stream.JsonParser jsonParser, boolean root, int start) { // this variable is needed to skip the first object start String mapKey = ""; diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/json/arraynokey/JsonArrayParser.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/json/arraynokey/JsonArrayParser.java index b028563eb..f0baa3eb7 100644 --- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/json/arraynokey/JsonArrayParser.java +++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/json/arraynokey/JsonArrayParser.java @@ -23,11 +23,9 @@ import org.apache.streampipes.commons.exceptions.SpRuntimeException; import org.apache.streampipes.dataformat.json.JsonDataFormatDefinition; import org.apache.streampipes.extensions.api.connect.EmitBinaryEvent; import org.apache.streampipes.extensions.api.connect.exception.ParseException; -import org.apache.streampipes.extensions.management.connect.adapter.format.util.JsonEventProperty; +import org.apache.streampipes.extensions.management.connect.adapter.format.json.AbstractJsonParser; import org.apache.streampipes.extensions.management.connect.adapter.model.generic.Parser; import org.apache.streampipes.model.connect.grounding.FormatDescription; -import org.apache.streampipes.model.schema.EventProperty; -import org.apache.streampipes.model.schema.EventSchema; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,7 +40,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -public class JsonArrayParser extends Parser { +public class JsonArrayParser extends AbstractJsonParser { Logger logger = LoggerFactory.getLogger(JsonArrayParser.class); @@ -58,7 +56,6 @@ public class JsonArrayParser extends Parser { @Override public void parse(InputStream data, EmitBinaryEvent emitBinaryEvent) throws ParseException { JsonParserFactory factory = Json.createParserFactory(null); - String s = data.toString(); JsonParser jsonParser = factory.createParser(data); // Find first event in array @@ -77,11 +74,10 @@ public class JsonArrayParser extends Parser { JsonDataFormatDefinition jsonDefinition = new JsonDataFormatDefinition(); boolean isEvent = true; boolean result = true; - int objectCount = 0; while (jsonParser.hasNext() && isEvent && result) { Map<String, Object> objectMap = parseObject(jsonParser, true, 1); if (objectMap != null) { - byte[] tmp = new byte[0]; + byte[] tmp; try { tmp = jsonDefinition.fromMap(objectMap); } catch (SpRuntimeException e) { @@ -97,32 +93,6 @@ public class JsonArrayParser extends Parser { } } - @Override - public EventSchema getEventSchema(List<byte[]> oneEvent) { - EventSchema resultSchema = new EventSchema(); - - JsonDataFormatDefinition jsonDefinition = new JsonDataFormatDefinition(); - - Map<String, Object> exampleEvent = null; - - try { - exampleEvent = jsonDefinition.toMap(oneEvent.get(0)); - } catch (SpRuntimeException e) { - e.printStackTrace(); - } - - for (Map.Entry<String, Object> entry : exampleEvent.entrySet()) { -// System.out.println(entry.getKey() + "/" + entry.getValue()); - EventProperty p = JsonEventProperty.getEventProperty(entry.getKey(), entry.getValue()); - - resultSchema.addEventProperty(p); - - } - - return resultSchema; - } - - public Map<String, Object> parseObject(JsonParser jsonParser, boolean root, int start) { // this variable is needed to skip the first object start String mapKey = ""; diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/json/object/JsonObjectParser.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/json/object/JsonObjectParser.java index 58efc8e83..02a186dea 100644 --- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/json/object/JsonObjectParser.java +++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/json/object/JsonObjectParser.java @@ -23,13 +23,9 @@ import org.apache.streampipes.commons.exceptions.SpRuntimeException; import org.apache.streampipes.dataformat.json.JsonDataFormatDefinition; import org.apache.streampipes.extensions.api.connect.EmitBinaryEvent; import org.apache.streampipes.extensions.api.connect.exception.ParseException; -import org.apache.streampipes.extensions.management.connect.adapter.format.util.JsonEventProperty; +import org.apache.streampipes.extensions.management.connect.adapter.format.json.AbstractJsonParser; import org.apache.streampipes.extensions.management.connect.adapter.model.generic.Parser; import org.apache.streampipes.model.connect.grounding.FormatDescription; -import org.apache.streampipes.model.connect.guess.AdapterGuessInfo; -import org.apache.streampipes.model.connect.guess.GuessTypeInfo; -import org.apache.streampipes.model.schema.EventProperty; -import org.apache.streampipes.model.schema.EventSchema; import com.fasterxml.jackson.databind.ObjectMapper; import org.slf4j.Logger; @@ -38,11 +34,9 @@ import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.InputStream; import java.util.HashMap; -import java.util.List; import java.util.Map; -import java.util.stream.Collectors; -public class JsonObjectParser extends Parser { +public class JsonObjectParser extends AbstractJsonParser { private static final Logger logger = LoggerFactory.getLogger(JsonObjectParser.class); @@ -76,42 +70,5 @@ public class JsonObjectParser extends Parser { } - @Override - public EventSchema getEventSchema(List<byte[]> oneEvent) { - return getSchemaAndSample(oneEvent).getEventSchema(); - } - - @Override - public boolean supportsPreview() { - return true; - } - - @Override - public AdapterGuessInfo getSchemaAndSample(List<byte[]> eventSample) throws ParseException { - EventSchema resultSchema = new EventSchema(); - - JsonDataFormatDefinition jsonDefinition = new JsonDataFormatDefinition(); - - Map<String, Object> exampleEvent = null; - - try { - exampleEvent = jsonDefinition.toMap(eventSample.get(0)); - var sample = exampleEvent - .entrySet() - .stream() - .collect(Collectors.toMap(Map.Entry::getKey, e -> - new GuessTypeInfo(e.getValue().getClass().getCanonicalName(), e.getValue()))); - - for (Map.Entry<String, Object> entry : exampleEvent.entrySet()) { - EventProperty p = JsonEventProperty.getEventProperty(entry.getKey(), entry.getValue()); - - resultSchema.addEventProperty(p); - } - - return new AdapterGuessInfo(resultSchema, sample); - } catch (SpRuntimeException e) { - throw new ParseException("Could not serialize event, did you choose the correct format?", e); - } - } } diff --git a/ui/cypress/fixtures/connect/formatTests/geoJson.json b/ui/cypress/fixtures/connect/formatTests/geoJson.json new file mode 100644 index 000000000..047e60497 --- /dev/null +++ b/ui/cypress/fixtures/connect/formatTests/geoJson.json @@ -0,0 +1,16 @@ +{ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [125.6, 10.1] + }, + "properties": { + "timestamp": 1667904471000, + "v1": 4.1 + } + } + ] +} diff --git a/ui/cypress/fixtures/connect/formatTests/jsonArray.json b/ui/cypress/fixtures/connect/formatTests/jsonArray.json new file mode 100644 index 000000000..db655b79d --- /dev/null +++ b/ui/cypress/fixtures/connect/formatTests/jsonArray.json @@ -0,0 +1,9 @@ +[ + { + "timestamp": 1667904471000, + "v1": 4.1, + "v2": "abc", + "v3": true, + "v4": 1 + } +] diff --git a/ui/cypress/fixtures/connect/formatTests/jsonArrayField.json b/ui/cypress/fixtures/connect/formatTests/jsonArrayField.json new file mode 100644 index 000000000..efa2ec775 --- /dev/null +++ b/ui/cypress/fixtures/connect/formatTests/jsonArrayField.json @@ -0,0 +1,11 @@ +{ + "field": [ + { + "timestamp": 1667904471000, + "v1": 4.1, + "v2": "abc", + "v3": true, + "v4": 1 + } + ] +} diff --git a/ui/cypress/fixtures/connect/formatTests/jsonObject.json b/ui/cypress/fixtures/connect/formatTests/jsonObject.json new file mode 100644 index 000000000..48b82b791 --- /dev/null +++ b/ui/cypress/fixtures/connect/formatTests/jsonObject.json @@ -0,0 +1,7 @@ +{ + "timestamp": 1667904471000, + "v1": 4.1, + "v2": "abc", + "v3": true, + "v4": 1 +} diff --git a/ui/cypress/support/utils/connect/ConnectBtns.ts b/ui/cypress/support/utils/connect/ConnectBtns.ts index e79089fa0..17887ff2a 100644 --- a/ui/cypress/support/utils/connect/ConnectBtns.ts +++ b/ui/cypress/support/utils/connect/ConnectBtns.ts @@ -39,4 +39,47 @@ export class ConnectBtns { public static storeEditAdapter() { return cy.dataCy('store-edit-adapter'); } + + public static formatSelectionNextBtn() { + return cy + .dataCy('sp-format-selection-next-button') + .contains('Next') + .parent(); + } + + // ================================================================= + // Format button options + public static json() { + return cy.dataCy('connect-select-json-formats'); + } + + public static jsonArrayField() { + return cy.dataCy('array_field'); + } + + public static geoJson() { + return cy.dataCy('geojson'); + } + + public static jsonObject() { + return cy.dataCy('single_object'); + } + + public static jsonArray() { + return cy.dataCy('array'); + } + + public static xml() { + return cy.dataCy('xml'); + } + + public static csv() { + return cy.dataCy('csv'); + } + + public static image() { + return cy.dataCy('image'); + } + + // ================================================================= } diff --git a/ui/cypress/support/utils/connect/ConnectUtils.ts b/ui/cypress/support/utils/connect/ConnectUtils.ts index 8b77258b2..feca0e004 100644 --- a/ui/cypress/support/utils/connect/ConnectUtils.ts +++ b/ui/cypress/support/utils/connect/ConnectUtils.ts @@ -25,6 +25,7 @@ import { AdapterInput } from '../../model/AdapterInput'; import { ConnectEventSchemaUtils } from '../ConnectEventSchemaUtils'; import { GenericAdapterBuilder } from '../../builder/GenericAdapterBuilder'; import { DataLakeUtils } from '../datalake/DataLakeUtils'; +import { ConnectBtns } from './ConnectBtns'; export class ConnectUtils { public static testSpecificStreamAdapter( @@ -163,11 +164,11 @@ export class ConnectUtils { public static configureFormat(adapterConfiguration: GenericAdapterInput) { // Select format if (adapterConfiguration.format.indexOf('json') !== -1) { - cy.dataCy('connect-select-json-formats').click(); + ConnectBtns.json().click(); if (adapterConfiguration.format.indexOf('object') !== -1) { - cy.dataCy('single_object').click(); + ConnectBtns.jsonObject().click(); } else { - cy.dataCy('array').click(); + ConnectBtns.jsonArray().click(); } } else { cy.dataCy(adapterConfiguration.format).click(); @@ -176,10 +177,7 @@ export class ConnectUtils { StaticPropertyUtils.input(adapterConfiguration.formatConfiguration); // Click next - cy.dataCy('sp-format-selection-next-button') - .contains('Next') - .parent() - .click(); + ConnectBtns.formatSelectionNextBtn().click(); } public static finishEventSchemaConfiguration() { diff --git a/ui/cypress/tests/adapter/formats/jsonObjectFormat.smoke.spec.ts b/ui/cypress/tests/adapter/formats/jsonObjectFormat.smoke.spec.ts new file mode 100644 index 000000000..cc8d40c38 --- /dev/null +++ b/ui/cypress/tests/adapter/formats/jsonObjectFormat.smoke.spec.ts @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { FileManagementUtils } from '../../../support/utils/FileManagementUtils'; +import { ConnectUtils } from '../../../support/utils/connect/ConnectUtils'; +import { ConnectBtns } from '../../../support/utils/connect/ConnectBtns'; +import { StaticPropertyUtils } from '../../../support/utils/StaticPropertyUtils'; +import { UserInputBuilder } from '../../../support/builder/UserInputBuilder'; + +describe('Test adapter formats', () => { + beforeEach('Setup Test', () => { + cy.initStreamPipesTest(); + }); + + const expected = { + timestamp: 1667904471000, + v1: 4.1, + v2: 'abc', + v3: true, + v4: 1, + }; + + it('Test json object format', () => { + // Set up test + FileManagementUtils.addFile('connect/formatTests/jsonObject.json'); + + navigateToFormatSelection(); + + // Set format configuration + ConnectBtns.json().click(); + ConnectBtns.jsonObject().click(); + + // Validate result + validateResult(expected); + }); + + it('Test array with json objects', () => { + // Set up test + FileManagementUtils.addFile('connect/formatTests/jsonArray.json'); + navigateToFormatSelection(); + + // Set format configuration + ConnectBtns.json().click(); + ConnectBtns.jsonArray().click(); + + // Validate result + validateResult(expected); + }); + + it('Test json with a field of type array', () => { + // Set up test + FileManagementUtils.addFile('connect/formatTests/jsonArrayField.json'); + navigateToFormatSelection(); + + // Set format configuration + ConnectBtns.json().click(); + ConnectBtns.jsonArrayField().click(); + const arrayFieldInput = UserInputBuilder.create() + .add('input', 'key', 'field') + .build(); + StaticPropertyUtils.input(arrayFieldInput); + + // Validate result + validateResult(expected); + }); + + it('Test geo json format', () => { + // Set up test + const geoJsonResultEvent = { + latitude: 10.1, + longitude: 125.6, + timestamp: 1667904471000, + v1: 4.1, + }; + FileManagementUtils.addFile('connect/formatTests/geoJson.json'); + navigateToFormatSelection(); + + // Set format configuration + ConnectBtns.json().click(); + ConnectBtns.geoJson().click(); + + // Validate result + validateResult(geoJsonResultEvent); + }); +}); + +const navigateToFormatSelection = () => { + ConnectUtils.goToConnect(); + + ConnectUtils.goToNewAdapterPage(); + + ConnectUtils.selectAdapter('File_Set'); + + ConnectUtils.configureAdapter([]); +}; + +const validateResult = expected => { + ConnectBtns.formatSelectionNextBtn().click(); + cy.dataCy('schema-preview-original-event', { timeout: 10000 }).then( + value => { + const jsonResult = removeWhitespaceExceptInQuotes(value.text()); + expect(jsonResult).to.deep.equal(expected); + }, + ); +}; + +const removeWhitespaceExceptInQuotes = (input: string): string => { + let inQuote = false; + let result = ''; + + for (const char of input) { + if (char === '"') { + inQuote = !inQuote; + } + + if (/\s/.test(char) && !inQuote) { + continue; + } + + result += char; + } + + return JSON.parse(result); +}; diff --git a/ui/src/app/connect/components/adapter-configuration/schema-editor/event-schema-preview/event-schema-preview.component.html b/ui/src/app/connect/components/adapter-configuration/schema-editor/event-schema-preview/event-schema-preview.component.html index d3cad53f6..6910eb9c7 100644 --- a/ui/src/app/connect/components/adapter-configuration/schema-editor/event-schema-preview/event-schema-preview.component.html +++ b/ui/src/app/connect/components/adapter-configuration/schema-editor/event-schema-preview/event-schema-preview.component.html @@ -19,7 +19,11 @@ <div fxFlex="100" fxLayout="row" fxLayoutGap="15px"> <div fxFlex="50" fxLayout="column"> <sp-basic-inner-panel [showTitle]="true" panelTitle="Original (Parsed)"> - <pre [innerHTML]="originalField | jsonpretty" class="preview-text"></pre> + <pre + [innerHTML]="originalField | jsonpretty" + class="preview-text" + data-cy="schema-preview-original-event" + ></pre> </sp-basic-inner-panel> </div>
