This is an automated email from the ASF dual-hosted git repository. zehnder pushed a commit to branch hotfix-fix-nightly-build in repository https://gitbox.apache.org/repos/asf/streampipes.git
commit 73fa98d7466b2e952824c7ecb3ad5eae0ee95e28 Author: Philipp Zehnder <[email protected]> AuthorDate: Mon Jan 16 17:27:29 2023 +0100 [hotfix] Fix failing e2e tests --- .../connect/adapter/format/csv/CsvParser.java | 9 +- .../connect/adapter/util/DatatypeUtils.java | 30 +++-- .../connect/adapter/format/csv/CsvParserTest.java | 45 +++++++ .../connect/adapter/util/DatatypeUtilsTest.java | 130 +++++++++++++++++++++ .../event-schema-preview.component.html | 1 + 5 files changed, 196 insertions(+), 19 deletions(-) diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/csv/CsvParser.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/csv/CsvParser.java index a0a68d963..12724b38a 100644 --- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/csv/CsvParser.java +++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/csv/CsvParser.java @@ -108,11 +108,11 @@ public class CsvParser extends Parser { EventSchema resultSchema = new EventSchema(); for (int i = 0; i < keys.length; i++) { EventPropertyPrimitive p = new EventPropertyPrimitive(); - var runtimeType = DatatypeUtils.getXsdDatatype(data[i], false); + var runtimeType = DatatypeUtils.getXsdDatatype(data[i], true); var convertedValue = DatatypeUtils.convertValue(data[i], runtimeType); p.setRuntimeName(keys[i]); p.setRuntimeType(runtimeType); - sample.put(keys[i], new GuessTypeInfo(DatatypeUtils.getCanonicalTypeClassName(data[i], false), convertedValue)); + sample.put(keys[i], new GuessTypeInfo(DatatypeUtils.getCanonicalTypeClassName(data[i], true), convertedValue)); resultSchema.addEventProperty(p); } @@ -165,11 +165,6 @@ public class CsvParser extends Parser { inQuotes = true; - //Fixed : allow "" in empty quote enclosed -// if (chars[0] != '"' && customQuote == '\"') { -// curVal.append('"'); -// } - //double quotes in column will hit this! if (startCollectChar) { curVal.append('"'); diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/util/DatatypeUtils.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/util/DatatypeUtils.java index 20655f112..4e1933ee0 100644 --- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/util/DatatypeUtils.java +++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/util/DatatypeUtils.java @@ -81,17 +81,29 @@ public class DatatypeUtils { } public static Class<?> getTypeClass(String value, - boolean preferFloat) { + boolean preferFloatingPointNumber) { + var targetClass = String.class; if (NumberUtils.isParsable(value)) { + Class<?> numberClass; try { - Integer.parseInt(value); - return preferFloat ? Float.class : Integer.class; + long longValue = Long.parseLong(value); + numberClass = longValue > Integer.MAX_VALUE ? Long.class : Integer.class; + if (preferFloatingPointNumber) { + return numberClass == Long.class ? Double.class : Float.class; + } else { + return numberClass; + } } catch (NumberFormatException ignored) { } try { - Long.parseLong(value); - return preferFloat ? Float.class : Long.class; + double doubleValue = Double.parseDouble(value); + numberClass = doubleValue > Float.MAX_VALUE ? Double.class : Float.class; + if (preferFloatingPointNumber) { + return numberClass == Double.class ? Double.class : Float.class; + } else { + return numberClass; + } } catch (NumberFormatException ignored) { } @@ -107,13 +119,7 @@ public class DatatypeUtils { return Boolean.class; } - return String.class; + return targetClass; } - public static void main(String[] args) { - long max = Long.MAX_VALUE; - String className = getCanonicalTypeClassName(String.valueOf(max), true); - System.out.println(className); - System.out.println(convertValue(max, className)); - } } diff --git a/streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/connect/adapter/format/csv/CsvParserTest.java b/streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/connect/adapter/format/csv/CsvParserTest.java new file mode 100644 index 000000000..f525319ad --- /dev/null +++ b/streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/connect/adapter/format/csv/CsvParserTest.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.extensions.management.connect.adapter.format.csv; + + +import org.junit.Test; + +import java.nio.charset.StandardCharsets; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +public class CsvParserTest { + + @Test + public void getSchemaAndSampleTimestamp() { + var csvValue = List.of( + "timestamp,value".getBytes(StandardCharsets.UTF_8), + "1667904471000,1".getBytes(StandardCharsets.UTF_8) + ); + + var csvParser = new CsvParser(",", true); + var result = csvParser.getSchemaAndSample(csvValue); + + assertEquals(result.getEventPreview().size(), 1); + assertEquals(1667904471000.0, result.getEventPreview().get(0).get("timestamp").getValue()); + } + +} \ No newline at end of file diff --git a/streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/connect/adapter/util/DatatypeUtilsTest.java b/streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/connect/adapter/util/DatatypeUtilsTest.java new file mode 100644 index 000000000..c5d273e07 --- /dev/null +++ b/streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/connect/adapter/util/DatatypeUtilsTest.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.streampipes.extensions.management.connect.adapter.util; + +import org.apache.streampipes.vocabulary.XSD; + +import org.junit.Test; + +import java.util.Locale; + +import static org.junit.Assert.assertEquals; + +public class DatatypeUtilsTest { + + @Test + /** + * This test ensures that timestamps represented as strings are correctly parsed. + * Often they are first parsed into floating point number before transformed back to long. + * The data type for those values should be Double and not Float, because the transformation to Float might change + * the value + */ + public void convertTimestampValue() { + var inputValue = "1667904471000"; + + var floatValue = DatatypeUtils.convertValue(inputValue, XSD.DOUBLE.toString()); + var longValue = DatatypeUtils.convertValue(floatValue, XSD.LONG.toString()); + + assertEquals(Long.parseLong(inputValue), longValue); + } + + String booleanInputValue = "true"; + @Test + public void getTypeClassNoPrefereFloatingPointBoolean() { + var result = DatatypeUtils.getTypeClass(booleanInputValue, false); + assertEquals(Boolean.class, result); + } + + @Test + public void getTypeClassWithPrefereFloatingPointBoolean() { + var result = DatatypeUtils.getTypeClass(booleanInputValue, true); + assertEquals(Boolean.class, result); + } + + String integerInputValue = "1"; + @Test + public void getTypeClassNoPrefereFloatingPointInteger() { + var result = DatatypeUtils.getTypeClass(integerInputValue, false); + assertEquals(Integer.class, result); + } + + @Test + public void getTypeClassWithPrefereFloatingPointInteger() { + var result = DatatypeUtils.getTypeClass(integerInputValue, true); + assertEquals(Float.class, result); + } + + String floatInputValue = "1.0"; + @Test + public void getTypeClassNoPrefereFloatingPointFloat() { + var result = DatatypeUtils.getTypeClass(floatInputValue, false); + assertEquals(Float.class, result); + } + + @Test + public void getTypeClassWithPrefereFloatingPointFloat() { + var result = DatatypeUtils.getTypeClass(floatInputValue, true); + assertEquals(Float.class, result); + } + + + String doubleInputValue = String.format(Locale.US, "%.2f", Double.MAX_VALUE); + @Test + public void getTypeClassNoPrefereFloatingPointDouble() { + var result = DatatypeUtils.getTypeClass(doubleInputValue, false); + assertEquals(Double.class, result); + } + + @Test + public void getTypeClassWithPrefereFloatingPointDouble() { + var result = DatatypeUtils.getTypeClass(doubleInputValue, true); + assertEquals(Double.class, result); + } + + + String longInputValue = "1667904471000"; + @Test + public void getTypeClassNoPrefereFloatingPointLong() { + var result = DatatypeUtils.getTypeClass(longInputValue, false); + assertEquals(Long.class, result); + } + + @Test + public void getTypeClassWithPrefereFloatingPointLong() { + var result = DatatypeUtils.getTypeClass(longInputValue, true); + assertEquals(Double.class, result); + } + + String stringInputValue = "one"; + @Test + public void getTypeClassNoPrefereFloatingPointString() { + var result = DatatypeUtils.getTypeClass(stringInputValue, false); + + assertEquals(String.class, result); + } + + @Test + public void getTypeClassWithPrefereFloatingPointString() { + var result = DatatypeUtils.getTypeClass(stringInputValue, true); + + assertEquals(String.class, result); + } + + +} \ No newline at end of file diff --git a/ui/src/app/connect/components/adapter-configuration/schema-editor/event-schema-preview/event-schema-preview.component.html b/ui/src/app/connect/components/adapter-configuration/schema-editor/event-schema-preview/event-schema-preview.component.html index a55ad96ce..6bb8b51e3 100644 --- a/ui/src/app/connect/components/adapter-configuration/schema-editor/event-schema-preview/event-schema-preview.component.html +++ b/ui/src/app/connect/components/adapter-configuration/schema-editor/event-schema-preview/event-schema-preview.component.html @@ -22,6 +22,7 @@ <pre [innerHTML]="originalField | jsonpretty" class="preview-text" + data-cy="schema-preview-original-event" ></pre> </sp-basic-inner-panel> </div>
