This is an automated email from the ASF dual-hosted git repository.
zehnder pushed a commit to branch
2995-timestamp-conversion-is-broken-for-unix-timestamps-in-seconds
in repository https://gitbox.apache.org/repos/asf/streampipes.git
The following commit(s) were added to
refs/heads/2995-timestamp-conversion-is-broken-for-unix-timestamps-in-seconds
by this push:
new b0c4fdfa11 refactor(#2964): Fix unix timestamp conversion for integer
values
b0c4fdfa11 is described below
commit b0c4fdfa1155fd2f8c016a9db9bcd9534f73f5c7
Author: Philipp Zehnder <[email protected]>
AuthorDate: Thu Jul 4 17:38:17 2024 +0200
refactor(#2964): Fix unix timestamp conversion for integer values
---
.../streampipes/connect/shared/DatatypeUtils.java | 20 ++-
.../preprocessing/transform/DatatypeUtilsTest.java | 148 ++++++++++++++++-----
ui/cypress/tests/adapter/fileStream.smoke.spec.ts | 39 +++---
3 files changed, 149 insertions(+), 58 deletions(-)
diff --git
a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/DatatypeUtils.java
b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/DatatypeUtils.java
index cdfaea25ce..8c95afb2f5 100644
---
a/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/DatatypeUtils.java
+++
b/streampipes-connect-shared/src/main/java/org/apache/streampipes/connect/shared/DatatypeUtils.java
@@ -28,6 +28,19 @@ public class DatatypeUtils {
private static final Logger LOG =
LoggerFactory.getLogger(DatatypeUtils.class);
+ /**
+ * Converts the given value to a specified XSD datatype.
+ * This method attempts to convert the input value to the target datatype
specified by the XSD string.
+ * It supports conversion to string, double, float, boolean, integer, and
long types.
+ * If the conversion is not possible due to a format mismatch, the original
value is returned.
+ * A number format exception during conversion is logged as an error.
+ *
+ * @param value The value to be converted. It can be of any type.
+ * @param targetDatatypeXsd The target XSD datatype as a string. Supported
types are XSD.STRING,
+ * XSD.DOUBLE, XSD.FLOAT, XSD.BOOLEAN, XSD.INTEGER,
and XSD.LONG.
+ * @return The converted value as an Object. If conversion fails, the
original value is returned.
+ * @throws NumberFormatException if the string does not contain a parsable
number for numeric conversions.
+ */
public static Object convertValue(Object value,
String targetDatatypeXsd) {
var stringValue = String.valueOf(value);
@@ -42,7 +55,7 @@ public class DatatypeUtils {
} else if (XSD.BOOLEAN.toString().equals(targetDatatypeXsd)) {
return Boolean.parseBoolean(stringValue);
} else if (XSD.INTEGER.toString().equals(targetDatatypeXsd)) {
- var floatingNumber = Float.parseFloat(stringValue);
+ var floatingNumber = Double.parseDouble(stringValue);
return Integer.parseInt(String.valueOf(Math.round(floatingNumber)));
} else if (XSD.LONG.toString().equals(targetDatatypeXsd)) {
var floatingNumber = Double.parseDouble(stringValue);
@@ -57,11 +70,6 @@ public class DatatypeUtils {
return value;
}
- public static String getCanonicalTypeClassName(String value,
- boolean preferFloat) {
- return getTypeClass(value, preferFloat).getCanonicalName();
- }
-
public static String getXsdDatatype(String value,
boolean preferFloat) {
var clazz = getTypeClass(value, preferFloat);
diff --git
a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/DatatypeUtilsTest.java
b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/DatatypeUtilsTest.java
index ef1235b4b8..7937d6875c 100644
---
a/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/DatatypeUtilsTest.java
+++
b/streampipes-connect-shared/src/test/java/org/apache/streampipes/connect/shared/preprocessing/transform/DatatypeUtilsTest.java
@@ -21,110 +21,192 @@ package
org.apache.streampipes.connect.shared.preprocessing.transform;
import org.apache.streampipes.connect.shared.DatatypeUtils;
import org.apache.streampipes.vocabulary.XSD;
-import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import java.util.Locale;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
public class DatatypeUtilsTest {
- @Test
/**
- * This test ensures that timestamps represented as strings are correctly
parsed.
+ * The following tests ensure that timestamps represented as strings are
correctly parsed.
* Often they are first parsed into floating point number before transformed
back to long.
* The data type for those values should be Double and not Float, because
the transformation to Float might change
* the value
*/
- public void convertTimestampValue() {
- var inputValue = "1667904471000";
+ @Test
+ public void convertValue_StringToStringValue() {
+ var inputValue = "testString";
+ var actualValue = DatatypeUtils.convertValue(inputValue,
XSD.STRING.toString());
+
+ assertEquals(inputValue, actualValue);
+ }
+
+ @Test
+ public void convertValue_StringToDoubleValue() {
+ var actualValue = DatatypeUtils.convertValue("1667904471000",
XSD.DOUBLE.toString());
+
+ assertEquals(1.667904471E12, actualValue);
+ }
+
+ @Test
+ public void convertValue_StringToFloatValue() {
+ var actualValue = DatatypeUtils.convertValue("123.45",
XSD.FLOAT.toString());
+
+ assertEquals(123.45f, actualValue);
+ }
+
+ @Test
+ public void convertValue_StringToInteger() {
+ var actualValue = DatatypeUtils.convertValue("1623871500",
XSD.INTEGER.toString());
+
+ assertEquals(1623871500, actualValue);
+ }
+
+ @Test
+ public void convertValue_StringToIntegerValue() {
+ var actualValue = DatatypeUtils.convertValue("123",
XSD.INTEGER.toString());
+
+ assertEquals(123, actualValue);
+ }
+
+ @Test
+ public void convertValue_StringToLongValue() {
+ var actualValue = DatatypeUtils.convertValue("1623871500000",
XSD.LONG.toString());
+
+ assertEquals(1623871500000L, actualValue);
+ }
+
+ @Test
+ public void convertValue_StringToBooleanTrueValue() {
+ var actualValue = DatatypeUtils.convertValue("true",
XSD.BOOLEAN.toString());
- var floatValue = DatatypeUtils.convertValue(inputValue,
XSD.DOUBLE.toString());
- var longValue = DatatypeUtils.convertValue(floatValue,
XSD.LONG.toString());
+ assertEquals(true, actualValue);
+ }
+
+ @Test
+ public void convertValue_StringToBooleanFalseValue() {
+ var actualValue = DatatypeUtils.convertValue("false",
XSD.BOOLEAN.toString());
+
+ assertEquals(false, actualValue);
+ }
+
+ @Test
+ public void convertValue_FloatToIntegerValue_Rounding() {
+ var actualValue = DatatypeUtils.convertValue(123.45f,
XSD.INTEGER.toString());
- Assertions.assertEquals(Long.parseLong(inputValue), longValue);
+ assertEquals(123, actualValue);
}
+ @Test
+ public void convertValue_DoubleToLongValue_Rounding1() {
+ var actualValue = DatatypeUtils.convertValue(1234567890.12345,
XSD.LONG.toString());
+
+ assertEquals(1234567890L, actualValue);
+ }
+
+ @Test
+ public void convertValue_DoubleToLongValue() {
+ var actualValue = DatatypeUtils.convertValue(1.667904471E12,
XSD.LONG.toString());
+
+ assertEquals(1667904471000L, actualValue);
+ }
+
+ @Test
+ public void convertValue_DoubleToLongValue_Rounding() {
+ var actualValue = DatatypeUtils.convertValue(1234567890.12345,
XSD.LONG.toString());
+
+ assertEquals(1234567890L, actualValue);
+ }
+
+
String booleanInputValue = "true";
+
@Test
- public void getTypeClassNoPrefereFloatingPointBoolean() {
+ public void getTypeClass_NoPrefereFloatingPointBoolean() {
var result = DatatypeUtils.getTypeClass(booleanInputValue, false);
- Assertions.assertEquals(Boolean.class, result);
+ assertEquals(Boolean.class, result);
}
@Test
- public void getTypeClassWithPrefereFloatingPointBoolean() {
+ public void getTypeClass_WithPrefereFloatingPointBoolean() {
var result = DatatypeUtils.getTypeClass(booleanInputValue, true);
- Assertions.assertEquals(Boolean.class, result);
+ assertEquals(Boolean.class, result);
}
String integerInputValue = "1";
+
@Test
- public void getTypeClassNoPrefereFloatingPointInteger() {
+ public void getTypeClass_NoPrefereFloatingPointInteger() {
var result = DatatypeUtils.getTypeClass(integerInputValue, false);
- Assertions.assertEquals(Integer.class, result);
+ assertEquals(Integer.class, result);
}
@Test
- public void getTypeClassWithPrefereFloatingPointInteger() {
+ public void getTypeClass_WithPrefereFloatingPointInteger() {
var result = DatatypeUtils.getTypeClass(integerInputValue, true);
- Assertions.assertEquals(Float.class, result);
+ assertEquals(Float.class, result);
}
String floatInputValue = "1.0";
+
@Test
- public void getTypeClassNoPrefereFloatingPointFloat() {
+ public void getTypeClass_NoPrefereFloatingPointFloat() {
var result = DatatypeUtils.getTypeClass(floatInputValue, false);
- Assertions.assertEquals(Float.class, result);
+ assertEquals(Float.class, result);
}
@Test
- public void getTypeClassWithPrefereFloatingPointFloat() {
+ public void getTypeClass_WithPrefereFloatingPointFloat() {
var result = DatatypeUtils.getTypeClass(floatInputValue, true);
- Assertions.assertEquals(Float.class, result);
+ assertEquals(Float.class, result);
}
String doubleInputValue = String.format(Locale.US, "%.2f", Double.MAX_VALUE);
+
@Test
- public void getTypeClassNoPrefereFloatingPointDouble() {
+ public void getTypeClass_NoPrefereFloatingPointDouble() {
var result = DatatypeUtils.getTypeClass(doubleInputValue, false);
- Assertions.assertEquals(Double.class, result);
+ assertEquals(Double.class, result);
}
@Test
- public void getTypeClassWithPrefereFloatingPointDouble() {
+ public void getTypeClass_WithPrefereFloatingPointDouble() {
var result = DatatypeUtils.getTypeClass(doubleInputValue, true);
- Assertions.assertEquals(Double.class, result);
+ assertEquals(Double.class, result);
}
String longInputValue = "1667904471000";
+
@Test
- public void getTypeClassNoPrefereFloatingPointLong() {
+ public void getTypeClass_NoPrefereFloatingPointLong() {
var result = DatatypeUtils.getTypeClass(longInputValue, false);
- Assertions.assertEquals(Long.class, result);
+ assertEquals(Long.class, result);
}
@Test
- public void getTypeClassWithPrefereFloatingPointLong() {
+ public void getTypeClass_WithPrefereFloatingPointLong() {
var result = DatatypeUtils.getTypeClass(longInputValue, true);
- Assertions.assertEquals(Double.class, result);
+ assertEquals(Double.class, result);
}
String stringInputValue = "one";
+
@Test
- public void getTypeClassNoPrefereFloatingPointString() {
+ public void getTypeClass_NoPrefereFloatingPointString() {
var result = DatatypeUtils.getTypeClass(stringInputValue, false);
- Assertions.assertEquals(String.class, result);
+ assertEquals(String.class, result);
}
@Test
- public void getTypeClassWithPrefereFloatingPointString() {
+ public void getTypeClass_WithPrefereFloatingPointString() {
var result = DatatypeUtils.getTypeClass(stringInputValue, true);
- Assertions.assertEquals(String.class, result);
+ assertEquals(String.class, result);
}
-
}
diff --git a/ui/cypress/tests/adapter/fileStream.smoke.spec.ts
b/ui/cypress/tests/adapter/fileStream.smoke.spec.ts
index 7024335b05..2cf9cc6cec 100644
--- a/ui/cypress/tests/adapter/fileStream.smoke.spec.ts
+++ b/ui/cypress/tests/adapter/fileStream.smoke.spec.ts
@@ -20,6 +20,7 @@ import { ConnectUtils } from
'../../support/utils/connect/ConnectUtils';
import { FileManagementUtils } from '../../support/utils/FileManagementUtils';
import { AdapterBuilder } from '../../support/builder/AdapterBuilder';
import { ConnectBtns } from '../../support/utils/connect/ConnectBtns';
+import { ConnectEventSchemaUtils } from
'../../support/utils/connect/ConnectEventSchemaUtils';
describe(
'Test File Replay Adapter',
@@ -62,25 +63,25 @@ describe(
ConnectUtils.testAdapter(adapterInput, true);
});
- // it('File Stream adapter with unix timestamp in seconds', () => {
- //
FileManagementUtils.addFile('connect/fileReplay/timestampInSeconds/input.csv');
- // const adapterConfiguration =
- // ConnectUtils.setUpPreprocessingRuleTest(false);
- //
- // // Edit timestamp property
- // ConnectEventSchemaUtils.editTimestampPropertyWithNumber(
- // 'timestamp',
- // 'Seconds',
- // );
- //
- // ConnectEventSchemaUtils.finishEventSchemaConfiguration();
- // ConnectUtils.tearDownPreprocessingRuleTest(
- // adapterConfiguration,
- //
'cypress/fixtures/connect/fileReplay/timestampInSeconds/expected.csv',
- // false,
- // 2000,
- // );
- // });
+ it('File Stream adapter with unix timestamp in seconds', () => {
+
FileManagementUtils.addFile('connect/fileReplay/timestampInSeconds/input.csv');
+ const adapterConfiguration =
+ ConnectUtils.setUpPreprocessingRuleTest(false);
+
+ // Edit timestamp property
+ ConnectEventSchemaUtils.editTimestampPropertyWithNumber(
+ 'timestamp',
+ 'Seconds',
+ );
+
+ ConnectEventSchemaUtils.finishEventSchemaConfiguration();
+ ConnectUtils.tearDownPreprocessingRuleTest(
+ adapterConfiguration,
+
'cypress/fixtures/connect/fileReplay/timestampInSeconds/expected.csv',
+ false,
+ 2000,
+ );
+ });
// it('File Stream adapter with unix timestamp in milliseconds', () =>
{
//
FileManagementUtils.addFile('connect/fileReplay/timestampInMilliseconds/input.csv');
