This is an automated email from the ASF dual-hosted git repository.
htowaileb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 6fb6bb4d73 [ASTERIXDB-3243][EXT]: Issue warning and skip file on
computed field type mismatch
6fb6bb4d73 is described below
commit 6fb6bb4d735417d5326e9df7a4ea72938a46cf0a
Author: Hussain Towaileb <[email protected]>
AuthorDate: Fri Aug 11 04:47:32 2023 +0300
[ASTERIXDB-3243][EXT]: Issue warning and skip file on computed field type
mismatch
Change-Id: I2b21bfb22a59163a5e66201b45aaf3ad7ae7dda9
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17719
Reviewed-by: Wail Alkowaileet <[email protected]>
Integration-Tests: Jenkins <[email protected]>
Tested-by: Hussain Towaileb <[email protected]>
---
.../runtimets/testsuite_external_dataset_s3.xml | 14 +++++++------
.../input/filter/ExternalFilterValueEvaluator.java | 24 +++++++++-------------
.../asterix/external/util/ExternalDataPrefix.java | 2 +-
3 files changed, 19 insertions(+), 21 deletions(-)
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
index e50ec3cc8e..cd40b1a0ec 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
@@ -221,6 +221,7 @@
</compilation-unit>
</test-case>
<!-- Parquet Tests End -->
+ <!-- Dynamic prefixes tests start -->
<test-case FilePath="external-dataset/s3/filter">
<compilation-unit name="one-field">
<output-dir compare="Text">one-field</output-dir>
@@ -229,14 +230,15 @@
<test-case FilePath="external-dataset/s3/filter" check-warnings="true">
<compilation-unit name="type-mismatch">
<output-dir compare="Text">type-mismatch</output-dir>
- <expected-warn>Failed to evaluate computed field. File:
'external-filter/department/accounting/0.json'. Computed Field Name: 'name'.
Computed Field Type: 'bigint'. Computed Field Value: 'department'. Reason:
'java.lang.NumberFormatException: For input string:
"department"'</expected-warn>
- <expected-warn>Failed to evaluate computed field. File:
'external-filter/department/engineering/0.json'. Computed Field Name: 'name'.
Computed Field Type: 'bigint'. Computed Field Value: 'department'. Reason:
'java.lang.NumberFormatException: For input string:
"department"'</expected-warn>
- <expected-warn>Failed to evaluate computed field. File:
'external-filter/department/hr/0.json'. Computed Field Name: 'name'. Computed
Field Type: 'bigint'. Computed Field Value: 'department'. Reason:
'java.lang.NumberFormatException: For input string:
"department"'</expected-warn>
- <expected-warn>Failed to evaluate computed field. File:
'external-filter/last-name/Jones/0.json'. Computed Field Name: 'name'. Computed
Field Type: 'bigint'. Computed Field Value: 'last-name'. Reason:
'java.lang.NumberFormatException: For input string: "last-name"'</expected-warn>
- <expected-warn>Failed to evaluate computed field. File:
'external-filter/last-name/miller/0.json'. Computed Field Name: 'name'.
Computed Field Type: 'bigint'. Computed Field Value: 'last-name'. Reason:
'java.lang.NumberFormatException: For input string: "last-name"'</expected-warn>
- <expected-warn>Failed to evaluate computed field. File:
'external-filter/last-name/smith/0.json'. Computed Field Name: 'name'. Computed
Field Type: 'bigint'. Computed Field Value: 'last-name'. Reason:
'java.lang.NumberFormatException: For input string: "last-name"'</expected-warn>
+ <expected-warn>Failed to evaluate computed field. File:
'external-filter/department/accounting/0.json'. Computed Field Name: 'name'.
Computed Field Type: 'bigint'. Computed Field Value: 'department'. Reason: 'For
input string: "department"'</expected-warn>
+ <expected-warn>Failed to evaluate computed field. File:
'external-filter/department/engineering/0.json'. Computed Field Name: 'name'.
Computed Field Type: 'bigint'. Computed Field Value: 'department'. Reason: 'For
input string: "department"'</expected-warn>
+ <expected-warn>Failed to evaluate computed field. File:
'external-filter/department/hr/0.json'. Computed Field Name: 'name'. Computed
Field Type: 'bigint'. Computed Field Value: 'department'. Reason: 'For input
string: "department"'</expected-warn>
+ <expected-warn>Failed to evaluate computed field. File:
'external-filter/last-name/Jones/0.json'. Computed Field Name: 'name'. Computed
Field Type: 'bigint'. Computed Field Value: 'last-name'. Reason: 'For input
string: "last-name"'</expected-warn>
+ <expected-warn>Failed to evaluate computed field. File:
'external-filter/last-name/miller/0.json'. Computed Field Name: 'name'.
Computed Field Type: 'bigint'. Computed Field Value: 'last-name'. Reason: 'For
input string: "last-name"'</expected-warn>
+ <expected-warn>Failed to evaluate computed field. File:
'external-filter/last-name/smith/0.json'. Computed Field Name: 'name'. Computed
Field Type: 'bigint'. Computed Field Value: 'last-name'. Reason: 'For input
string: "last-name"'</expected-warn>
</compilation-unit>
</test-case>
+ <!-- Dynamic prefixes tests end -->
<test-case FilePath="external-dataset">
<compilation-unit name="common/empty-string-definition">
<placeholder name="adapter" value="S3" />
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/ExternalFilterValueEvaluator.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/ExternalFilterValueEvaluator.java
index a07e067534..40b10d453d 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/ExternalFilterValueEvaluator.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/ExternalFilterValueEvaluator.java
@@ -62,20 +62,16 @@ class ExternalFilterValueEvaluator implements
IExternalFilterValueEvaluator {
DataOutput output = value.getDataOutput();
SerializerDeserializerUtil.serializeTag(typeTag, output);
- try {
- switch (typeTag) {
- case TINYINT:
- case SMALLINT:
- case INTEGER:
- case BIGINT:
-
Integer64SerializerDeserializer.write(Long.parseLong(stringValue), output);
- case DOUBLE:
-
DoubleSerializerDeserializer.write(Double.parseDouble(stringValue), output);
- case STRING:
- stringSerDer.serialize(stringValue, output);
- }
- } catch (Exception ex) {
- throw HyracksDataException.create(ex);
+ switch (typeTag) {
+ case TINYINT:
+ case SMALLINT:
+ case INTEGER:
+ case BIGINT:
+
Integer64SerializerDeserializer.write(Long.parseLong(stringValue), output);
+ case DOUBLE:
+
DoubleSerializerDeserializer.write(Double.parseDouble(stringValue), output);
+ case STRING:
+ stringSerDer.serialize(stringValue, output);
}
}
}
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
index 66fa445d62..9d45fedf2d 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
@@ -277,7 +277,7 @@ public class ExternalDataPrefix {
evaluator.setValue(i, computedFieldValue);
}
}
- } catch (HyracksDataException ex) {
+ } catch (NumberFormatException ex) {
if (warningCollector.shouldWarn()) {
warningCollector.warn(Warning.of(null,
ErrorCode.FAILED_TO_EVALUATE_COMPUTED_FIELD,
LogRedactionUtil.userData(key), computedFieldName,
computedFieldType,
