>From Hussain Towaileb <[email protected]>:
Hussain Towaileb has uploaded this change for review. (
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17719 )
Change subject: [ASTERIXDB-3243][EXT]: Issue warning and skip file on computed
field type mismatch
......................................................................
[ASTERIXDB-3243][EXT]: Issue warning and skip file on computed field type
mismatch
Change-Id: I2b21bfb22a59163a5e66201b45aaf3ad7ae7dda9
---
M
asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
M
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/ExternalFilterValueEvaluator.java
3 files changed, 28 insertions(+), 21 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/19/17719/1
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
index e50ec3c..cd40b1a 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
@@ -221,6 +221,7 @@
</compilation-unit>
</test-case>
<!-- Parquet Tests End -->
+ <!-- Dynamic prefixes tests start -->
<test-case FilePath="external-dataset/s3/filter">
<compilation-unit name="one-field">
<output-dir compare="Text">one-field</output-dir>
@@ -229,14 +230,15 @@
<test-case FilePath="external-dataset/s3/filter" check-warnings="true">
<compilation-unit name="type-mismatch">
<output-dir compare="Text">type-mismatch</output-dir>
- <expected-warn>Failed to evaluate computed field. File:
'external-filter/department/accounting/0.json'. Computed Field Name: 'name'.
Computed Field Type: 'bigint'. Computed Field Value: 'department'. Reason:
'java.lang.NumberFormatException: For input string:
"department"'</expected-warn>
- <expected-warn>Failed to evaluate computed field. File:
'external-filter/department/engineering/0.json'. Computed Field Name: 'name'.
Computed Field Type: 'bigint'. Computed Field Value: 'department'. Reason:
'java.lang.NumberFormatException: For input string:
"department"'</expected-warn>
- <expected-warn>Failed to evaluate computed field. File:
'external-filter/department/hr/0.json'. Computed Field Name: 'name'. Computed
Field Type: 'bigint'. Computed Field Value: 'department'. Reason:
'java.lang.NumberFormatException: For input string:
"department"'</expected-warn>
- <expected-warn>Failed to evaluate computed field. File:
'external-filter/last-name/Jones/0.json'. Computed Field Name: 'name'. Computed
Field Type: 'bigint'. Computed Field Value: 'last-name'. Reason:
'java.lang.NumberFormatException: For input string: "last-name"'</expected-warn>
- <expected-warn>Failed to evaluate computed field. File:
'external-filter/last-name/miller/0.json'. Computed Field Name: 'name'.
Computed Field Type: 'bigint'. Computed Field Value: 'last-name'. Reason:
'java.lang.NumberFormatException: For input string: "last-name"'</expected-warn>
- <expected-warn>Failed to evaluate computed field. File:
'external-filter/last-name/smith/0.json'. Computed Field Name: 'name'. Computed
Field Type: 'bigint'. Computed Field Value: 'last-name'. Reason:
'java.lang.NumberFormatException: For input string: "last-name"'</expected-warn>
+ <expected-warn>Failed to evaluate computed field. File:
'external-filter/department/accounting/0.json'. Computed Field Name: 'name'.
Computed Field Type: 'bigint'. Computed Field Value: 'department'. Reason: 'For
input string: "department"'</expected-warn>
+ <expected-warn>Failed to evaluate computed field. File:
'external-filter/department/engineering/0.json'. Computed Field Name: 'name'.
Computed Field Type: 'bigint'. Computed Field Value: 'department'. Reason: 'For
input string: "department"'</expected-warn>
+ <expected-warn>Failed to evaluate computed field. File:
'external-filter/department/hr/0.json'. Computed Field Name: 'name'. Computed
Field Type: 'bigint'. Computed Field Value: 'department'. Reason: 'For input
string: "department"'</expected-warn>
+ <expected-warn>Failed to evaluate computed field. File:
'external-filter/last-name/Jones/0.json'. Computed Field Name: 'name'. Computed
Field Type: 'bigint'. Computed Field Value: 'last-name'. Reason: 'For input
string: "last-name"'</expected-warn>
+ <expected-warn>Failed to evaluate computed field. File:
'external-filter/last-name/miller/0.json'. Computed Field Name: 'name'.
Computed Field Type: 'bigint'. Computed Field Value: 'last-name'. Reason: 'For
input string: "last-name"'</expected-warn>
+ <expected-warn>Failed to evaluate computed field. File:
'external-filter/last-name/smith/0.json'. Computed Field Name: 'name'. Computed
Field Type: 'bigint'. Computed Field Value: 'last-name'. Reason: 'For input
string: "last-name"'</expected-warn>
</compilation-unit>
</test-case>
+ <!-- Dynamic prefixes tests end -->
<test-case FilePath="external-dataset">
<compilation-unit name="common/empty-string-definition">
<placeholder name="adapter" value="S3" />
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/ExternalFilterValueEvaluator.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/ExternalFilterValueEvaluator.java
index a07e067..40b10d4 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/ExternalFilterValueEvaluator.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/ExternalFilterValueEvaluator.java
@@ -62,20 +62,16 @@
DataOutput output = value.getDataOutput();
SerializerDeserializerUtil.serializeTag(typeTag, output);
- try {
- switch (typeTag) {
- case TINYINT:
- case SMALLINT:
- case INTEGER:
- case BIGINT:
-
Integer64SerializerDeserializer.write(Long.parseLong(stringValue), output);
- case DOUBLE:
-
DoubleSerializerDeserializer.write(Double.parseDouble(stringValue), output);
- case STRING:
- stringSerDer.serialize(stringValue, output);
- }
- } catch (Exception ex) {
- throw HyracksDataException.create(ex);
+ switch (typeTag) {
+ case TINYINT:
+ case SMALLINT:
+ case INTEGER:
+ case BIGINT:
+
Integer64SerializerDeserializer.write(Long.parseLong(stringValue), output);
+ case DOUBLE:
+
DoubleSerializerDeserializer.write(Double.parseDouble(stringValue), output);
+ case STRING:
+ stringSerDer.serialize(stringValue, output);
}
}
}
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
index 66fa445..9d45fed 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
@@ -277,7 +277,7 @@
evaluator.setValue(i, computedFieldValue);
}
}
- } catch (HyracksDataException ex) {
+ } catch (NumberFormatException ex) {
if (warningCollector.shouldWarn()) {
warningCollector.warn(Warning.of(null,
ErrorCode.FAILED_TO_EVALUATE_COMPUTED_FIELD,
LogRedactionUtil.userData(key), computedFieldName,
computedFieldType,
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17719
To unsubscribe, or for help writing mail filters, visit
https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: I2b21bfb22a59163a5e66201b45aaf3ad7ae7dda9
Gerrit-Change-Number: 17719
Gerrit-PatchSet: 1
Gerrit-Owner: Hussain Towaileb <[email protected]>
Gerrit-MessageType: newchange
