>From Hussain Towaileb <[email protected]>: Hussain Towaileb has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17884 )
Change subject: [NO ISSUE][EXT]: Handle type mismatch when computed field is not passed in query ...................................................................... [NO ISSUE][EXT]: Handle type mismatch when computed field is not passed in query Change-Id: I18e90d1dae8f61801d4467bcad02a0e13aed6c9d --- A asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/type-mismatch/result.011.adm A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/type-mismatch/test.011.query.sqlpp M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml 5 files changed, 58 insertions(+), 0 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/84/17884/1 diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/type-mismatch/test.011.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/type-mismatch/test.011.query.sqlpp new file mode 100644 index 0000000..afc8b4a --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/type-mismatch/test.011.query.sqlpp @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// param max-warnings:json=10 + +USE test; + +SELECT value count(*) +FROM test t; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/type-mismatch/result.011.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/type-mismatch/result.011.adm new file mode 100644 index 0000000..c227083 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/type-mismatch/result.011.adm @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml index e252ab2..acf7693 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml @@ -288,6 +288,10 @@ <expected-warn>Failed to evaluate computed field. File: 'external-filter/department/engineering/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'engineering'. Reason: 'java.lang.NumberFormatException: For input string: "engineering"'</expected-warn> <expected-warn>Failed to evaluate computed field. File: 'external-filter/department/hr/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'hr'. Reason: 'java.lang.NumberFormatException: For input string: "hr"'</expected-warn> <expected-warn>The provided external dataset configuration returned no files from the external source</expected-warn> + <expected-warn>Failed to evaluate computed field. File: 'external-filter/department/accounting/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'accounting'. Reason: 'java.lang.NumberFormatException: For input string: "accounting"'</expected-warn> + <expected-warn>Failed to evaluate computed field. File: 'external-filter/department/engineering/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'engineering'. Reason: 'java.lang.NumberFormatException: For input string: "engineering"'</expected-warn> + <expected-warn>Failed to evaluate computed field. File: 'external-filter/department/hr/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'hr'. Reason: 'java.lang.NumberFormatException: For input string: "hr"'</expected-warn> + <expected-warn>The provided external dataset configuration returned no files from the external source</expected-warn> </compilation-unit> </test-case> <test-case FilePath="external-dataset/common/dynamic-prefixes"> diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml index 6d9de7a..7515996 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml @@ -243,6 +243,10 @@ <expected-warn>Failed to evaluate computed field. File: 'external-filter/department/engineering/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'engineering'. Reason: 'java.lang.NumberFormatException: For input string: "engineering"'</expected-warn> <expected-warn>Failed to evaluate computed field. File: 'external-filter/department/hr/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'hr'. Reason: 'java.lang.NumberFormatException: For input string: "hr"'</expected-warn> <expected-warn>The provided external dataset configuration returned no files from the external source</expected-warn> + <expected-warn>Failed to evaluate computed field. File: 'external-filter/department/accounting/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'accounting'. Reason: 'java.lang.NumberFormatException: For input string: "accounting"'</expected-warn> + <expected-warn>Failed to evaluate computed field. File: 'external-filter/department/engineering/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'engineering'. Reason: 'java.lang.NumberFormatException: For input string: "engineering"'</expected-warn> + <expected-warn>Failed to evaluate computed field. File: 'external-filter/department/hr/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'hr'. Reason: 'java.lang.NumberFormatException: For input string: "hr"'</expected-warn> + <expected-warn>The provided external dataset configuration returned no files from the external source</expected-warn> </compilation-unit> </test-case> <test-case FilePath="external-dataset/common/dynamic-prefixes"> diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java index 11de155..160641e 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java @@ -19,6 +19,7 @@ package org.apache.asterix.external.util; +import static org.apache.asterix.external.input.filter.ExternalFilterValueEvaluator.writeValue; import static org.apache.asterix.external.util.ExternalDataConstants.COMPUTED_FIELD_PATTERN; import static org.apache.asterix.external.util.ExternalDataConstants.PREFIX_DEFAULT_DELIMITER; @@ -37,6 +38,7 @@ import org.apache.asterix.common.exceptions.CompilationException; import org.apache.asterix.common.exceptions.ErrorCode; import org.apache.asterix.common.external.IExternalFilterEvaluator; +import org.apache.asterix.dataflow.data.nontagged.serde.AStringSerializerDeserializer; import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.ATypeTag; import org.apache.asterix.om.types.BuiltinType; @@ -47,7 +49,9 @@ import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.exceptions.IWarningCollector; import org.apache.hyracks.api.exceptions.Warning; +import org.apache.hyracks.data.std.util.ArrayBackedValueStorage; import org.apache.hyracks.util.LogRedactionUtil; +import org.apache.hyracks.util.string.UTF8StringWriter; public final class ExternalDataPrefix implements Serializable { private static final long serialVersionUID = -7612997190679310483L; @@ -277,6 +281,12 @@ String computedFieldName = null; IAType computedFieldType = null; String computedFieldValue = null; + + // TODO(htowaileb): this is temporary workaround, we should do something better. + // Added here to avoid serialization issues + ArrayBackedValueStorage arrayBackedStorage = new ArrayBackedValueStorage(); + AStringSerializerDeserializer stringSerDer = new AStringSerializerDeserializer(new UTF8StringWriter(), null); + try { for (int i = 0; i < computedFieldNames.size(); i++) { computedFieldName = computedFieldNames.get(i); @@ -285,6 +295,11 @@ if (evaluator.isComputedFieldUsed(i)) { evaluator.setValue(i, computedFieldValue); + } else { + // TODO(htowaileb): This is temporary workaround, we should do something better + // although the computed field is not used, we need to validate since this will go to the value embedder + arrayBackedStorage.reset(); + writeValue(computedFieldType.getTypeTag(), computedFieldValue, arrayBackedStorage, stringSerDer); } } } catch (NumberFormatException ex) { -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17884 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: I18e90d1dae8f61801d4467bcad02a0e13aed6c9d Gerrit-Change-Number: 17884 Gerrit-PatchSet: 1 Gerrit-Owner: Hussain Towaileb <[email protected]> Gerrit-MessageType: newchange
