>From Hussain Towaileb <[email protected]>: Hussain Towaileb has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17896 )
Change subject: [ASTERIXDB-3294][EXT]: Fail if there are conflicting types for computed fields ...................................................................... [ASTERIXDB-3294][EXT]: Fail if there are conflicting types for computed fields Change-Id: I936c7afbf7e95271ede2230c08c7841a27995a21 --- A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/conflicted-fields/test.002.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/conflicted-fields/test.001.ddl.sqlpp M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/conflicted-fields/test.999.ddl.sqlpp M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml M asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/conflicted-fields/test.000.ddl.sqlpp 9 files changed, 178 insertions(+), 17 deletions(-) git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/96/17896/1 diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/conflicted-fields/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/conflicted-fields/test.000.ddl.sqlpp new file mode 100644 index 0000000..0e265e3 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/conflicted-fields/test.000.ddl.sqlpp @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +DROP DATAVERSE test IF EXISTS; +CREATE DATAVERSE test; +USE test; + +CREATE TYPE test AS { +}; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/conflicted-fields/test.001.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/conflicted-fields/test.001.ddl.sqlpp new file mode 100644 index 0000000..14230dc --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/conflicted-fields/test.001.ddl.sqlpp @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +USE test; + +CREATE EXTERNAL DATASET test1(test) USING %adapter% ( + %template%, + ("container"="dynamic-prefix-at-start-container"), + ("definition"="{person.name.first:string}/{person.name:string}"), + ("embed-filter-values" = "true"), + ("format"="json") +); + diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/conflicted-fields/test.002.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/conflicted-fields/test.002.ddl.sqlpp new file mode 100644 index 0000000..18a03a1 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/conflicted-fields/test.002.ddl.sqlpp @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +USE test; + +CREATE EXTERNAL DATASET test1(test) USING %adapter% ( + %template%, + ("container"="dynamic-prefix-at-start-container"), + ("definition"="{person.name:string}/{person.name.first:string}"), + ("embed-filter-values" = "true"), + ("format"="json") +); + diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/conflicted-fields/test.999.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/conflicted-fields/test.999.ddl.sqlpp new file mode 100644 index 0000000..36b2bab --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/conflicted-fields/test.999.ddl.sqlpp @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +DROP DATAVERSE test IF EXISTS; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml index 7322b1a..89b1de9 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml @@ -284,13 +284,13 @@ <compilation-unit name="type-mismatch"> <placeholder name="adapter" value="AZUREBLOB" /> <output-dir compare="Text">type-mismatch</output-dir> - <expected-warn>Failed to evaluate computed field. File: 'external-filter/department/accounting/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'accounting'. Reason: 'java.lang.NumberFormatException: For input string: "accounting"'</expected-warn> - <expected-warn>Failed to evaluate computed field. File: 'external-filter/department/engineering/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'engineering'. Reason: 'java.lang.NumberFormatException: For input string: "engineering"'</expected-warn> - <expected-warn>Failed to evaluate computed field. File: 'external-filter/department/hr/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'hr'. Reason: 'java.lang.NumberFormatException: For input string: "hr"'</expected-warn> + <expected-warn>In the file 'external-filter/department/accounting/0.json' the expected type for the 'name' field is 'bigint' but the value 'accounting' was found instead.</expected-warn> + <expected-warn>In the file 'external-filter/department/engineering/0.json' the expected type for the 'name' field is 'bigint' but the value 'engineering' was found instead.</expected-warn> + <expected-warn>In the file 'external-filter/department/hr/0.json' the expected type for the 'name' field is 'bigint' but the value 'hr' was found instead.</expected-warn> <expected-warn>The provided external dataset configuration returned no files from the external source</expected-warn> - <expected-warn>Failed to evaluate computed field. File: 'external-filter/department/accounting/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'accounting'. Reason: 'java.lang.NumberFormatException: For input string: "accounting"'</expected-warn> - <expected-warn>Failed to evaluate computed field. File: 'external-filter/department/engineering/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'engineering'. Reason: 'java.lang.NumberFormatException: For input string: "engineering"'</expected-warn> - <expected-warn>Failed to evaluate computed field. File: 'external-filter/department/hr/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'hr'. Reason: 'java.lang.NumberFormatException: For input string: "hr"'</expected-warn> + <expected-warn>In the file 'external-filter/department/accounting/0.json' the expected type for the 'name' field is 'bigint' but the value 'accounting' was found instead.</expected-warn> + <expected-warn>In the file 'external-filter/department/engineering/0.json' the expected type for the 'name' field is 'bigint' but the value 'engineering' was found instead.</expected-warn> + <expected-warn>In the file 'external-filter/department/hr/0.json' the expected type for the 'name' field is 'bigint' but the value 'hr' was found instead.</expected-warn> <expected-warn>The provided external dataset configuration returned no files from the external source</expected-warn> </compilation-unit> </test-case> @@ -359,6 +359,14 @@ <expected-error>Duplicate field name 'a.b.c.d'</expected-error> </compilation-unit> </test-case> + <test-case FilePath="external-dataset/common/dynamic-prefixes"> + <compilation-unit name="conflicted-fields"> + <placeholder name="adapter" value="AZUREBLOB" /> + <output-dir compare="Text">conflicted-fields</output-dir> + <expected-error>A computed field cannot have more than one type. 'person.name' is both 'string' and 'object'.</expected-error> + <expected-error>A computed field cannot have more than one type. 'person.name' is both 'string' and 'object'.</expected-error> + </compilation-unit> + </test-case> <!-- Dynamic prefixes tests end --> <test-case FilePath="external-dataset"> <compilation-unit name="common/empty-string-definition"> diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml index 5b53b41..00a62f2 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml @@ -291,13 +291,13 @@ <compilation-unit name="type-mismatch"> <placeholder name="adapter" value="S3" /> <output-dir compare="Text">type-mismatch</output-dir> - <expected-warn>Failed to evaluate computed field. File: 'external-filter/department/accounting/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'accounting'. Reason: 'java.lang.NumberFormatException: For input string: "accounting"'</expected-warn> - <expected-warn>Failed to evaluate computed field. File: 'external-filter/department/engineering/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'engineering'. Reason: 'java.lang.NumberFormatException: For input string: "engineering"'</expected-warn> - <expected-warn>Failed to evaluate computed field. File: 'external-filter/department/hr/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'hr'. Reason: 'java.lang.NumberFormatException: For input string: "hr"'</expected-warn> + <expected-warn>In the file 'external-filter/department/accounting/0.json' the expected type for the 'name' field is 'bigint' but the value 'accounting' was found instead.</expected-warn> + <expected-warn>In the file 'external-filter/department/engineering/0.json' the expected type for the 'name' field is 'bigint' but the value 'engineering' was found instead.</expected-warn> + <expected-warn>In the file 'external-filter/department/hr/0.json' the expected type for the 'name' field is 'bigint' but the value 'hr' was found instead.</expected-warn> <expected-warn>The provided external dataset configuration returned no files from the external source</expected-warn> - <expected-warn>Failed to evaluate computed field. File: 'external-filter/department/accounting/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'accounting'. Reason: 'java.lang.NumberFormatException: For input string: "accounting"'</expected-warn> - <expected-warn>Failed to evaluate computed field. File: 'external-filter/department/engineering/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'engineering'. Reason: 'java.lang.NumberFormatException: For input string: "engineering"'</expected-warn> - <expected-warn>Failed to evaluate computed field. File: 'external-filter/department/hr/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'hr'. Reason: 'java.lang.NumberFormatException: For input string: "hr"'</expected-warn> + <expected-warn>In the file 'external-filter/department/accounting/0.json' the expected type for the 'name' field is 'bigint' but the value 'accounting' was found instead.</expected-warn> + <expected-warn>In the file 'external-filter/department/engineering/0.json' the expected type for the 'name' field is 'bigint' but the value 'engineering' was found instead.</expected-warn> + <expected-warn>In the file 'external-filter/department/hr/0.json' the expected type for the 'name' field is 'bigint' but the value 'hr' was found instead.</expected-warn> <expected-warn>The provided external dataset configuration returned no files from the external source</expected-warn> </compilation-unit> </test-case> @@ -386,6 +386,14 @@ <expected-error>Duplicate field name 'a.b.c.d'</expected-error> </compilation-unit> </test-case> + <test-case FilePath="external-dataset/common/dynamic-prefixes"> + <compilation-unit name="conflicted-fields"> + <placeholder name="adapter" value="S3" /> + <output-dir compare="Text">conflicted-fields</output-dir> + <expected-error>A computed field cannot have more than one type. 'person.name' is both 'string' and 'object'.</expected-error> + <expected-error>A computed field cannot have more than one type. 'person.name' is both 'string' and 'object'.</expected-error> + </compilation-unit> + </test-case> <test-case FilePath="external-dataset/common/dynamic-prefixes/parquet"> <compilation-unit name="one-field"> <placeholder name="adapter" value="S3" /> @@ -396,9 +404,9 @@ <compilation-unit name="type-mismatch"> <placeholder name="adapter" value="S3" /> <output-dir compare="Text">type-mismatch</output-dir> - <expected-warn>Failed to evaluate computed field. File: 'parquet-data/external-filter/department/accounting/0.parquet'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'accounting'. Reason: 'java.lang.NumberFormatException: For input string: "accounting"'</expected-warn> - <expected-warn>Failed to evaluate computed field. File: 'parquet-data/external-filter/department/engineering/0.parquet'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'engineering'. Reason: 'java.lang.NumberFormatException: For input string: "engineering"'</expected-warn> - <expected-warn>Failed to evaluate computed field. File: 'parquet-data/external-filter/department/hr/0.parquet'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'hr'. Reason: 'java.lang.NumberFormatException: For input string: "hr"'</expected-warn> + <expected-warn>In the file 'parquet-data/external-filter/department/accounting/0.parquet' the expected type for the 'name' field is 'bigint' but the value 'accounting' was found instead.</expected-warn> + <expected-warn>In the file 'parquet-data/external-filter/department/engineering/0.parquet' the expected type for the 'name' field is 'bigint' but the value 'engineering' was found instead.</expected-warn> + <expected-warn>In the file 'parquet-data/external-filter/department/hr/0.parquet' the expected type for the 'name' field is 'bigint' but the value 'hr' was found instead.</expected-warn> <expected-warn>The provided external dataset configuration returned no files from the external source</expected-warn> </compilation-unit> </test-case> diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java index 0c3d8ca..fd308b2 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java @@ -288,6 +288,7 @@ CANNOT_DROP_DATABASE_DEPENDENT_EXISTS(1187), UNSUPPORTED_WRITING_ADAPTER(1188), UNSUPPORTED_WRITING_FORMAT(1189), + COMPUTED_FIELD_CHILD_OF_ANOTHER_COMPUTED_FIELD(1190), // Feed errors DATAFLOW_ILLEGAL_STATE(3001), diff --git a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties index 1d48e1e..0da70b2 100644 --- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties +++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties @@ -283,13 +283,14 @@ 1180 = Error reading iceberg data 1181 = Unsupported computed field type: '%1$s' 1182 = Failed to calculate computed fields: %1$s -1183 = Failed to evaluate computed field. File: '%1$s'. Computed Field Name: '%2$s'. Computed Field Type: '%3$s'. Computed Field Value: '%4$s'. Reason: '%5$s' +1183 = In the file '%1$s' the expected type for the '%2$s' field is '%3$s' but the value '%4$s' was found instead. 1184 = Compilation error: %1$s: %2$s dataset is not supported on datasets with meta records 1185 = Cannot find database with name %1$s 1186 = A database with this name %1$s already exists 1187 = Cannot drop database: %1$s %2$s being used by %3$s %4$s 1188 = Unsupported writing adapter '%1$s'. Supported adapters: %2$s 1189 = Unsupported writing format '%1$s'. Supported formats: %2$s +1190 = A computed field cannot have more than one type. '%1$s' is both '%2$s' and 'object'. # Feed Errors 3001 = Illegal state. diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java index 7eefc8a..30a61a7 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java @@ -164,10 +164,12 @@ if (type == null) { throw new CompilationException(ErrorCode.UNSUPPORTED_COMPUTED_FIELD_TYPE, typePart); } - type = getUpdatedType(type); validateSupported(type.getTypeTag()); + // ensure not child field + validateIfChildField(namePart, type.getTypeTag()); + computedFieldNames.add(namePart); computedFieldTypes.add(type); computedFieldSegmentIndexes.add(i); @@ -188,6 +190,35 @@ } } + /** + * This will ensure that the incoming computed field is not a child of an existing computed field. For example: + * existing computed field: {person.name:string} + * incoming computed field: {person.name.first:string} + * <p> + * This should fail as person.name.first is expecting person.name to be an object, but it is a string + * + * @param name computed field name + * @param type computed field type + * @throws CompilationException CompilationException + */ + private void validateIfChildField(String name, ATypeTag type) throws CompilationException { + for (int i = 0; i < computedFieldNames.size(); i++) { + String existingName = computedFieldNames.get(i); + ATypeTag existingType = computedFieldTypes.get(i).getTypeTag(); + + // We do the "." check because the following should be fine: + // "person.name_first" starts with "person.name" but these are fine to co-exist + if (name.startsWith(existingName) && name.charAt(existingName.length()) == '.') { + throw new CompilationException(ErrorCode.COMPUTED_FIELD_CHILD_OF_ANOTHER_COMPUTED_FIELD, existingName, + existingType); + } + + if (existingName.startsWith(name) && existingName.charAt(name.length()) == '.') { + throw new CompilationException(ErrorCode.COMPUTED_FIELD_CHILD_OF_ANOTHER_COMPUTED_FIELD, name, type); + } + } + } + private void updateIndexToComputedFieldMap(int segmentIndex, String computedFieldName, IAType computedFieldType) { if (indexToComputedFieldsMap.containsKey(segmentIndex)) { PrefixSegment prefixSegment = indexToComputedFieldsMap.get(segmentIndex); -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17896 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: I936c7afbf7e95271ede2230c08c7841a27995a21 Gerrit-Change-Number: 17896 Gerrit-PatchSet: 1 Gerrit-Owner: Hussain Towaileb <[email protected]> Gerrit-MessageType: newchange
