>From Hussain Towaileb <[email protected]>: Hussain Towaileb has submitted this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17896 )
Change subject: [ASTERIXDB-3294][EXT]: Fail if there are conflicting types for computed fields ...................................................................... [ASTERIXDB-3294][EXT]: Fail if there are conflicting types for computed fields Change-Id: I936c7afbf7e95271ede2230c08c7841a27995a21 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17896 Reviewed-by: Hussain Towaileb <[email protected]> Reviewed-by: Wail Alkowaileet <[email protected]> Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> --- A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/conflicted-fields/test.002.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/conflicted-fields/test.001.ddl.sqlpp M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/duplicate-field/test.005.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/conflicted-fields/test.999.ddl.sqlpp M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java M asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/duplicate-field/test.005.ddl.sqlpp M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml M asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/conflicted-fields/test.000.ddl.sqlpp 11 files changed, 213 insertions(+), 27 deletions(-) Approvals: Wail Alkowaileet: Looks good to me, approved Hussain Towaileb: Looks good to me, but someone else must approve Jenkins: Verified; Verified diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/conflicted-fields/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/conflicted-fields/test.000.ddl.sqlpp new file mode 100644 index 0000000..0e265e3 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/conflicted-fields/test.000.ddl.sqlpp @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +DROP DATAVERSE test IF EXISTS; +CREATE DATAVERSE test; +USE test; + +CREATE TYPE test AS { +}; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/conflicted-fields/test.001.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/conflicted-fields/test.001.ddl.sqlpp new file mode 100644 index 0000000..14230dc --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/conflicted-fields/test.001.ddl.sqlpp @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +USE test; + +CREATE EXTERNAL DATASET test1(test) USING %adapter% ( + %template%, + ("container"="dynamic-prefix-at-start-container"), + ("definition"="{person.name.first:string}/{person.name:string}"), + ("embed-filter-values" = "true"), + ("format"="json") +); + diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/conflicted-fields/test.002.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/conflicted-fields/test.002.ddl.sqlpp new file mode 100644 index 0000000..18a03a1 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/conflicted-fields/test.002.ddl.sqlpp @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +USE test; + +CREATE EXTERNAL DATASET test1(test) USING %adapter% ( + %template%, + ("container"="dynamic-prefix-at-start-container"), + ("definition"="{person.name:string}/{person.name.first:string}"), + ("embed-filter-values" = "true"), + ("format"="json") +); + diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/conflicted-fields/test.999.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/conflicted-fields/test.999.ddl.sqlpp new file mode 100644 index 0000000..36b2bab --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/conflicted-fields/test.999.ddl.sqlpp @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +DROP DATAVERSE test IF EXISTS; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/duplicate-field/test.005.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/duplicate-field/test.005.ddl.sqlpp index 1c6fa81..1316e3f 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/duplicate-field/test.005.ddl.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/duplicate-field/test.005.ddl.sqlpp @@ -22,7 +22,7 @@ CREATE EXTERNAL DATASET test1(test) USING %adapter% ( %template%, ("container"="dynamic-prefix-at-start-container"), - ("definition"="{a.b:int}/foo/{a.b.c:int}/{a.b.c.d:int}-{a.b.c.d:int}/"), + ("definition"="{a.b:int}/foo/{a.b:string}"), ("embed-filter-values" = "true"), ("format"="json") ); diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/duplicate-field/test.005.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/duplicate-field/test.005.ddl.sqlpp index 0e1157f..03b043a 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/duplicate-field/test.005.ddl.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/duplicate-field/test.005.ddl.sqlpp @@ -22,7 +22,7 @@ CREATE EXTERNAL DATASET test1(test) USING %adapter% ( %template%, ("container"="dynamic-prefix-at-start-container"), - ("definition"="{a.b:int}/foo/{a.b.c:int}/{a.b.c.d:int}-{a.b.c.d:int}/"), + ("definition"="{a.b:int}/foo/{a.b:string}"), ("embed-filter-values" = "true"), ("format"="parquet") ); diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml index 7322b1a..af0ae36 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml @@ -284,13 +284,13 @@ <compilation-unit name="type-mismatch"> <placeholder name="adapter" value="AZUREBLOB" /> <output-dir compare="Text">type-mismatch</output-dir> - <expected-warn>Failed to evaluate computed field. File: 'external-filter/department/accounting/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'accounting'. Reason: 'java.lang.NumberFormatException: For input string: "accounting"'</expected-warn> - <expected-warn>Failed to evaluate computed field. File: 'external-filter/department/engineering/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'engineering'. Reason: 'java.lang.NumberFormatException: For input string: "engineering"'</expected-warn> - <expected-warn>Failed to evaluate computed field. File: 'external-filter/department/hr/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'hr'. Reason: 'java.lang.NumberFormatException: For input string: "hr"'</expected-warn> + <expected-warn>In the file 'external-filter/department/accounting/0.json' the expected type for the 'name' field is 'bigint' but the value 'accounting' was found instead.</expected-warn> + <expected-warn>In the file 'external-filter/department/engineering/0.json' the expected type for the 'name' field is 'bigint' but the value 'engineering' was found instead.</expected-warn> + <expected-warn>In the file 'external-filter/department/hr/0.json' the expected type for the 'name' field is 'bigint' but the value 'hr' was found instead.</expected-warn> <expected-warn>The provided external dataset configuration returned no files from the external source</expected-warn> - <expected-warn>Failed to evaluate computed field. File: 'external-filter/department/accounting/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'accounting'. Reason: 'java.lang.NumberFormatException: For input string: "accounting"'</expected-warn> - <expected-warn>Failed to evaluate computed field. File: 'external-filter/department/engineering/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'engineering'. Reason: 'java.lang.NumberFormatException: For input string: "engineering"'</expected-warn> - <expected-warn>Failed to evaluate computed field. File: 'external-filter/department/hr/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'hr'. Reason: 'java.lang.NumberFormatException: For input string: "hr"'</expected-warn> + <expected-warn>In the file 'external-filter/department/accounting/0.json' the expected type for the 'name' field is 'bigint' but the value 'accounting' was found instead.</expected-warn> + <expected-warn>In the file 'external-filter/department/engineering/0.json' the expected type for the 'name' field is 'bigint' but the value 'engineering' was found instead.</expected-warn> + <expected-warn>In the file 'external-filter/department/hr/0.json' the expected type for the 'name' field is 'bigint' but the value 'hr' was found instead.</expected-warn> <expected-warn>The provided external dataset configuration returned no files from the external source</expected-warn> </compilation-unit> </test-case> @@ -356,7 +356,15 @@ <expected-error>Duplicate field name 'year'</expected-error> <expected-error>Duplicate field name 'year'</expected-error> <expected-error>Duplicate field name 'year'</expected-error> - <expected-error>Duplicate field name 'a.b.c.d'</expected-error> + <expected-error>Duplicate field name 'a.b'</expected-error> + </compilation-unit> + </test-case> + <test-case FilePath="external-dataset/common/dynamic-prefixes"> + <compilation-unit name="conflicted-fields"> + <placeholder name="adapter" value="AZUREBLOB" /> + <output-dir compare="Text">conflicted-fields</output-dir> + <expected-error>A computed field cannot have more than one type. 'person.name' is both 'string' and 'object'.</expected-error> + <expected-error>A computed field cannot have more than one type. 'person.name' is both 'string' and 'object'.</expected-error> </compilation-unit> </test-case> <!-- Dynamic prefixes tests end --> diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml index 5b53b41..bd9e6ca 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml @@ -291,13 +291,13 @@ <compilation-unit name="type-mismatch"> <placeholder name="adapter" value="S3" /> <output-dir compare="Text">type-mismatch</output-dir> - <expected-warn>Failed to evaluate computed field. File: 'external-filter/department/accounting/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'accounting'. Reason: 'java.lang.NumberFormatException: For input string: "accounting"'</expected-warn> - <expected-warn>Failed to evaluate computed field. File: 'external-filter/department/engineering/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'engineering'. Reason: 'java.lang.NumberFormatException: For input string: "engineering"'</expected-warn> - <expected-warn>Failed to evaluate computed field. File: 'external-filter/department/hr/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'hr'. Reason: 'java.lang.NumberFormatException: For input string: "hr"'</expected-warn> + <expected-warn>In the file 'external-filter/department/accounting/0.json' the expected type for the 'name' field is 'bigint' but the value 'accounting' was found instead.</expected-warn> + <expected-warn>In the file 'external-filter/department/engineering/0.json' the expected type for the 'name' field is 'bigint' but the value 'engineering' was found instead.</expected-warn> + <expected-warn>In the file 'external-filter/department/hr/0.json' the expected type for the 'name' field is 'bigint' but the value 'hr' was found instead.</expected-warn> <expected-warn>The provided external dataset configuration returned no files from the external source</expected-warn> - <expected-warn>Failed to evaluate computed field. File: 'external-filter/department/accounting/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'accounting'. Reason: 'java.lang.NumberFormatException: For input string: "accounting"'</expected-warn> - <expected-warn>Failed to evaluate computed field. File: 'external-filter/department/engineering/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'engineering'. Reason: 'java.lang.NumberFormatException: For input string: "engineering"'</expected-warn> - <expected-warn>Failed to evaluate computed field. File: 'external-filter/department/hr/0.json'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'hr'. Reason: 'java.lang.NumberFormatException: For input string: "hr"'</expected-warn> + <expected-warn>In the file 'external-filter/department/accounting/0.json' the expected type for the 'name' field is 'bigint' but the value 'accounting' was found instead.</expected-warn> + <expected-warn>In the file 'external-filter/department/engineering/0.json' the expected type for the 'name' field is 'bigint' but the value 'engineering' was found instead.</expected-warn> + <expected-warn>In the file 'external-filter/department/hr/0.json' the expected type for the 'name' field is 'bigint' but the value 'hr' was found instead.</expected-warn> <expected-warn>The provided external dataset configuration returned no files from the external source</expected-warn> </compilation-unit> </test-case> @@ -383,7 +383,15 @@ <expected-error>Duplicate field name 'year'</expected-error> <expected-error>Duplicate field name 'year'</expected-error> <expected-error>Duplicate field name 'year'</expected-error> - <expected-error>Duplicate field name 'a.b.c.d'</expected-error> + <expected-error>Duplicate field name 'a.b'</expected-error> + </compilation-unit> + </test-case> + <test-case FilePath="external-dataset/common/dynamic-prefixes"> + <compilation-unit name="conflicted-fields"> + <placeholder name="adapter" value="S3" /> + <output-dir compare="Text">conflicted-fields</output-dir> + <expected-error>A computed field cannot have more than one type. 'person.name' is both 'string' and 'object'.</expected-error> + <expected-error>A computed field cannot have more than one type. 'person.name' is both 'string' and 'object'.</expected-error> </compilation-unit> </test-case> <test-case FilePath="external-dataset/common/dynamic-prefixes/parquet"> @@ -396,9 +404,9 @@ <compilation-unit name="type-mismatch"> <placeholder name="adapter" value="S3" /> <output-dir compare="Text">type-mismatch</output-dir> - <expected-warn>Failed to evaluate computed field. File: 'parquet-data/external-filter/department/accounting/0.parquet'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'accounting'. Reason: 'java.lang.NumberFormatException: For input string: "accounting"'</expected-warn> - <expected-warn>Failed to evaluate computed field. File: 'parquet-data/external-filter/department/engineering/0.parquet'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'engineering'. Reason: 'java.lang.NumberFormatException: For input string: "engineering"'</expected-warn> - <expected-warn>Failed to evaluate computed field. File: 'parquet-data/external-filter/department/hr/0.parquet'. Computed Field Name: 'name'. Computed Field Type: 'bigint'. Computed Field Value: 'hr'. Reason: 'java.lang.NumberFormatException: For input string: "hr"'</expected-warn> + <expected-warn>In the file 'parquet-data/external-filter/department/accounting/0.parquet' the expected type for the 'name' field is 'bigint' but the value 'accounting' was found instead.</expected-warn> + <expected-warn>In the file 'parquet-data/external-filter/department/engineering/0.parquet' the expected type for the 'name' field is 'bigint' but the value 'engineering' was found instead.</expected-warn> + <expected-warn>In the file 'parquet-data/external-filter/department/hr/0.parquet' the expected type for the 'name' field is 'bigint' but the value 'hr' was found instead.</expected-warn> <expected-warn>The provided external dataset configuration returned no files from the external source</expected-warn> </compilation-unit> </test-case> @@ -458,7 +466,7 @@ <expected-error>Duplicate field name 'year'</expected-error> <expected-error>Duplicate field name 'year'</expected-error> <expected-error>Duplicate field name 'year'</expected-error> - <expected-error>Duplicate field name 'a.b.c.d'</expected-error> + <expected-error>Duplicate field name 'a.b'</expected-error> </compilation-unit> </test-case> <test-case FilePath="external-dataset/common/dynamic-prefixes/csv"> diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java index 0c3d8ca..f780ee1 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java @@ -288,6 +288,7 @@ CANNOT_DROP_DATABASE_DEPENDENT_EXISTS(1187), UNSUPPORTED_WRITING_ADAPTER(1188), UNSUPPORTED_WRITING_FORMAT(1189), + COMPUTED_FIELD_CONFLICTING_TYPE(1190), // Feed errors DATAFLOW_ILLEGAL_STATE(3001), diff --git a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties index 1d48e1e..0da70b2 100644 --- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties +++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties @@ -283,13 +283,14 @@ 1180 = Error reading iceberg data 1181 = Unsupported computed field type: '%1$s' 1182 = Failed to calculate computed fields: %1$s -1183 = Failed to evaluate computed field. File: '%1$s'. Computed Field Name: '%2$s'. Computed Field Type: '%3$s'. Computed Field Value: '%4$s'. Reason: '%5$s' +1183 = In the file '%1$s' the expected type for the '%2$s' field is '%3$s' but the value '%4$s' was found instead. 1184 = Compilation error: %1$s: %2$s dataset is not supported on datasets with meta records 1185 = Cannot find database with name %1$s 1186 = A database with this name %1$s already exists 1187 = Cannot drop database: %1$s %2$s being used by %3$s %4$s 1188 = Unsupported writing adapter '%1$s'. Supported adapters: %2$s 1189 = Unsupported writing format '%1$s'. Supported formats: %2$s +1190 = A computed field cannot have more than one type. '%1$s' is both '%2$s' and 'object'. # Feed Errors 3001 = Illegal state. diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java index 7eefc8a..0ed0622 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java @@ -57,6 +57,7 @@ private final List<String> segments; private final List<String> computedFieldNames = new ArrayList<>(); + private final Map<String, ATypeTag> computedFieldsParts = new HashMap<>(); private final List<IAType> computedFieldTypes = new ArrayList<>(); private final List<Integer> computedFieldSegmentIndexes = new ArrayList<>(); private final List<ARecordType> paths = new ArrayList<>(); @@ -154,26 +155,24 @@ String namePart = splits[0].substring(1); String typePart = splits[1].substring(0, splits[1].length() - 1); - // ensure no duplicate fields - if (computedFieldNames.contains(namePart)) { - throw new CompilationException(ErrorCode.DUPLICATE_FIELD_NAME, namePart); - } - // ensure supported type IAType type = BuiltinTypeMap.getBuiltinType(typePart); if (type == null) { throw new CompilationException(ErrorCode.UNSUPPORTED_COMPUTED_FIELD_TYPE, typePart); } - type = getUpdatedType(type); validateSupported(type.getTypeTag()); + // ensure no issues with the incoming computed field + validateConflictingFields(namePart, type.getTypeTag()); + computedFieldNames.add(namePart); computedFieldTypes.add(type); computedFieldSegmentIndexes.add(i); updateIndexToComputedFieldMap(i, namePart, type); List<String> nameParts = List.of(namePart.split("\\.")); + addNameParts(nameParts, type.getTypeTag()); paths.add(ProjectionFiltrationTypeUtil.getPathRecordType(nameParts)); expression.append("(.+)"); @@ -188,6 +187,58 @@ } } + /** + * Adds the computed field parts with their respective types. + * Given the following computed field: {person.name.first:string}, the following is added to the map: + * person -> object + * person.name -> object + * person.name.first -> string + * + * @param nameParts name parts of the computed field + * @param type type of the computed field + * @throws CompilationException CompilationException + */ + private void addNameParts(List<String> nameParts, ATypeTag type) throws CompilationException { + String concat = ""; + for (int i = 0; i < nameParts.size() - 1; i++) { + concat += nameParts.get(i); + + ATypeTag existingType = computedFieldsParts.get(concat); + if (existingType != null && existingType != ATypeTag.OBJECT) { + throw new CompilationException(ErrorCode.COMPUTED_FIELD_CONFLICTING_TYPE, concat, existingType); + } + computedFieldsParts.putIfAbsent(concat, ATypeTag.OBJECT); + concat += "."; + } + + concat += nameParts.get(nameParts.size() - 1); + computedFieldsParts.put(concat, type); + } + + /** + * This will ensure that the incoming computed field is not conflicting with an existing computed field. For example: + * existing computed field: {person.name:string} + * incoming computed field: {person.name.first:string} + * <p> + * This should fail as person.name.first is expecting person.name to be an object, but it is a string + * + * @param name computed field name + * @param type computed field type + * @throws CompilationException CompilationException + */ + private void validateConflictingFields(String name, ATypeTag type) throws CompilationException { + ATypeTag existingType = computedFieldsParts.get(name); + + // the provided computed field already exists, check if duplicate or conflicting + if (existingType != null) { + if (existingType == ATypeTag.OBJECT) { + throw new CompilationException(ErrorCode.COMPUTED_FIELD_CONFLICTING_TYPE, name, type); + } else { + throw new CompilationException(ErrorCode.DUPLICATE_FIELD_NAME, name); + } + } + } + private void updateIndexToComputedFieldMap(int segmentIndex, String computedFieldName, IAType computedFieldType) { if (indexToComputedFieldsMap.containsKey(segmentIndex)) { PrefixSegment prefixSegment = indexToComputedFieldsMap.get(segmentIndex); -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17896 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: I936c7afbf7e95271ede2230c08c7841a27995a21 Gerrit-Change-Number: 17896 Gerrit-PatchSet: 3 Gerrit-Owner: Hussain Towaileb <[email protected]> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Hussain Towaileb <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Wail Alkowaileet <[email protected]> Gerrit-MessageType: merged
