>From Hussain Towaileb <[email protected]>: Hussain Towaileb has submitted this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17809 )
Change subject: [ASTERIXDB-3270][EXT]: Properly handle unsupported computed field types ...................................................................... [ASTERIXDB-3270][EXT]: Properly handle unsupported computed field types Change-Id: If78ccda42fde738c6eba67e095358edcc1984b4d Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17809 Integration-Tests: Jenkins <[email protected]> Reviewed-by: Hussain Towaileb <[email protected]> Reviewed-by: Murtadha Hubail <[email protected]> Tested-by: Jenkins <[email protected]> --- A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-unsupported-type/test.001.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-unsupported-type/test.000.ddl.sqlpp M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-unsupported-type/test.999.ddl.sqlpp M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob_storage/AzureUtils.java A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-unsupported-type/test.001.ddl.sqlpp M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSUtils.java A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-unsupported-type/test.000.ddl.sqlpp M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-unsupported-type/test.999.ddl.sqlpp M asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties 12 files changed, 237 insertions(+), 1 deletion(-) Approvals: Murtadha Hubail: Looks good to me, approved Hussain Towaileb: Looks good to me, but someone else must approve Jenkins: Verified; Verified diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-unsupported-type/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-unsupported-type/test.000.ddl.sqlpp new file mode 100644 index 0000000..161cd14 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-unsupported-type/test.000.ddl.sqlpp @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +DROP DATAVERSE test IF EXISTS; +CREATE DATAVERSE test; +USE test; + +CREATE TYPE test AS { +}; + +CREATE EXTERNAL DATASET test1(test) USING %adapter% ( + %template%, + ("container"="playground"), + ("definition"="external-filter/computed-field-between-static-parts-1/department/{name:polygon}/students"), + ("embed-filter-values" = "true"), + ("format"="json") +); diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-unsupported-type/test.001.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-unsupported-type/test.001.ddl.sqlpp new file mode 100644 index 0000000..c50b793 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-unsupported-type/test.001.ddl.sqlpp @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +DROP DATAVERSE test IF EXISTS; +CREATE DATAVERSE test; +USE test; + +CREATE TYPE test AS { +}; + +CREATE EXTERNAL DATASET test1(test) USING %adapter% ( + %template%, + ("container"="playground"), + ("definition"="external-filter/computed-field-between-static-parts-1/department/{name:zft}/students"), + ("embed-filter-values" = "true"), + ("format"="json") +); diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-unsupported-type/test.999.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-unsupported-type/test.999.ddl.sqlpp new file mode 100644 index 0000000..36b2bab --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-unsupported-type/test.999.ddl.sqlpp @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +DROP DATAVERSE test IF EXISTS; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-unsupported-type/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-unsupported-type/test.000.ddl.sqlpp new file mode 100644 index 0000000..cebfbbd --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-unsupported-type/test.000.ddl.sqlpp @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +DROP DATAVERSE test IF EXISTS; +CREATE DATAVERSE test; +USE test; + +CREATE TYPE test AS { +}; + +CREATE EXTERNAL DATASET test1(test) USING %adapter% ( + %template%, + ("container"="playground"), + ("definition"="parquet-data/external-filter/computed-field-between-static-parts-1/department/{name:polygon}/students"), + ("embed-filter-values" = "true"), + ("format"="parquet") +); diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-unsupported-type/test.001.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-unsupported-type/test.001.ddl.sqlpp new file mode 100644 index 0000000..e4c0a05 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-unsupported-type/test.001.ddl.sqlpp @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +DROP DATAVERSE test IF EXISTS; +CREATE DATAVERSE test; +USE test; + +CREATE TYPE test AS { +}; + +CREATE EXTERNAL DATASET test1(test) USING %adapter% ( + %template%, + ("container"="playground"), + ("definition"="parquet-data/external-filter/computed-field-between-static-parts-1/department/{name:zft}/students"), + ("embed-filter-values" = "true"), + ("format"="parquet") +); diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-unsupported-type/test.999.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-unsupported-type/test.999.ddl.sqlpp new file mode 100644 index 0000000..36b2bab --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-unsupported-type/test.999.ddl.sqlpp @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +DROP DATAVERSE test IF EXISTS; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml index 5809912..fc73e9e 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml @@ -308,6 +308,14 @@ <output-dir compare="Text">computed-field-at-start</output-dir> </compilation-unit> </test-case> + <test-case FilePath="external-dataset/common/dynamic-prefixes"> + <compilation-unit name="computed-field-unsupported-type"> + <placeholder name="adapter" value="AZUREBLOB" /> + <output-dir compare="Text">computed-field-unsupported-type</output-dir> + <expected-error>Unsupported computed field type: 'polygon'</expected-error> + <expected-error>Unsupported computed field type: 'zft'</expected-error> + </compilation-unit> + </test-case> <!-- <test-case FilePath="external-dataset/common/dynamic-prefixes/parquet"> <compilation-unit name="computed-field-segment-pattern-mismatch"> @@ -316,6 +324,14 @@ </compilation-unit> </test-case> --> + <test-case FilePath="external-dataset/common/dynamic-prefixes"> + <compilation-unit name="computed-field-unsupported-type"> + <placeholder name="adapter" value="AZUREBLOB" /> + <output-dir compare="Text">computed-field-unsupported-type</output-dir> + <expected-error>Unsupported computed field type: 'polygon'</expected-error> + <expected-error>Unsupported computed field type: 'zft'</expected-error> + </compilation-unit> + </test-case> <!-- Dynamic prefixes tests end --> <test-case FilePath="external-dataset"> <compilation-unit name="common/empty-string-definition"> diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml index 54758e3..276c602 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml @@ -280,6 +280,14 @@ <output-dir compare="Text">computed-field-at-start</output-dir> </compilation-unit> </test-case> + <test-case FilePath="external-dataset/common/dynamic-prefixes"> + <compilation-unit name="computed-field-unsupported-type"> + <placeholder name="adapter" value="S3" /> + <output-dir compare="Text">computed-field-unsupported-type</output-dir> + <expected-error>Unsupported computed field type: 'polygon'</expected-error> + <expected-error>Unsupported computed field type: 'zft'</expected-error> + </compilation-unit> + </test-case> <test-case FilePath="external-dataset/common/dynamic-prefixes/parquet"> <compilation-unit name="one-field"> <placeholder name="adapter" value="S3" /> @@ -338,6 +346,14 @@ <output-dir compare="Text">computed-field-segment-pattern-mismatch</output-dir> </compilation-unit> </test-case> + <test-case FilePath="external-dataset/common/dynamic-prefixes/parquet"> + <compilation-unit name="computed-field-unsupported-type"> + <placeholder name="adapter" value="S3" /> + <output-dir compare="Text">computed-field-unsupported-type</output-dir> + <expected-error>Unsupported computed field type: 'polygon'</expected-error> + <expected-error>Unsupported computed field type: 'zft'</expected-error> + </compilation-unit> + </test-case> <!-- Dynamic prefixes tests end --> <test-case FilePath="external-dataset"> <compilation-unit name="common/empty-string-definition"> diff --git a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties index 72d118e..3f61b85 100644 --- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties +++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties @@ -277,7 +277,7 @@ 1178 = Unsupported iceberg table 1179 = Unsupported iceberg format version 1180 = Error reading iceberg data -1181 = Unsupported computed field type: %1$s +1181 = Unsupported computed field type: '%1$s' 1182 = Failed to calculate computed fields: %1$s 1183 = Failed to evaluate computed field. File: '%1$s'. Computed Field Name: '%2$s'. Computed Field Type: '%3$s'. Computed Field Value: '%4$s'. Reason: '%5$s' 1184 = Compilation error: %1$s: %2$s dataset is not supported on datasets with meta records diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java index 2edf326..11de155 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java @@ -155,6 +155,9 @@ String typePart = splits[1].substring(0, splits[1].length() - 1); IAType type = BuiltinTypeMap.getBuiltinType(typePart); + if (type == null) { + throw new CompilationException(ErrorCode.UNSUPPORTED_COMPUTED_FIELD_TYPE, typePart); + } type = getUpdatedType(type); validateSupported(type.getTypeTag()); diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob_storage/AzureUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob_storage/AzureUtils.java index 1250267..6330db2 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob_storage/AzureUtils.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob_storage/AzureUtils.java @@ -65,6 +65,7 @@ import org.apache.asterix.external.util.ExternalDataUtils; import org.apache.asterix.external.util.HDFSUtils; import org.apache.hadoop.mapred.JobConf; +import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.exceptions.IWarningCollector; import org.apache.hyracks.api.exceptions.SourceLocation; @@ -534,6 +535,12 @@ } validateIncludeExclude(configuration); + try { + // TODO(htowaileb): maybe something better, this will check to ensure type is supported before creation + new ExternalDataPrefix(configuration); + } catch (AlgebricksException ex) { + throw new CompilationException(ErrorCode.FAILED_TO_CALCULATE_COMPUTED_FIELDS, ex); + } // Check if the bucket is present BlobServiceClient blobServiceClient; diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSUtils.java index bbd36de..50c75ff 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSUtils.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSUtils.java @@ -53,6 +53,7 @@ import org.apache.asterix.external.util.ExternalDataUtils; import org.apache.asterix.external.util.HDFSUtils; import org.apache.hadoop.mapred.JobConf; +import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.exceptions.IWarningCollector; import org.apache.hyracks.api.exceptions.SourceLocation; @@ -137,6 +138,13 @@ } validateIncludeExclude(configuration); + try { + // TODO(htowaileb): maybe something better, this will check to ensure type is supported before creation + new ExternalDataPrefix(configuration); + } catch (AlgebricksException ex) { + throw new CompilationException(ErrorCode.FAILED_TO_CALCULATE_COMPUTED_FIELDS, ex); + } + String container = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME); try { -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17809 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: If78ccda42fde738c6eba67e095358edcc1984b4d Gerrit-Change-Number: 17809 Gerrit-PatchSet: 4 Gerrit-Owner: Hussain Towaileb <[email protected]> Gerrit-Reviewer: Hussain Towaileb <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Murtadha Hubail <[email protected]> Gerrit-MessageType: merged
