This is an automated email from the ASF dual-hosted git repository.
htowaileb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 6189803b39 [ASTERIXDB-3270][EXT]: Properly handle unsupported computed
field types
6189803b39 is described below
commit 6189803b395f9c77c8ea9872791acacf1536b2a7
Author: Hussain Towaileb <[email protected]>
AuthorDate: Tue Sep 26 11:01:25 2023 +0300
[ASTERIXDB-3270][EXT]: Properly handle unsupported computed field types
Change-Id: If78ccda42fde738c6eba67e095358edcc1984b4d
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17809
Integration-Tests: Jenkins <[email protected]>
Reviewed-by: Hussain Towaileb <[email protected]>
Reviewed-by: Murtadha Hubail <[email protected]>
Tested-by: Jenkins <[email protected]>
---
.../test.000.ddl.sqlpp | 33 ++++++++++++++++++++++
.../test.001.ddl.sqlpp | 33 ++++++++++++++++++++++
.../test.999.ddl.sqlpp | 20 +++++++++++++
.../test.000.ddl.sqlpp | 33 ++++++++++++++++++++++
.../test.001.ddl.sqlpp | 33 ++++++++++++++++++++++
.../test.999.ddl.sqlpp | 20 +++++++++++++
...stsuite_external_dataset_azure_blob_storage.xml | 16 +++++++++++
.../runtimets/testsuite_external_dataset_s3.xml | 16 +++++++++++
.../src/main/resources/asx_errormsg/en.properties | 2 +-
.../asterix/external/util/ExternalDataPrefix.java | 3 ++
.../util/azure/blob_storage/AzureUtils.java | 7 +++++
.../asterix/external/util/google/gcs/GCSUtils.java | 8 ++++++
12 files changed, 223 insertions(+), 1 deletion(-)
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-unsupported-type/test.000.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-unsupported-type/test.000.ddl.sqlpp
new file mode 100644
index 0000000000..161cd14625
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-unsupported-type/test.000.ddl.sqlpp
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+USE test;
+
+CREATE TYPE test AS {
+};
+
+CREATE EXTERNAL DATASET test1(test) USING %adapter% (
+ %template%,
+ ("container"="playground"),
+
("definition"="external-filter/computed-field-between-static-parts-1/department/{name:polygon}/students"),
+ ("embed-filter-values" = "true"),
+ ("format"="json")
+);
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-unsupported-type/test.001.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-unsupported-type/test.001.ddl.sqlpp
new file mode 100644
index 0000000000..c50b7930ef
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-unsupported-type/test.001.ddl.sqlpp
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+USE test;
+
+CREATE TYPE test AS {
+};
+
+CREATE EXTERNAL DATASET test1(test) USING %adapter% (
+ %template%,
+ ("container"="playground"),
+
("definition"="external-filter/computed-field-between-static-parts-1/department/{name:zft}/students"),
+ ("embed-filter-values" = "true"),
+ ("format"="json")
+);
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-unsupported-type/test.999.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-unsupported-type/test.999.ddl.sqlpp
new file mode 100644
index 0000000000..36b2bab543
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-unsupported-type/test.999.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-unsupported-type/test.000.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-unsupported-type/test.000.ddl.sqlpp
new file mode 100644
index 0000000000..cebfbbdd2a
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-unsupported-type/test.000.ddl.sqlpp
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+USE test;
+
+CREATE TYPE test AS {
+};
+
+CREATE EXTERNAL DATASET test1(test) USING %adapter% (
+ %template%,
+ ("container"="playground"),
+
("definition"="parquet-data/external-filter/computed-field-between-static-parts-1/department/{name:polygon}/students"),
+ ("embed-filter-values" = "true"),
+ ("format"="parquet")
+);
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-unsupported-type/test.001.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-unsupported-type/test.001.ddl.sqlpp
new file mode 100644
index 0000000000..e4c0a0505f
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-unsupported-type/test.001.ddl.sqlpp
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+USE test;
+
+CREATE TYPE test AS {
+};
+
+CREATE EXTERNAL DATASET test1(test) USING %adapter% (
+ %template%,
+ ("container"="playground"),
+
("definition"="parquet-data/external-filter/computed-field-between-static-parts-1/department/{name:zft}/students"),
+ ("embed-filter-values" = "true"),
+ ("format"="parquet")
+);
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-unsupported-type/test.999.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-unsupported-type/test.999.ddl.sqlpp
new file mode 100644
index 0000000000..36b2bab543
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-unsupported-type/test.999.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
index 5809912a46..fc73e9e8da 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
@@ -308,6 +308,14 @@
<output-dir compare="Text">computed-field-at-start</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="external-dataset/common/dynamic-prefixes">
+ <compilation-unit name="computed-field-unsupported-type">
+ <placeholder name="adapter" value="AZUREBLOB" />
+ <output-dir compare="Text">computed-field-unsupported-type</output-dir>
+ <expected-error>Unsupported computed field type:
'polygon'</expected-error>
+ <expected-error>Unsupported computed field type: 'zft'</expected-error>
+ </compilation-unit>
+ </test-case>
<!--
<test-case FilePath="external-dataset/common/dynamic-prefixes/parquet">
<compilation-unit name="computed-field-segment-pattern-mismatch">
@@ -316,6 +324,14 @@
</compilation-unit>
</test-case>
-->
+ <test-case FilePath="external-dataset/common/dynamic-prefixes">
+ <compilation-unit name="computed-field-unsupported-type">
+ <placeholder name="adapter" value="AZUREBLOB" />
+ <output-dir compare="Text">computed-field-unsupported-type</output-dir>
+ <expected-error>Unsupported computed field type:
'polygon'</expected-error>
+ <expected-error>Unsupported computed field type: 'zft'</expected-error>
+ </compilation-unit>
+ </test-case>
<!-- Dynamic prefixes tests end -->
<test-case FilePath="external-dataset">
<compilation-unit name="common/empty-string-definition">
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
index 54758e3e6b..276c6020bd 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
@@ -280,6 +280,14 @@
<output-dir compare="Text">computed-field-at-start</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="external-dataset/common/dynamic-prefixes">
+ <compilation-unit name="computed-field-unsupported-type">
+ <placeholder name="adapter" value="S3" />
+ <output-dir compare="Text">computed-field-unsupported-type</output-dir>
+ <expected-error>Unsupported computed field type:
'polygon'</expected-error>
+ <expected-error>Unsupported computed field type: 'zft'</expected-error>
+ </compilation-unit>
+ </test-case>
<test-case FilePath="external-dataset/common/dynamic-prefixes/parquet">
<compilation-unit name="one-field">
<placeholder name="adapter" value="S3" />
@@ -338,6 +346,14 @@
<output-dir
compare="Text">computed-field-segment-pattern-mismatch</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="external-dataset/common/dynamic-prefixes/parquet">
+ <compilation-unit name="computed-field-unsupported-type">
+ <placeholder name="adapter" value="S3" />
+ <output-dir compare="Text">computed-field-unsupported-type</output-dir>
+ <expected-error>Unsupported computed field type:
'polygon'</expected-error>
+ <expected-error>Unsupported computed field type: 'zft'</expected-error>
+ </compilation-unit>
+ </test-case>
<!-- Dynamic prefixes tests end -->
<test-case FilePath="external-dataset">
<compilation-unit name="common/empty-string-definition">
diff --git
a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
index 72d118e0c7..3f61b85359 100644
--- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
+++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
@@ -277,7 +277,7 @@
1178 = Unsupported iceberg table
1179 = Unsupported iceberg format version
1180 = Error reading iceberg data
-1181 = Unsupported computed field type: %1$s
+1181 = Unsupported computed field type: '%1$s'
1182 = Failed to calculate computed fields: %1$s
1183 = Failed to evaluate computed field. File: '%1$s'. Computed Field Name:
'%2$s'. Computed Field Type: '%3$s'. Computed Field Value: '%4$s'. Reason:
'%5$s'
1184 = Compilation error: %1$s: %2$s dataset is not supported on datasets with
meta records
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
index 2edf326ff0..11de1554ae 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
@@ -155,6 +155,9 @@ public final class ExternalDataPrefix implements
Serializable {
String typePart = splits[1].substring(0,
splits[1].length() - 1);
IAType type = BuiltinTypeMap.getBuiltinType(typePart);
+ if (type == null) {
+ throw new
CompilationException(ErrorCode.UNSUPPORTED_COMPUTED_FIELD_TYPE, typePart);
+ }
type = getUpdatedType(type);
validateSupported(type.getTypeTag());
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob_storage/AzureUtils.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob_storage/AzureUtils.java
index 12502673f6..6330db248e 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob_storage/AzureUtils.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob_storage/AzureUtils.java
@@ -65,6 +65,7 @@ import org.apache.asterix.external.util.ExternalDataPrefix;
import org.apache.asterix.external.util.ExternalDataUtils;
import org.apache.asterix.external.util.HDFSUtils;
import org.apache.hadoop.mapred.JobConf;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.IWarningCollector;
import org.apache.hyracks.api.exceptions.SourceLocation;
@@ -534,6 +535,12 @@ public class AzureUtils {
}
validateIncludeExclude(configuration);
+ try {
+ // TODO(htowaileb): maybe something better, this will check to
ensure type is supported before creation
+ new ExternalDataPrefix(configuration);
+ } catch (AlgebricksException ex) {
+ throw new
CompilationException(ErrorCode.FAILED_TO_CALCULATE_COMPUTED_FIELDS, ex);
+ }
// Check if the bucket is present
BlobServiceClient blobServiceClient;
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSUtils.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSUtils.java
index bbd36de7d6..50c75ffbbf 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSUtils.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSUtils.java
@@ -53,6 +53,7 @@ import org.apache.asterix.external.util.ExternalDataPrefix;
import org.apache.asterix.external.util.ExternalDataUtils;
import org.apache.asterix.external.util.HDFSUtils;
import org.apache.hadoop.mapred.JobConf;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.IWarningCollector;
import org.apache.hyracks.api.exceptions.SourceLocation;
@@ -137,6 +138,13 @@ public class GCSUtils {
}
validateIncludeExclude(configuration);
+ try {
+ // TODO(htowaileb): maybe something better, this will check to
ensure type is supported before creation
+ new ExternalDataPrefix(configuration);
+ } catch (AlgebricksException ex) {
+ throw new
CompilationException(ErrorCode.FAILED_TO_CALCULATE_COMPUTED_FIELDS, ex);
+ }
+
String container =
configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
try {
