This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new ce2975bebe [ASTERIXDB-3310][COMP] Enforce supported types in columnar
collections
ce2975bebe is described below
commit ce2975bebe7456980a6fd502b680b5e03025cf23
Author: Wail Alkowaileet <[email protected]>
AuthorDate: Mon Dec 4 12:58:17 2023 -0800
[ASTERIXDB-3310][COMP] Enforce supported types in columnar collections
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
Ensure supported types when create a dataset and when
inserting/upserting records.
Change-Id: Ie2b5e2f0ad702be8f4cbcd1e81821144a5fe3c44
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17950
Integration-Tests: Jenkins <[email protected]>
Tested-by: Jenkins <[email protected]>
Reviewed-by: Wail Alkowaileet <[email protected]>
Reviewed-by: Ali Alsuliman <[email protected]>
---
.../asterix/optimizer/base/RuleCollections.java | 3 +
.../rules/EnsureColumnarSupportedTypesRule.java | 91 +++++++++++++++
.../asterix/app/translator/QueryTranslator.java | 58 +++++++---
.../supported-types/supported-types.001.ddl.sqlpp} | 38 ++-----
.../supported-types/supported-types.002.ddl.sqlpp} | 33 +-----
.../supported-types/supported-types.003.ddl.sqlpp} | 33 +-----
.../supported-types/supported-types.004.ddl.sqlpp} | 36 ++----
.../supported-types.005.update.sqlpp} | 32 +-----
.../supported-types.006.update.sqlpp} | 33 +-----
.../src/test/resources/runtimets/sqlpp_queries.xml | 10 ++
.../column/metadata/schema/UnionSchemaNode.java | 4 +-
.../visitor/ColumnSupportedTypesValidator.java | 124 +++++++++++++++++++++
.../operation/lsm/flush/FlushColumnMetadata.java | 15 +--
.../asterix/column/util/ColumnValuesUtil.java | 15 +++
.../asterix/common/exceptions/ErrorCode.java | 1 +
.../src/main/resources/asx_errormsg/en.properties | 1 +
16 files changed, 330 insertions(+), 197 deletions(-)
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
index 0a0aa5c89d..7a06f4c20d 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
@@ -42,6 +42,7 @@ import
org.apache.asterix.optimizer.rules.CleanupWriteOperatorRule;
import org.apache.asterix.optimizer.rules.ConstantFoldingRule;
import org.apache.asterix.optimizer.rules.CountVarToCountOneRule;
import org.apache.asterix.optimizer.rules.DisjunctivePredicateToJoinRule;
+import org.apache.asterix.optimizer.rules.EnsureColumnarSupportedTypesRule;
import
org.apache.asterix.optimizer.rules.ExtractBatchableExternalFunctionCallsRule;
import org.apache.asterix.optimizer.rules.ExtractDistinctByExpressionsRule;
import org.apache.asterix.optimizer.rules.ExtractOrderExpressionsRule;
@@ -206,6 +207,8 @@ public final class RuleCollections {
normalization.add(new ExtractDistinctByExpressionsRule());
normalization.add(new ExtractOrderExpressionsRule());
normalization.add(new ExtractWindowExpressionsRule());
+ // EnsureColumnarSupportedTypesRule should go before cast rules
+ normalization.add(new EnsureColumnarSupportedTypesRule());
// IntroduceStaticTypeCastRule should go before
// IntroduceDynamicTypeCastRule to
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/EnsureColumnarSupportedTypesRule.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/EnsureColumnarSupportedTypesRule.java
new file mode 100644
index 0000000000..8840c8fd42
--- /dev/null
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/EnsureColumnarSupportedTypesRule.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.optimizer.rules;
+
+import
org.apache.asterix.column.metadata.schema.visitor.ColumnSupportedTypesValidator;
+import org.apache.asterix.common.config.DatasetConfig;
+import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.metadata.declared.DataSource;
+import org.apache.asterix.metadata.declared.MetadataProvider;
+import org.apache.asterix.metadata.entities.Dataset;
+import org.apache.asterix.om.types.IAType;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import
org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import
org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteUpsertOperator;
+import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+import org.apache.hyracks.api.exceptions.SourceLocation;
+
+/**
+ * This rule enforces that inserted or upserted records do not contain
unsupported
+ * types for {@link DatasetConfig.DatasetFormat#COLUMN} datasets.
+ */
+public class EnsureColumnarSupportedTypesRule implements IAlgebraicRewriteRule
{
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef,
IOptimizationContext context)
+ throws AlgebricksException {
+ ILogicalOperator op = opRef.getValue();
+ if (op.getOperatorTag() != LogicalOperatorTag.INSERT_DELETE_UPSERT ||
context.checkIfInDontApplySet(this, op)) {
+ return false;
+ }
+
+ InsertDeleteUpsertOperator modOp = (InsertDeleteUpsertOperator) op;
+ // Do not apply the rule again to this operator
+ context.addToDontApplySet(this, op);
+
+ InsertDeleteUpsertOperator.Kind operation = modOp.getOperation();
+ if (operation != InsertDeleteUpsertOperator.Kind.INSERT
+ && operation != InsertDeleteUpsertOperator.Kind.UPSERT) {
+ return false;
+ }
+
+ DatasetConfig.DatasetFormat format = getFormat(modOp, context);
+ if (format != DatasetConfig.DatasetFormat.COLUMN) {
+ return false;
+ }
+
+ IVariableTypeEnvironment typeEnv =
context.getOutputTypeEnvironment(modOp);
+ IAType type = (IAType)
typeEnv.getType(modOp.getPayloadExpression().getValue());
+ SourceLocation srcLoc =
modOp.getPayloadExpression().getValue().getSourceLocation();
+ ColumnSupportedTypesValidator.validate(format, type, srcLoc);
+
+ return false;
+ }
+
+ private DatasetConfig.DatasetFormat getFormat(InsertDeleteUpsertOperator
modOp, IOptimizationContext context)
+ throws AlgebricksException {
+ MetadataProvider metadataProvider = (MetadataProvider)
context.getMetadataProvider();
+ DataSource dataSource = (DataSource) modOp.getDataSource();
+ if (dataSource == null) {
+ return null;
+ }
+ DataverseName dataverse = dataSource.getId().getDataverseName();
+ String datasetName = dataSource.getId().getDatasourceName();
+ String database = dataSource.getId().getDatabaseName();
+ Dataset dataset = metadataProvider.findDataset(database, dataverse,
datasetName);
+ if (dataset != null) {
+ return dataset.getDatasetFormatInfo().getFormat();
+ }
+
+ return null;
+ }
+}
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 74f53059da..2f8b391751 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -64,6 +64,7 @@ import org.apache.asterix.app.result.fields.ErrorsPrinter;
import org.apache.asterix.app.result.fields.ResultHandlePrinter;
import org.apache.asterix.app.result.fields.ResultsPrinter;
import org.apache.asterix.app.result.fields.StatusPrinter;
+import
org.apache.asterix.column.metadata.schema.visitor.ColumnSupportedTypesValidator;
import org.apache.asterix.common.api.IApplicationContext;
import org.apache.asterix.common.api.IClientRequest;
import org.apache.asterix.common.api.IMetadataLockManager;
@@ -72,6 +73,7 @@ import org.apache.asterix.common.api.IRequestTracker;
import org.apache.asterix.common.api.IResponsePrinter;
import org.apache.asterix.common.cluster.IClusterStateManager;
import org.apache.asterix.common.cluster.IGlobalTxManager;
+import org.apache.asterix.common.config.DatasetConfig;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.common.config.DatasetConfig.IndexType;
import org.apache.asterix.common.config.DatasetConfig.TransactionState;
@@ -915,9 +917,14 @@ public class QueryTranslator extends
AbstractLangTranslator implements IStatemen
throw new CompilationException(ErrorCode.DATASET_EXISTS,
sourceLoc, datasetName, dataverseName);
}
}
+ List<TypeExpression> partitioningExprTypes = null;
+ if (dsType == DatasetType.INTERNAL) {
+ partitioningExprTypes = ((InternalDetailsDecl)
dd.getDatasetDetailsDecl()).getPartitioningExprTypes();
+ }
- Pair<Datatype, Boolean> itemTypePair =
fetchDatasetItemType(mdTxnCtx, dsType, itemTypeDatabaseName,
- itemTypeDataverseName, itemTypeName, itemTypeExpr, false,
metadataProvider, sourceLoc);
+ Pair<Datatype, Boolean> itemTypePair =
fetchDatasetItemType(mdTxnCtx, dsType, datasetFormatInfo.getFormat(),
+ partitioningExprTypes, itemTypeDatabaseName,
itemTypeDataverseName, itemTypeName, itemTypeExpr,
+ false, metadataProvider, sourceLoc);
itemTypeEntity = itemTypePair.first;
IAType itemType = itemTypeEntity.getDatatype();
boolean itemTypeIsInline = itemTypePair.second;
@@ -938,9 +945,10 @@ public class QueryTranslator extends
AbstractLangTranslator implements IStatemen
switch (dsType) {
case INTERNAL:
if (metaItemTypeExpr != null) {
- Pair<Datatype, Boolean> metaItemTypePair =
fetchDatasetItemType(mdTxnCtx, dsType,
- metaItemTypeDatabaseName,
metaItemTypeDataverseName, metaItemTypeName, metaItemTypeExpr,
- true, metadataProvider, sourceLoc);
+ Pair<Datatype, Boolean> metaItemTypePair =
+ fetchDatasetItemType(mdTxnCtx, dsType,
datasetFormatInfo.getFormat(),
+ partitioningExprTypes,
metaItemTypeDatabaseName, metaItemTypeDataverseName,
+ metaItemTypeName, metaItemTypeExpr,
true, metadataProvider, sourceLoc);
metaItemTypeEntity = metaItemTypePair.first;
metaItemType = metaItemTypeEntity.getDatatype();
metaItemTypeIsInline = metaItemTypePair.second;
@@ -949,8 +957,7 @@ public class QueryTranslator extends AbstractLangTranslator
implements IStatemen
List<List<String>> partitioningExprs =
((InternalDetailsDecl)
dd.getDatasetDetailsDecl()).getPartitioningExprs();
- List<TypeExpression> partitioningExprTypes =
- ((InternalDetailsDecl)
dd.getDatasetDetailsDecl()).getPartitioningExprTypes();
+
List<Integer> keySourceIndicators =
((InternalDetailsDecl)
dd.getDatasetDetailsDecl()).getKeySourceIndicators();
boolean autogenerated = ((InternalDetailsDecl)
dd.getDatasetDetailsDecl()).isAutogenerated();
@@ -1130,9 +1137,10 @@ public class QueryTranslator extends
AbstractLangTranslator implements IStatemen
}
protected Pair<Datatype, Boolean>
fetchDatasetItemType(MetadataTransactionContext mdTxnCtx, DatasetType
datasetType,
- String itemTypeDatabaseName, DataverseName itemTypeDataverseName,
String itemTypeName,
- TypeExpression itemTypeExpr, boolean isMetaItemType,
MetadataProvider metadataProvider,
- SourceLocation sourceLoc) throws AlgebricksException {
+ DatasetConfig.DatasetFormat format, List<TypeExpression>
partitioningExprTypes, String itemTypeDatabaseName,
+ DataverseName itemTypeDataverseName, String itemTypeName,
TypeExpression itemTypeExpr,
+ boolean isMetaItemType, MetadataProvider metadataProvider,
SourceLocation sourceLoc)
+ throws AlgebricksException {
switch (itemTypeExpr.getTypeKind()) {
case TYPEREFERENCE:
Datatype itemTypeEntity =
@@ -1143,12 +1151,14 @@ public class QueryTranslator extends
AbstractLangTranslator implements IStatemen
DatasetUtil.getFullyQualifiedDisplayName(itemTypeDataverseName, itemTypeName));
}
IAType itemType = itemTypeEntity.getDatatype();
- validateDatasetItemType(datasetType, itemType, isMetaItemType,
sourceLoc);
+ validateDatasetItemType(datasetType, format,
partitioningExprTypes, itemType, isMetaItemType,
+ sourceLoc);
return new Pair<>(itemTypeEntity, false);
case RECORD:
itemType = translateType(itemTypeDatabaseName,
itemTypeDataverseName, itemTypeName, itemTypeExpr,
mdTxnCtx);
- validateDatasetItemType(datasetType, itemType, isMetaItemType,
sourceLoc);
+ validateDatasetItemType(datasetType, format,
partitioningExprTypes, itemType, isMetaItemType,
+ sourceLoc);
itemTypeEntity =
new Datatype(itemTypeDatabaseName,
itemTypeDataverseName, itemTypeName, itemType, true);
return new Pair<>(itemTypeEntity, true);
@@ -1158,7 +1168,8 @@ public class QueryTranslator extends
AbstractLangTranslator implements IStatemen
}
}
- protected void validateDatasetItemType(DatasetType datasetType, IAType
itemType, boolean isMetaItemType,
+ protected void validateDatasetItemType(DatasetType datasetType,
DatasetConfig.DatasetFormat format,
+ List<TypeExpression> partitioningExprTypes, IAType itemType,
boolean isMetaItemType,
SourceLocation sourceLoc) throws AlgebricksException {
if (itemType.getTypeTag() != ATypeTag.OBJECT) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR,
sourceLoc,
@@ -1169,6 +1180,21 @@ public class QueryTranslator extends
AbstractLangTranslator implements IStatemen
if (datasetType == DatasetType.VIEW) {
ViewUtil.validateViewItemType((ARecordType) itemType, sourceLoc);
}
+
+ // Validate columnar type
+ if (datasetType == DatasetType.INTERNAL) {
+ ColumnSupportedTypesValidator.validate(format, itemType,
sourceLoc);
+ if (partitioningExprTypes != null) {
+ for (TypeExpression typeExpr : partitioningExprTypes) {
+ String typeName = ((TypeReferenceExpression)
typeExpr).getIdent().second.getValue();
+ IAType type = BuiltinTypeMap.getBuiltinType(typeName);
+ if (type != null) {
+ // type will be validated next
+ ColumnSupportedTypesValidator.validate(format, type,
sourceLoc);
+ }
+ }
+ }
+ }
}
protected Map<String, String> createExternalDatasetProperties(String
databaseName, DataverseName dataverseName,
@@ -2780,9 +2806,9 @@ public class QueryTranslator extends
AbstractLangTranslator implements IStatemen
List<ViewDetails.ForeignKey> foreignKeys = null;
String datetimeFormat = null, dateFormat = null, timeFormat = null;
if (cvs.hasItemType()) {
- Pair<Datatype, Boolean> itemTypePair =
- fetchDatasetItemType(mdTxnCtx, DatasetType.VIEW,
itemTypeDatabaseName, itemTypeDataverseName,
- itemTypeName, cvs.getItemType(), false,
metadataProvider, sourceLoc);
+ Pair<Datatype, Boolean> itemTypePair =
fetchDatasetItemType(mdTxnCtx, DatasetType.VIEW,
+ DatasetConfig.DatasetFormat.ROW, null,
itemTypeDatabaseName, itemTypeDataverseName,
+ itemTypeName, cvs.getItemType(), false,
metadataProvider, sourceLoc);
itemTypeEntity = itemTypePair.first;
itemTypeIsInline = itemTypePair.second;
ARecordType itemType = (ARecordType)
itemTypeEntity.getDatatype();
diff --git
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/util/ColumnValuesUtil.java
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/supported-types/supported-types.001.ddl.sqlpp
similarity index 50%
copy from
asterixdb/asterix-column/src/main/java/org/apache/asterix/column/util/ColumnValuesUtil.java
copy to
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/supported-types/supported-types.001.ddl.sqlpp
index 0ecdeefb91..a938e7e6b4 100644
---
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/util/ColumnValuesUtil.java
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/supported-types/supported-types.001.ddl.sqlpp
@@ -16,33 +16,15 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.column.util;
-public class ColumnValuesUtil {
- private ColumnValuesUtil() {
- }
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+USE test;
- public static int getBitWidth(int level) {
- //+1 for the null bit
- return (32 - Integer.numberOfLeadingZeros(level)) + 1;
- }
-
- public static int getNullMask(int level) {
- return 1 << getBitWidth(level) - 1;
- }
-
- public static boolean isNull(int mask, int level) {
- return (mask & level) == mask;
- }
-
- public static int getChildValue(int parentMask, int childMask, int level) {
- if (isNull(parentMask, level)) {
- return clearNullBit(parentMask, level) | childMask;
- }
- return level;
- }
-
- public static int clearNullBit(int nullBitMask, int level) {
- return (nullBitMask - 1) & level;
- }
-}
+CREATE TYPE InvalidType AS {
+ uid: uuid,
+ birthdate: datetime,
+ created_time: time,
+ created_date: date,
+ online_since: duration
+};
\ No newline at end of file
diff --git
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/util/ColumnValuesUtil.java
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/supported-types/supported-types.002.ddl.sqlpp
similarity index 50%
copy from
asterixdb/asterix-column/src/main/java/org/apache/asterix/column/util/ColumnValuesUtil.java
copy to
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/supported-types/supported-types.002.ddl.sqlpp
index 0ecdeefb91..6544da632a 100644
---
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/util/ColumnValuesUtil.java
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/supported-types/supported-types.002.ddl.sqlpp
@@ -16,33 +16,10 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.column.util;
-public class ColumnValuesUtil {
- private ColumnValuesUtil() {
- }
+USE test;
- public static int getBitWidth(int level) {
- //+1 for the null bit
- return (32 - Integer.numberOfLeadingZeros(level)) + 1;
- }
-
- public static int getNullMask(int level) {
- return 1 << getBitWidth(level) - 1;
- }
-
- public static boolean isNull(int mask, int level) {
- return (mask & level) == mask;
- }
-
- public static int getChildValue(int parentMask, int childMask, int level) {
- if (isNull(parentMask, level)) {
- return clearNullBit(parentMask, level) | childMask;
- }
- return level;
- }
-
- public static int clearNullBit(int nullBitMask, int level) {
- return (nullBitMask - 1) & level;
- }
-}
+CREATE DATASET ColumnUnsupportedType
+PRIMARY KEY (birthdate: datetime) WITH {
+ "storage-format": {"format": "column"}
+}
\ No newline at end of file
diff --git
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/util/ColumnValuesUtil.java
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/supported-types/supported-types.003.ddl.sqlpp
similarity index 50%
copy from
asterixdb/asterix-column/src/main/java/org/apache/asterix/column/util/ColumnValuesUtil.java
copy to
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/supported-types/supported-types.003.ddl.sqlpp
index 0ecdeefb91..93df9b4f20 100644
---
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/util/ColumnValuesUtil.java
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/supported-types/supported-types.003.ddl.sqlpp
@@ -16,33 +16,10 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.column.util;
-public class ColumnValuesUtil {
- private ColumnValuesUtil() {
- }
+USE test;
- public static int getBitWidth(int level) {
- //+1 for the null bit
- return (32 - Integer.numberOfLeadingZeros(level)) + 1;
- }
-
- public static int getNullMask(int level) {
- return 1 << getBitWidth(level) - 1;
- }
-
- public static boolean isNull(int mask, int level) {
- return (mask & level) == mask;
- }
-
- public static int getChildValue(int parentMask, int childMask, int level) {
- if (isNull(parentMask, level)) {
- return clearNullBit(parentMask, level) | childMask;
- }
- return level;
- }
-
- public static int clearNullBit(int nullBitMask, int level) {
- return (nullBitMask - 1) & level;
- }
-}
+CREATE DATASET ColumnUnsupportedType(InvalidType)
+PRIMARY KEY uid WITH {
+ "storage-format": {"format": "column"}
+}
\ No newline at end of file
diff --git
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/util/ColumnValuesUtil.java
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/supported-types/supported-types.004.ddl.sqlpp
similarity index 50%
copy from
asterixdb/asterix-column/src/main/java/org/apache/asterix/column/util/ColumnValuesUtil.java
copy to
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/supported-types/supported-types.004.ddl.sqlpp
index 0ecdeefb91..c3a6c7e199 100644
---
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/util/ColumnValuesUtil.java
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/supported-types/supported-types.004.ddl.sqlpp
@@ -16,33 +16,15 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.column.util;
-public class ColumnValuesUtil {
- private ColumnValuesUtil() {
- }
+USE test;
- public static int getBitWidth(int level) {
- //+1 for the null bit
- return (32 - Integer.numberOfLeadingZeros(level)) + 1;
- }
+CREATE DATASET ColumnUnsupportedType
+PRIMARY KEY (uid: uuid) AUTOGENERATED WITH {
+ "storage-format": {"format": "column"}
+};
- public static int getNullMask(int level) {
- return 1 << getBitWidth(level) - 1;
- }
-
- public static boolean isNull(int mask, int level) {
- return (mask & level) == mask;
- }
-
- public static int getChildValue(int parentMask, int childMask, int level) {
- if (isNull(parentMask, level)) {
- return clearNullBit(parentMask, level) | childMask;
- }
- return level;
- }
-
- public static int clearNullBit(int nullBitMask, int level) {
- return (nullBitMask - 1) & level;
- }
-}
+CREATE DATASET RowDataset
+PRIMARY KEY (birthdate: datetime) WITH {
+ "storage-format": {"format": "row"}
+};
\ No newline at end of file
diff --git
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/util/ColumnValuesUtil.java
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/supported-types/supported-types.005.update.sqlpp
similarity index 50%
copy from
asterixdb/asterix-column/src/main/java/org/apache/asterix/column/util/ColumnValuesUtil.java
copy to
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/supported-types/supported-types.005.update.sqlpp
index 0ecdeefb91..9a9428c8ae 100644
---
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/util/ColumnValuesUtil.java
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/supported-types/supported-types.005.update.sqlpp
@@ -16,33 +16,9 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.column.util;
-public class ColumnValuesUtil {
- private ColumnValuesUtil() {
- }
+USE test;
- public static int getBitWidth(int level) {
- //+1 for the null bit
- return (32 - Integer.numberOfLeadingZeros(level)) + 1;
- }
-
- public static int getNullMask(int level) {
- return 1 << getBitWidth(level) - 1;
- }
-
- public static boolean isNull(int mask, int level) {
- return (mask & level) == mask;
- }
-
- public static int getChildValue(int parentMask, int childMask, int level) {
- if (isNull(parentMask, level)) {
- return clearNullBit(parentMask, level) | childMask;
- }
- return level;
- }
-
- public static int clearNullBit(int nullBitMask, int level) {
- return (nullBitMask - 1) & level;
- }
-}
+INSERT INTO ColumnUnsupportedType (
+ {"birthdate": datetime("2022-01-01T00:00:00")}
+)
\ No newline at end of file
diff --git
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/util/ColumnValuesUtil.java
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/supported-types/supported-types.006.update.sqlpp
similarity index 50%
copy from
asterixdb/asterix-column/src/main/java/org/apache/asterix/column/util/ColumnValuesUtil.java
copy to
asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/supported-types/supported-types.006.update.sqlpp
index 0ecdeefb91..2595748f51 100644
---
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/util/ColumnValuesUtil.java
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/column/supported-types/supported-types.006.update.sqlpp
@@ -16,33 +16,10 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.asterix.column.util;
-public class ColumnValuesUtil {
- private ColumnValuesUtil() {
- }
+USE test;
- public static int getBitWidth(int level) {
- //+1 for the null bit
- return (32 - Integer.numberOfLeadingZeros(level)) + 1;
- }
-
- public static int getNullMask(int level) {
- return 1 << getBitWidth(level) - 1;
- }
-
- public static boolean isNull(int mask, int level) {
- return (mask & level) == mask;
- }
-
- public static int getChildValue(int parentMask, int childMask, int level) {
- if (isNull(parentMask, level)) {
- return clearNullBit(parentMask, level) | childMask;
- }
- return level;
- }
-
- public static int clearNullBit(int nullBitMask, int level) {
- return (nullBitMask - 1) & level;
- }
-}
+INSERT INTO ColumnUnsupportedType (
+ SELECT VALUE r
+ FROM RowDataset r
+)
\ No newline at end of file
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml
b/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml
index aa7248944c..0b65d9328e 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml
@@ -16438,6 +16438,16 @@
<output-dir compare="Text">metadata</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="column">
+ <compilation-unit name="supported-types">
+ <output-dir compare="Text">supported-types</output-dir>
+ <expected-error>ASX0067: Type(s) '[datetime]' are not supported in
columnar storage format. Supported types are [bigint, double, string, boolean,
uuid]</expected-error>
+ <expected-error>ASX0067: Type(s) '[datetime, date, time, duration]'
are not supported in columnar storage format. Supported types are [bigint,
double, string, boolean, uuid]</expected-error>
+ <expected-error>ASX0067: Type(s) '[datetime]' are not supported in
columnar storage format. Supported types are [bigint, double, string, boolean,
uuid]</expected-error>
+ <expected-error>ASX0067: Type(s) '[datetime]' are not supported in
columnar storage format. Supported types are [bigint, double, string, boolean,
uuid]</expected-error>
+ <source-location>false</source-location>
+ </compilation-unit>
+ </test-case>
</test-group>
<test-group name="copy-from">
<test-case FilePath="copy-from">
diff --git
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/UnionSchemaNode.java
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/UnionSchemaNode.java
index eba5ac04f1..3cacb8ada5 100644
---
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/UnionSchemaNode.java
+++
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/UnionSchemaNode.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.column.metadata.schema;
+import static
org.apache.asterix.column.util.ColumnValuesUtil.getNormalizedTypeTag;
+
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
@@ -69,7 +71,7 @@ public final class UnionSchemaNode extends
AbstractSchemaNestedNode {
public AbstractSchemaNode getOrCreateChild(ATypeTag childTypeTag,
FlushColumnMetadata columnMetadata)
throws HyracksDataException {
- ATypeTag normalizedTypeTag =
FlushColumnMetadata.getNormalizedTypeTag(childTypeTag);
+ ATypeTag normalizedTypeTag = getNormalizedTypeTag(childTypeTag);
AbstractSchemaNode currentChild = children.get(normalizedTypeTag);
//The parent of a union child should be the actual parent
AbstractSchemaNode newChild =
columnMetadata.getOrCreateChild(currentChild, normalizedTypeTag);
diff --git
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/visitor/ColumnSupportedTypesValidator.java
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/visitor/ColumnSupportedTypesValidator.java
new file mode 100644
index 0000000000..5b27a74b4f
--- /dev/null
+++
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/visitor/ColumnSupportedTypesValidator.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.metadata.schema.visitor;
+
+import static
org.apache.asterix.column.util.ColumnValuesUtil.getNormalizedTypeTag;
+
+import java.util.HashSet;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.asterix.column.values.reader.ColumnValueReaderFactory;
+import org.apache.asterix.column.values.writer.ColumnValuesWriterFactory;
+import org.apache.asterix.common.config.DatasetConfig;
+import org.apache.asterix.common.exceptions.CompilationException;
+import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.AUnionType;
+import org.apache.asterix.om.types.AbstractCollectionType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.types.IATypeVisitor;
+import org.apache.hyracks.api.exceptions.SourceLocation;
+
+/**
+ * Validates supported types for datasets with {@link
DatasetConfig.DatasetFormat#COLUMN} format
+ *
+ * @see ColumnValuesWriterFactory
+ * @see ColumnValueReaderFactory
+ */
+public class ColumnSupportedTypesValidator implements IATypeVisitor<Void,
Set<ATypeTag>> {
+ private static final Set<ATypeTag> SUPPORTED_PRIMITIVE_TYPES =
+ Set.of(ATypeTag.BOOLEAN, ATypeTag.BIGINT, ATypeTag.DOUBLE,
ATypeTag.STRING, ATypeTag.UUID);
+ private static final String SUPPORTED_TYPES_STRING =
+
SUPPORTED_PRIMITIVE_TYPES.stream().sorted().collect(Collectors.toList()).toString();
+ private static final ColumnSupportedTypesValidator VALIDATOR = new
ColumnSupportedTypesValidator();
+
+ private ColumnSupportedTypesValidator() {
+ }
+
+ /**
+ * Ensure dataset format nested type includes only supported types
+ *
+ * @param format dataset format
+ * @param type to check
+ * @throws CompilationException if an unsupported type is encountered
+ */
+ public static void validate(DatasetConfig.DatasetFormat format, IAType
type) throws CompilationException {
+ validate(format, type, null);
+ }
+
+ /**
+ * Ensure dataset format nested type includes only supported types
+ *
+ * @param format dataset format
+ * @param type to check
+ * @param sourceLocation source location (if any)
+ * @throws CompilationException if an unsupported type is encountered
+ */
+ public static void validate(DatasetConfig.DatasetFormat format, IAType
type, SourceLocation sourceLocation)
+ throws CompilationException {
+ if (format != DatasetConfig.DatasetFormat.COLUMN) {
+ return;
+ }
+
+ Set<ATypeTag> unsupportedTypes = new HashSet<>();
+ type.accept(VALIDATOR, unsupportedTypes);
+ if (!unsupportedTypes.isEmpty()) {
+ String unsupportedList =
unsupportedTypes.stream().sorted().collect(Collectors.toList()).toString();
+ throw
CompilationException.create(ErrorCode.UNSUPPORTED_COLUMN_TYPE, sourceLocation,
unsupportedList,
+ SUPPORTED_TYPES_STRING);
+ }
+ }
+
+ @Override
+ public Void visit(ARecordType recordType, Set<ATypeTag> arg) {
+ for (IAType fieldType : recordType.getFieldTypes()) {
+ fieldType.accept(this, arg);
+ }
+
+ return null;
+ }
+
+ @Override
+ public Void visit(AbstractCollectionType collectionType, Set<ATypeTag>
arg) {
+ return collectionType.getItemType().accept(this, arg);
+ }
+
+ @Override
+ public Void visit(AUnionType unionType, Set<ATypeTag> arg) {
+ for (IAType fieldType : unionType.getUnionList()) {
+ fieldType.accept(this, arg);
+ }
+
+ return null;
+ }
+
+ @Override
+ public Void visitFlat(IAType flatType, Set<ATypeTag> arg) {
+ ATypeTag typeTag = getNormalizedTypeTag(flatType.getTypeTag());
+ // Allow ANY
+ if (typeTag != ATypeTag.ANY && typeTag != ATypeTag.NULL && typeTag !=
ATypeTag.MISSING
+ && !SUPPORTED_PRIMITIVE_TYPES.contains(typeTag)) {
+ arg.add(typeTag);
+ }
+
+ return null;
+ }
+}
diff --git
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/FlushColumnMetadata.java
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/FlushColumnMetadata.java
index 0d7404d980..e89a120965 100644
---
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/FlushColumnMetadata.java
+++
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/FlushColumnMetadata.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.column.operation.lsm.flush;
+import static
org.apache.asterix.column.util.ColumnValuesUtil.getNormalizedTypeTag;
+
import java.io.ByteArrayInputStream;
import java.io.DataInput;
import java.io.DataInputStream;
@@ -563,19 +565,6 @@ public final class FlushColumnMetadata extends
AbstractColumnMetadata {
}
}
- public static ATypeTag getNormalizedTypeTag(ATypeTag typeTag) {
- switch (typeTag) {
- case TINYINT:
- case SMALLINT:
- case INTEGER:
- return ATypeTag.BIGINT;
- case FLOAT:
- return ATypeTag.DOUBLE;
- default:
- return typeTag;
- }
- }
-
public void close() {
//Dereference multiPageOp
multiPageOpRef.setValue(null);
diff --git
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/util/ColumnValuesUtil.java
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/util/ColumnValuesUtil.java
index 0ecdeefb91..3094abece2 100644
---
a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/util/ColumnValuesUtil.java
+++
b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/util/ColumnValuesUtil.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.column.util;
+import org.apache.asterix.om.types.ATypeTag;
+
public class ColumnValuesUtil {
private ColumnValuesUtil() {
}
@@ -45,4 +47,17 @@ public class ColumnValuesUtil {
public static int clearNullBit(int nullBitMask, int level) {
return (nullBitMask - 1) & level;
}
+
+ public static ATypeTag getNormalizedTypeTag(ATypeTag typeTag) {
+ switch (typeTag) {
+ case TINYINT:
+ case SMALLINT:
+ case INTEGER:
+ return ATypeTag.BIGINT;
+ case FLOAT:
+ return ATypeTag.DOUBLE;
+ default:
+ return typeTag;
+ }
+ }
}
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
index 97b80603de..a49ca551a0 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
@@ -94,6 +94,7 @@ public enum ErrorCode implements IError {
NON_STRING_WRITE_PATH(64),
WRITE_PATH_LENGTH_EXCEEDS_MAX_LENGTH(65),
TYPE_MISMATCH_EXTRA_FIELD(66),
+ UNSUPPORTED_COLUMN_TYPE(67),
UNSUPPORTED_JRE(100),
diff --git
a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
index 0e8072009c..f97b7b6004 100644
--- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
+++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
@@ -101,6 +101,7 @@
64 = Path expression produced a value of type '%1$s'. Path must be of type
string
65 = Length of the file path '%1$s' exceeds the maximum length of '%2$s bytes'
allowed in %3$s
66 = Type mismatch: including an extra field %1$s
+67 = Type(s) '%1$s' are not supported in columnar storage format. Supported
types are %2$s
100 = Unsupported JRE: %1$s
