This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 3e2623cf64827e891ecd8c6dea13d03f5599c6a5 Author: Dmitry Lychagin <[email protected]> AuthorDate: Wed Apr 1 11:20:04 2020 -0700 [NO ISSUE][COMP] CREATE DATASET with inline type definition - user model changes: yes - storage format changes: no - interface changes: no Details: - Support inline type specification in CREATE DATASET statement: CREATE DATASET customer(cid INTEGER NOT NULL, first_name STRING) ... - Add testcases Change-Id: Ic88ccacd016a6144f1b05a2f79a07b2a980d9c9b Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/5524 Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Reviewed-by: Dmitry Lychagin <[email protected]> Reviewed-by: Hussain Towaileb <[email protected]> --- .../asterix/app/translator/QueryTranslator.java | 147 ++++++++++++++++++--- .../create-dataset-inline-type-1.1.ddl.sqlpp | 141 ++++++++++++++++++++ .../create-dataset-inline-type-1.2.query.sqlpp | 22 +++ .../create-dataset-inline-type-1.3.ddl.sqlpp | 30 +++++ .../create-dataset-inline-type-1.4.query.sqlpp | 26 ++++ .../create-dataset-inline-type-1.2.adm | 21 +++ .../create-dataset-inline-type-1.4.adm | 1 + .../cross-dataverse/cross-dv01/cross-dv01.1.ast | 6 +- .../test/resources/runtimets/testsuite_sqlpp.xml | 5 + asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj | 14 +- .../asterix/lang/common/statement/DatasetDecl.java | 64 ++------- .../lang/common/visitor/FormatPrintVisitor.java | 12 +- .../lang/common/visitor/QueryPrintVisitor.java | 15 ++- .../asterix-lang-sqlpp/src/main/javacc/SQLPP.jj | 128 ++++++++++++++---- .../metadata/bootstrap/MetadataRecordTypes.java | 2 +- .../DatasetTupleTranslator.java | 4 +- .../apache/asterix/metadata/utils/DatasetUtil.java | 21 ++- .../asterix/metadata/utils/MetadataUtil.java | 1 + 18 files changed, 533 insertions(+), 127 deletions(-) diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java index b8a048d..a440ae4 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java @@ -91,6 +91,8 @@ import org.apache.asterix.lang.common.base.IRewriterFactory; import org.apache.asterix.lang.common.base.IStatementRewriter; import org.apache.asterix.lang.common.base.Statement; import org.apache.asterix.lang.common.expression.IndexedTypeExpression; +import org.apache.asterix.lang.common.expression.TypeExpression; +import org.apache.asterix.lang.common.expression.TypeReferenceExpression; import org.apache.asterix.lang.common.statement.CompactStatement; import org.apache.asterix.lang.common.statement.ConnectFeedStatement; import org.apache.asterix.lang.common.statement.CreateDataverseStatement; @@ -560,10 +562,45 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen String dataverseName = getActiveDataverse(dd.getDataverse()); String datasetName = dd.getName().getValue(); DatasetType dsType = dd.getDatasetType(); - String itemTypeDataverseName = getActiveDataverse(dd.getItemTypeDataverse()); - String itemTypeName = dd.getItemTypeName().getValue(); - String metaItemTypeDataverseName = getActiveDataverse(dd.getMetaItemTypeDataverse()); - String metaItemTypeName = dd.getMetaItemTypeName().getValue(); + TypeExpression itemTypeExpr = dd.getItemType(); + String itemTypeDataverseName = null, itemTypeName = null, itemTypeFullyQualifiedName = null; + switch (itemTypeExpr.getTypeKind()) { + case TYPEREFERENCE: + TypeReferenceExpression itemTypeRefExpr = (TypeReferenceExpression) itemTypeExpr; + Identifier itemTypeDataverseIdent = itemTypeRefExpr.getIdent().first; + itemTypeDataverseName = itemTypeDataverseIdent != null && itemTypeDataverseIdent.getValue() != null + ? itemTypeDataverseIdent.getValue() : dataverseName; + itemTypeName = itemTypeRefExpr.getIdent().second.getValue(); + itemTypeFullyQualifiedName = itemTypeDataverseName + '.' + itemTypeName; + break; + case RECORD: + break; + default: + throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, sourceLoc, + String.valueOf(itemTypeExpr.getTypeKind())); + } + + TypeExpression metaItemTypeExpr = dd.getMetaItemType(); + String metaItemTypeDataverseName = null, metaItemTypeName = null, metaItemTypeFullyQualifiedName = null; + if (metaItemTypeExpr != null) { + switch (metaItemTypeExpr.getTypeKind()) { + case TYPEREFERENCE: + TypeReferenceExpression metaItemTypeRefExpr = (TypeReferenceExpression) metaItemTypeExpr; + Identifier metaItemTypeDataverseIdent = metaItemTypeRefExpr.getIdent().first; + metaItemTypeDataverseName = + metaItemTypeDataverseIdent != null && metaItemTypeDataverseIdent.getValue() != null + ? metaItemTypeDataverseIdent.getValue() : dataverseName; + metaItemTypeName = metaItemTypeRefExpr.getIdent().second.getValue(); + metaItemTypeFullyQualifiedName = metaItemTypeDataverseName + '.' + metaItemTypeName; + break; + case RECORD: + break; + default: + throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, sourceLoc, + String.valueOf(metaItemTypeExpr.getTypeKind())); + } + } + Identifier ngNameId = dd.getNodegroupName(); String nodegroupName = ngNameId == null ? null : ngNameId.getValue(); String compactionPolicy = dd.getCompactionPolicy(); @@ -575,9 +612,9 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen boolean bActiveTxn = true; metadataProvider.setMetadataTxnContext(mdTxnCtx); MetadataLockUtil.createDatasetBegin(lockManager, metadataProvider.getLocks(), dataverseName, - itemTypeDataverseName, itemTypeDataverseName + "." + itemTypeName, metaItemTypeDataverseName, - metaItemTypeDataverseName + "." + metaItemTypeName, nodegroupName, compactionPolicy, - dataverseName + "." + datasetName, defaultCompactionPolicy); + itemTypeDataverseName, itemTypeFullyQualifiedName, metaItemTypeDataverseName, + metaItemTypeFullyQualifiedName, nodegroupName, compactionPolicy, dataverseName + "." + datasetName, + defaultCompactionPolicy); Dataset dataset = null; try { IDatasetDetails datasetDetails = null; @@ -590,10 +627,23 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen throw new CompilationException(ErrorCode.DATASET_EXISTS, sourceLoc, datasetName, dataverseName); } } - Datatype dt = MetadataManager.INSTANCE.getDatatype(metadataProvider.getMetadataTxnContext(), - itemTypeDataverseName, itemTypeName); - if (dt == null) { - throw new CompilationException(ErrorCode.UNKNOWN_TYPE, sourceLoc, itemTypeName); + IAType itemType; + switch (itemTypeExpr.getTypeKind()) { + case TYPEREFERENCE: + itemType = metadataProvider.findType(itemTypeDataverseName, itemTypeName); + break; + case RECORD: + itemTypeDataverseName = dataverseName; + itemTypeName = DatasetUtil.createInlineTypeName(datasetName, false); + MetadataLockUtil.createTypeBegin(lockManager, metadataProvider.getLocks(), itemTypeDataverseName, + itemTypeDataverseName + "." + itemTypeName); + itemType = translateType(itemTypeDataverseName, itemTypeName, itemTypeExpr, mdTxnCtx); + MetadataManager.INSTANCE.addDatatype(mdTxnCtx, + new Datatype(itemTypeDataverseName, itemTypeName, itemType, true)); + break; + default: + throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, sourceLoc, + String.valueOf(itemTypeExpr.getTypeKind())); } String ngName = ngNameId != null ? ngNameId.getValue() : configureNodegroupForDataset(appCtx, dd.getHints(), dataverseName, datasetName, metadataProvider, @@ -607,19 +657,35 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen } switch (dd.getDatasetType()) { case INTERNAL: - IAType itemType = dt.getDatatype(); if (itemType.getTypeTag() != ATypeTag.OBJECT) { throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, "Dataset type has to be a record type."); } IAType metaItemType = null; - if (metaItemTypeDataverseName != null && metaItemTypeName != null) { - metaItemType = metadataProvider.findType(metaItemTypeDataverseName, metaItemTypeName); - } - if (metaItemType != null && metaItemType.getTypeTag() != ATypeTag.OBJECT) { - throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, - "Dataset meta type has to be a record type."); + if (metaItemTypeExpr != null) { + switch (metaItemTypeExpr.getTypeKind()) { + case TYPEREFERENCE: + metaItemType = metadataProvider.findType(metaItemTypeDataverseName, metaItemTypeName); + if (metaItemType.getTypeTag() != ATypeTag.OBJECT) { + throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, + "Dataset meta type has to be a record type."); + } + break; + case RECORD: + metaItemTypeDataverseName = dataverseName; + metaItemTypeName = DatasetUtil.createInlineTypeName(datasetName, true); + MetadataLockUtil.createTypeBegin(lockManager, metadataProvider.getLocks(), + metaItemTypeDataverseName, metaItemTypeDataverseName + "." + metaItemTypeName); + metaItemType = translateType(metaItemTypeDataverseName, metaItemTypeName, + metaItemTypeExpr, mdTxnCtx); + MetadataManager.INSTANCE.addDatatype(mdTxnCtx, + new Datatype(metaItemTypeDataverseName, metaItemTypeName, metaItemType, true)); + break; + default: + throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, sourceLoc, + String.valueOf(metaItemTypeExpr.getTypeKind())); + } } ARecordType metaRecType = (ARecordType) metaItemType; @@ -1234,10 +1300,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, "Cannot redefine builtin type " + typeName + "."); } else { - Map<TypeSignature, IAType> typeMap = TypeTranslator.computeTypes(mdTxnCtx, - stmtCreateType.getTypeDef(), stmtCreateType.getIdent().getValue(), dataverseName); - TypeSignature typeSignature = new TypeSignature(dataverseName, typeName); - IAType type = typeMap.get(typeSignature); + IAType type = translateType(dataverseName, typeName, stmtCreateType.getTypeDef(), mdTxnCtx); MetadataManager.INSTANCE.addDatatype(mdTxnCtx, new Datatype(dataverseName, typeName, type, false)); } } @@ -1250,6 +1313,13 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen } } + private IAType translateType(String dataverseName, String typeName, TypeExpression typeDef, + MetadataTransactionContext mdTxnCtx) throws AlgebricksException { + Map<TypeSignature, IAType> typeMap = TypeTranslator.computeTypes(mdTxnCtx, typeDef, typeName, dataverseName); + TypeSignature typeSignature = new TypeSignature(dataverseName, typeName); + return typeMap.get(typeSignature); + } + protected void handleDataverseDropStatement(MetadataProvider metadataProvider, Statement stmt, IHyracksClientConnection hcc, IRequestParameters requestParameters) throws Exception { DataverseDropStatement stmtDelete = (DataverseDropStatement) stmt; @@ -1459,8 +1529,41 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen } } validateDatasetState(metadataProvider, ds, sourceLoc); + + // prepare to drop item and meta types if they were created as inline types + String itemTypeDataverseName = ds.getItemTypeDataverseName(); + String itemTypeName = ds.getItemTypeName(); + boolean isInlineItemType = DatasetUtil.isInlineTypeName(ds, itemTypeDataverseName, itemTypeName); + if (isInlineItemType) { + MetadataLockUtil.dropTypeBegin(lockManager, metadataProvider.getLocks(), itemTypeDataverseName, + itemTypeDataverseName + '.' + itemTypeName); + } + String metaTypeDataverseName = ds.getMetaItemTypeDataverseName(); + String metaTypeName = ds.getMetaItemTypeName(); + boolean isInlineMetaType = + metaTypeName != null && DatasetUtil.isInlineTypeName(ds, metaTypeDataverseName, metaTypeName); + if (isInlineMetaType) { + MetadataLockUtil.dropTypeBegin(lockManager, metadataProvider.getLocks(), metaTypeDataverseName, + metaTypeDataverseName + '.' + metaTypeName); + } + Datatype inlineItemType = isInlineItemType + ? MetadataManager.INSTANCE.getDatatype(mdTxnCtx.getValue(), itemTypeDataverseName, itemTypeName) + : null; + Datatype inlineMetaType = isInlineMetaType + ? MetadataManager.INSTANCE.getDatatype(mdTxnCtx.getValue(), metaTypeDataverseName, metaTypeName) + : null; + ds.drop(metadataProvider, mdTxnCtx, jobsToExecute, bActiveTxn, progress, hcc, dropCorrespondingNodeGroup, sourceLoc); + + // drop inline item and meta types + if (isInlineItemType && inlineItemType.getIsAnonymous()) { + MetadataManager.INSTANCE.dropDatatype(mdTxnCtx.getValue(), itemTypeDataverseName, itemTypeName); + } + if (isInlineMetaType && inlineMetaType.getIsAnonymous()) { + MetadataManager.INSTANCE.dropDatatype(mdTxnCtx.getValue(), metaTypeDataverseName, metaTypeName); + } + MetadataManager.INSTANCE.commitTransaction(mdTxnCtx.getValue()); } catch (Exception e) { if (bActiveTxn.booleanValue()) { diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.1.ddl.sqlpp new file mode 100644 index 0000000..6230d4b --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.1.ddl.sqlpp @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +DROP DATAVERSE test IF EXISTS; +CREATE DATAVERSE test; +USE test; + +/* Metadata test function */ + +CREATE FUNCTION listMetadata() { + SELECT "Dataset" AS en, d.DatasetName, d.DatatypeDataverseName, d.DatatypeName, + d.MetatypeDataverseName, d.MetatypeName + FROM Metadata.`Dataset` d + WHERE d.DataverseName = "test" + UNION ALL + SELECT "Datatype" AS en, dt.DatatypeName, dt.Derived + FROM Metadata.`Datatype` dt + WHERE dt.DataverseName = "test" + ORDER BY en, DatasetName, DatatypeName +}; + +/* Internal datasets */ + +CREATE DATASET A_Customers_Default_Closed( + c_custkey integer not null, + c_name string not null, + c_phone string, + c_comment string +) PRIMARY KEY c_custkey ; + +CREATE DATASET A_Customers_Closed( + c_custkey integer not null, + c_name string not null, + c_phone string, + c_comment string +) CLOSED TYPE PRIMARY KEY c_custkey; + +CREATE DATASET A_Customers_Open( + c_custkey integer not null, + c_name string not null, + c_phone string, + c_comment string +) OPEN TYPE PRIMARY KEY c_custkey; + +/* External datasets */ + +CREATE EXTERNAL DATASET B_Orders_Default_Closed( + o_orderkey integer not null, + o_custkey integer not null, + o_orderstatus string not null, + o_totalprice double not null, + o_orderdate string not null, + o_orderpriority string not null, + o_clerk string not null, + o_shippriority integer not null, + o_comment string +) +USING `localfs` +((`path`=`asterix_nc1://data/tpch0.001/orders.tbl`), +(`input-format`=`text-input-format`),(`format`=`delimited-text`),(`delimiter`=`|`)); + +CREATE EXTERNAL DATASET B_Orders_Closed( + o_orderkey integer not null, + o_custkey integer not null, + o_orderstatus string not null, + o_totalprice double not null, + o_orderdate string not null, + o_orderpriority string not null, + o_clerk string not null, + o_shippriority integer not null, + o_comment string +) CLOSED TYPE +USING `localfs` +((`path`=`asterix_nc1://data/tpch0.001/orders.tbl`), +(`input-format`=`text-input-format`),(`format`=`delimited-text`),(`delimiter`=`|`)); + +CREATE EXTERNAL DATASET B_Orders_Open( + o_orderkey integer not null, + o_custkey integer not null, + o_orderstatus string not null, + o_totalprice double not null, + o_orderdate string not null, + o_orderpriority string not null, + o_clerk string not null, + o_shippriority integer not null, + o_comment string +) OPEN TYPE +USING `localfs` +((`path`=`asterix_nc1://data/tpch0.001/orders.tbl`), +(`input-format`=`text-input-format`),(`format`=`delimited-text`),(`delimiter`=`|`)); + +/* Internal datasets with inline META type */ + +CREATE DATASET C_Customers_Meta_Default_Closed( + c_custkey integer not null, + c_name string not null, + c_phone string, + c_comment string +) +WITH META(c_x integer not null, c_y integer) +PRIMARY KEY c_custkey ; + +CREATE DATASET C_Customers_Meta_Closed( + c_custkey integer not null, + c_name string not null, + c_phone string, + c_comment string +) CLOSED TYPE +WITH META( + c_x integer not null, + c_y integer +) CLOSED TYPE +PRIMARY KEY c_custkey; + +CREATE DATASET C_Customers_Meta_Open( + c_custkey integer not null, + c_name string not null, + c_phone string, + c_comment string +) OPEN TYPE +WITH META( + c_x integer not null, + c_y integer +) OPEN TYPE +PRIMARY KEY c_custkey; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.2.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.2.query.sqlpp new file mode 100644 index 0000000..52a6324 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.2.query.sqlpp @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +USE test; + +listMetadata(); \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.3.ddl.sqlpp new file mode 100644 index 0000000..8a08888 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.3.ddl.sqlpp @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +USE test; + +DROP DATASET A_Customers_Default_Closed; +DROP DATASET A_Customers_Closed; +DROP DATASET A_Customers_Open; +DROP DATASET B_Orders_Default_Closed; +DROP DATASET B_Orders_Closed; +DROP DATASET B_Orders_Open; +DROP DATASET C_Customers_Meta_Default_Closed; +DROP DATASET C_Customers_Meta_Closed; +DROP DATASET C_Customers_Meta_Open; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.4.query.sqlpp new file mode 100644 index 0000000..ad16400 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.4.query.sqlpp @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Test that inline types are deleted when dataset is dropped + */ + +USE test; + +array_count(listMetadata()); \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.2.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.2.adm new file mode 100644 index 0000000..8a539c8 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.2.adm @@ -0,0 +1,21 @@ +{ "en": "Dataset", "DatatypeName": "$d$t$i$A_Customers_Closed", "DatasetName": "A_Customers_Closed", "DatatypeDataverseName": "test" } +{ "en": "Dataset", "DatatypeName": "$d$t$i$A_Customers_Default_Closed", "DatasetName": "A_Customers_Default_Closed", "DatatypeDataverseName": "test" } +{ "en": "Dataset", "DatatypeName": "$d$t$i$A_Customers_Open", "DatasetName": "A_Customers_Open", "DatatypeDataverseName": "test" } +{ "en": "Dataset", "DatatypeName": "$d$t$i$B_Orders_Closed", "DatasetName": "B_Orders_Closed", "DatatypeDataverseName": "test" } +{ "en": "Dataset", "DatatypeName": "$d$t$i$B_Orders_Default_Closed", "DatasetName": "B_Orders_Default_Closed", "DatatypeDataverseName": "test" } +{ "en": "Dataset", "DatatypeName": "$d$t$i$B_Orders_Open", "DatasetName": "B_Orders_Open", "DatatypeDataverseName": "test" } +{ "en": "Dataset", "DatatypeName": "$d$t$i$C_Customers_Meta_Closed", "DatasetName": "C_Customers_Meta_Closed", "DatatypeDataverseName": "test", "MetatypeDataverseName": "test", "MetatypeName": "$d$t$m$C_Customers_Meta_Closed" } +{ "en": "Dataset", "DatatypeName": "$d$t$i$C_Customers_Meta_Default_Closed", "DatasetName": "C_Customers_Meta_Default_Closed", "DatatypeDataverseName": "test", "MetatypeDataverseName": "test", "MetatypeName": "$d$t$m$C_Customers_Meta_Default_Closed" } +{ "en": "Dataset", "DatatypeName": "$d$t$i$C_Customers_Meta_Open", "DatasetName": "C_Customers_Meta_Open", "DatatypeDataverseName": "test", "MetatypeDataverseName": "test", "MetatypeName": "$d$t$m$C_Customers_Meta_Open" } +{ "en": "Datatype", "DatatypeName": "$d$t$i$A_Customers_Closed", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": false, "Fields": [ { "FieldName": "c_custkey", "FieldType": "int32", "IsNullable": false }, { "FieldName": "c_name", "FieldType": "string", "IsNullable": false }, { "FieldName": "c_phone", "FieldType": "string", "IsNullable": true }, { "FieldName": "c_comment", "FieldType": "string", "IsNullable": true } ] } } } +{ "en": "Datatype", "DatatypeName": "$d$t$i$A_Customers_Default_Closed", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": false, "Fields": [ { "FieldName": "c_custkey", "FieldType": "int32", "IsNullable": false }, { "FieldName": "c_name", "FieldType": "string", "IsNullable": false }, { "FieldName": "c_phone", "FieldType": "string", "IsNullable": true }, { "FieldName": "c_comment", "FieldType": "string", "IsNullable": true } ] } } } +{ "en": "Datatype", "DatatypeName": "$d$t$i$A_Customers_Open", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "c_custkey", "FieldType": "int32", "IsNullable": false }, { "FieldName": "c_name", "FieldType": "string", "IsNullable": false }, { "FieldName": "c_phone", "FieldType": "string", "IsNullable": true }, { "FieldName": "c_comment", "FieldType": "string", "IsNullable": true } ] } } } +{ "en": "Datatype", "DatatypeName": "$d$t$i$B_Orders_Closed", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": false, "Fields": [ { "FieldName": "o_orderkey", "FieldType": "int32", "IsNullable": false }, { "FieldName": "o_custkey", "FieldType": "int32", "IsNullable": false }, { "FieldName": "o_orderstatus", "FieldType": "string", "IsNullable": false }, { "FieldName": "o_totalprice", "FieldType": "double", "IsNullable": false }, { "FieldName": "o_orderdate", "Field [...] +{ "en": "Datatype", "DatatypeName": "$d$t$i$B_Orders_Default_Closed", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": false, "Fields": [ { "FieldName": "o_orderkey", "FieldType": "int32", "IsNullable": false }, { "FieldName": "o_custkey", "FieldType": "int32", "IsNullable": false }, { "FieldName": "o_orderstatus", "FieldType": "string", "IsNullable": false }, { "FieldName": "o_totalprice", "FieldType": "double", "IsNullable": false }, { "FieldName": "o_orderdate" [...] +{ "en": "Datatype", "DatatypeName": "$d$t$i$B_Orders_Open", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "o_orderkey", "FieldType": "int32", "IsNullable": false }, { "FieldName": "o_custkey", "FieldType": "int32", "IsNullable": false }, { "FieldName": "o_orderstatus", "FieldType": "string", "IsNullable": false }, { "FieldName": "o_totalprice", "FieldType": "double", "IsNullable": false }, { "FieldName": "o_orderdate", "FieldTyp [...] +{ "en": "Datatype", "DatatypeName": "$d$t$i$C_Customers_Meta_Closed", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": false, "Fields": [ { "FieldName": "c_custkey", "FieldType": "int32", "IsNullable": false }, { "FieldName": "c_name", "FieldType": "string", "IsNullable": false }, { "FieldName": "c_phone", "FieldType": "string", "IsNullable": true }, { "FieldName": "c_comment", "FieldType": "string", "IsNullable": true } ] } } } +{ "en": "Datatype", "DatatypeName": "$d$t$i$C_Customers_Meta_Default_Closed", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": false, "Fields": [ { "FieldName": "c_custkey", "FieldType": "int32", "IsNullable": false }, { "FieldName": "c_name", "FieldType": "string", "IsNullable": false }, { "FieldName": "c_phone", "FieldType": "string", "IsNullable": true }, { "FieldName": "c_comment", "FieldType": "string", "IsNullable": true } ] } } } +{ "en": "Datatype", "DatatypeName": "$d$t$i$C_Customers_Meta_Open", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "c_custkey", "FieldType": "int32", "IsNullable": false }, { "FieldName": "c_name", "FieldType": "string", "IsNullable": false }, { "FieldName": "c_phone", "FieldType": "string", "IsNullable": true }, { "FieldName": "c_comment", "FieldType": "string", "IsNullable": true } ] } } } +{ "en": "Datatype", "DatatypeName": "$d$t$m$C_Customers_Meta_Closed", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": false, "Fields": [ { "FieldName": "c_x", "FieldType": "int32", "IsNullable": false }, { "FieldName": "c_y", "FieldType": "int32", "IsNullable": true } ] } } } +{ "en": "Datatype", "DatatypeName": "$d$t$m$C_Customers_Meta_Default_Closed", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": false, "Fields": [ { "FieldName": "c_x", "FieldType": "int32", "IsNullable": false }, { "FieldName": "c_y", "FieldType": "int32", "IsNullable": true } ] } } } +{ "en": "Datatype", "DatatypeName": "$d$t$m$C_Customers_Meta_Open", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "c_x", "FieldType": "int32", "IsNullable": false }, { "FieldName": "c_y", "FieldType": "int32", "IsNullable": true } ] } } } diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.4.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.4.adm new file mode 100644 index 0000000..c227083 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.4.adm @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/cross-dataverse/cross-dv01/cross-dv01.1.ast b/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/cross-dataverse/cross-dv01/cross-dv01.1.ast index c3a4877..ffe5bc3 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/cross-dataverse/cross-dv01/cross-dv01.1.ast +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/cross-dataverse/cross-dv01/cross-dv01.1.ast @@ -17,7 +17,7 @@ TypeDecl tchrType [ dept : string } ] -DatasetDecl ugdstd(stdType) partitioned by [[id]] -DatasetDecl gdstd(stdType) partitioned by [[id]] +DatasetDecl ugdstd(student.stdType) partitioned by [[id]] +DatasetDecl gdstd(student.stdType) partitioned by [[id]] DatasetDecl prof(tchrType) partitioned by [[id]] -DatasetDecl pstdoc(tchrType) partitioned by [[id]] +DatasetDecl pstdoc(tchrType) partitioned by [[id]] \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml index 5ef5808..0d17fb1 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml @@ -3860,6 +3860,11 @@ </compilation-unit> </test-case> <test-case FilePath="ddl"> + <compilation-unit name="create-dataset-inline-type-1"> + <output-dir compare="Text">create-dataset-inline-type-1</output-dir> + </compilation-unit> + </test-case> + <test-case FilePath="ddl"> <compilation-unit name="drop-primary-index"> <output-dir compare="Text">drop-primary-index</output-dir> <expected-error>Cannot drop index "ds". Drop dataset "ds" to remove this index</expected-error> diff --git a/asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj b/asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj index 4c8820f..9b54562 100644 --- a/asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj +++ b/asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj @@ -476,7 +476,7 @@ DatasetDecl DatasetSpecification() throws ParseException: DatasetDecl dsetDecl = null; boolean autogenerated = false; Pair<Integer, List<String>> filterField = null; - Pair<Identifier,Identifier> metaTypeComponents = new Pair<Identifier, Identifier>(null, null); + Pair<Identifier,Identifier> metaTypeComponents = null; RecordConstructor withRecord = null; } { @@ -495,10 +495,8 @@ DatasetDecl DatasetSpecification() throws ParseException: try{ dsetDecl = new DatasetDecl(nameComponents.first, nameComponents.second, - typeComponents.first, - typeComponents.second, - metaTypeComponents.first, - metaTypeComponents.second, + new TypeReferenceExpression(typeComponents), + null, nodeGroupName != null? new Identifier(nodeGroupName): null, hints, DatasetType.EXTERNAL, @@ -542,10 +540,8 @@ DatasetDecl DatasetSpecification() throws ParseException: try{ dsetDecl = new DatasetDecl(nameComponents.first, nameComponents.second, - typeComponents.first, - typeComponents.second, - metaTypeComponents.first, - metaTypeComponents.second, + new TypeReferenceExpression(typeComponents), + metaTypeComponents != null ? new TypeReferenceExpression(metaTypeComponents) : null, nodeGroupName != null ? new Identifier(nodeGroupName) : null, hints, DatasetType.INTERNAL, diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DatasetDecl.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DatasetDecl.java index 45fc33a..22753d0 100644 --- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DatasetDecl.java +++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DatasetDecl.java @@ -25,6 +25,7 @@ import org.apache.asterix.common.exceptions.CompilationException; import org.apache.asterix.lang.common.base.AbstractStatement; import org.apache.asterix.lang.common.base.Statement; import org.apache.asterix.lang.common.expression.RecordConstructor; +import org.apache.asterix.lang.common.expression.TypeExpression; import org.apache.asterix.lang.common.struct.Identifier; import org.apache.asterix.lang.common.util.ConfigurationUtil; import org.apache.asterix.lang.common.util.DatasetDeclParametersUtil; @@ -36,10 +37,8 @@ import org.apache.asterix.runtime.compression.CompressionManager; public class DatasetDecl extends AbstractStatement { protected final Identifier name; protected final Identifier dataverse; - protected final Identifier itemTypeDataverse; - protected final Identifier itemTypeName; - protected final Identifier metaItemTypeDataverse; - protected final Identifier metaItemTypeName; + protected final TypeExpression itemType; + protected final TypeExpression metaItemType; protected final Identifier nodegroupName; protected final DatasetType datasetType; protected final IDatasetDetailsDecl datasetDetailsDecl; @@ -47,24 +46,13 @@ public class DatasetDecl extends AbstractStatement { private AdmObjectNode withObjectNode; protected final boolean ifNotExists; - public DatasetDecl(Identifier dataverse, Identifier name, Identifier itemTypeDataverse, Identifier itemTypeName, - Identifier metaItemTypeDataverse, Identifier metaItemTypeName, Identifier nodeGroupName, - Map<String, String> hints, DatasetType datasetType, IDatasetDetailsDecl idd, RecordConstructor withRecord, - boolean ifNotExists) throws CompilationException { + public DatasetDecl(Identifier dataverse, Identifier name, TypeExpression itemType, TypeExpression metaItemType, + Identifier nodeGroupName, Map<String, String> hints, DatasetType datasetType, IDatasetDetailsDecl idd, + RecordConstructor withRecord, boolean ifNotExists) throws CompilationException { this.dataverse = dataverse; this.name = name; - this.itemTypeName = itemTypeName; - if (itemTypeDataverse.getValue() == null) { - this.itemTypeDataverse = dataverse; - } else { - this.itemTypeDataverse = itemTypeDataverse; - } - this.metaItemTypeName = metaItemTypeName; - if (metaItemTypeDataverse == null || metaItemTypeDataverse.getValue() == null) { - this.metaItemTypeDataverse = dataverse; - } else { - this.metaItemTypeDataverse = metaItemTypeDataverse; - } + this.itemType = itemType; + this.metaItemType = metaItemType; this.nodegroupName = nodeGroupName; this.hints = hints; this.withObjectNode = DatasetDeclParametersUtil.validateAndGetWithObjectNode(withRecord, datasetType); @@ -85,40 +73,12 @@ public class DatasetDecl extends AbstractStatement { return name; } - public Identifier getItemTypeName() { - return itemTypeName; - } - - public Identifier getItemTypeDataverse() { - return itemTypeDataverse; + public TypeExpression getItemType() { + return itemType; } - public String getQualifiedTypeName() { - if (itemTypeDataverse == dataverse) { - return itemTypeName.getValue(); - } else { - return itemTypeDataverse.getValue() + "." + itemTypeName.getValue(); - } - } - - public Identifier getMetaName() { - return name; - } - - public Identifier getMetaItemTypeName() { - return metaItemTypeName == null ? new Identifier() : metaItemTypeName; - } - - public Identifier getMetaItemTypeDataverse() { - return metaItemTypeDataverse == null ? new Identifier() : metaItemTypeDataverse; - } - - public String getQualifiedMetaTypeName() { - if (metaItemTypeDataverse == dataverse) { - return metaItemTypeName.getValue(); - } else { - return metaItemTypeDataverse.getValue() + "." + metaItemTypeName.getValue(); - } + public TypeExpression getMetaItemType() { + return metaItemType; } public Identifier getNodegroupName() { diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java index 6b734dd..c123348 100644 --- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java +++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java @@ -457,16 +457,18 @@ public class FormatPrintVisitor implements ILangVisitor<Void, Integer> { public Void visit(DatasetDecl dd, Integer step) throws CompilationException { if (dd.getDatasetType() == DatasetType.INTERNAL) { out.print(skip(step) + "create " + datasetSymbol + generateFullName(dd.getDataverse(), dd.getName()) - + generateIfNotExists(dd.getIfNotExists()) + "(" + dd.getQualifiedTypeName() + ")" - + " primary key "); + + generateIfNotExists(dd.getIfNotExists()) + "("); + dd.getItemType().accept(this, step + 2); + out.print(skip(step) + ") primary key "); printDelimitedKeys(((InternalDetailsDecl) dd.getDatasetDetailsDecl()).getPartitioningExprs(), ","); if (((InternalDetailsDecl) dd.getDatasetDetailsDecl()).isAutogenerated()) { out.print(" autogenerated "); } } else if (dd.getDatasetType() == DatasetType.EXTERNAL) { - out.print( - skip(step) + "create external " + datasetSymbol + generateFullName(dd.getDataverse(), dd.getName()) - + "(" + dd.getQualifiedTypeName() + ")" + generateIfNotExists(dd.getIfNotExists())); + out.print(skip(step) + "create external " + datasetSymbol + + generateFullName(dd.getDataverse(), dd.getName()) + "("); + dd.getItemType().accept(this, step + 2); + out.print(skip(step) + ")" + generateIfNotExists(dd.getIfNotExists())); ExternalDetailsDecl externalDetails = (ExternalDetailsDecl) dd.getDatasetDetailsDecl(); out.print(" using " + revertStringToQuoted(externalDetails.getAdapter())); printConfiguration(externalDetails.getProperties()); diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/QueryPrintVisitor.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/QueryPrintVisitor.java index ff55880..5b8c9c0 100644 --- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/QueryPrintVisitor.java +++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/QueryPrintVisitor.java @@ -388,15 +388,18 @@ public abstract class QueryPrintVisitor extends AbstractQueryExpressionVisitor<V @Override public Void visit(DatasetDecl dd, Integer step) throws CompilationException { if (dd.getDatasetType() == DatasetType.INTERNAL) { - String line = skip(step) + "DatasetDecl " + dd.getName() + "(" + dd.getItemTypeName() + ")" - + " partitioned by " + ((InternalDetailsDecl) dd.getDatasetDetailsDecl()).getPartitioningExprs(); + out.print(skip(step) + "DatasetDecl " + dd.getName() + "("); + dd.getItemType().accept(this, step + 2); + out.print(skip(step) + ") partitioned by " + + ((InternalDetailsDecl) dd.getDatasetDetailsDecl()).getPartitioningExprs()); if (((InternalDetailsDecl) dd.getDatasetDetailsDecl()).isAutogenerated()) { - line += " [autogenerated]"; + out.print(" [autogenerated]"); } - out.println(line); + out.println(); } else if (dd.getDatasetType() == DatasetType.EXTERNAL) { - out.println(skip(step) + "DatasetDecl " + dd.getName() + "(" + dd.getItemTypeName() + ")" - + "is an external dataset"); + out.print(skip(step) + "DatasetDecl " + dd.getName() + "("); + dd.getItemType().accept(this, step + 2); + out.println(skip(step) + ")is an external dataset"); } return null; } diff --git a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj index e0a4341..5aa45bc 100644 --- a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj +++ b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj @@ -335,15 +335,15 @@ class SQLPPParser extends ScopeChecker implements IParser { hintCollector.clear(); try { return parseFunction.parse(); + } catch (SqlppParseException e) { + throw new CompilationException(ErrorCode.PARSE_ERROR, e.getSourceLocation(), LogRedactionUtil.userData(getMessage(e))); + } catch (ParseException e) { + throw new CompilationException(ErrorCode.PARSE_ERROR, LogRedactionUtil.userData(getMessage(e))); } catch (Error e) { // this is here as the JavaCharStream that's below the lexer sometimes throws Errors that are not handled // by the ANTLR-generated lexer or parser (e.g it does this for invalid backslash u + 4 hex digits escapes) final String msg = e.getClass().getSimpleName() + (e.getMessage() != null ? ": " + e.getMessage() : ""); throw new CompilationException(ErrorCode.PARSE_ERROR, LogRedactionUtil.userData(msg)); - } catch (SqlppParseException e) { - throw new CompilationException(ErrorCode.PARSE_ERROR, e.getSourceLocation(), LogRedactionUtil.userData(getMessage(e))); - } catch (ParseException e) { - throw new CompilationException(ErrorCode.PARSE_ERROR, LogRedactionUtil.userData(getMessage(e))); } finally { reportUnclaimedHints(); } @@ -428,7 +428,11 @@ class SQLPPParser extends ScopeChecker implements IParser { } private SqlppParseException createUnexpectedTokenError() { - return new SqlppParseException(getSourceLocation(token), "Unexpected token: " + LogRedactionUtil.userData(token.image)); + return createUnexpectedTokenError(token); + } + + private SqlppParseException createUnexpectedTokenError(Token t) { + return new SqlppParseException(getSourceLocation(t), "Unexpected token: " + LogRedactionUtil.userData(t.image)); } private boolean laToken(int idx, int kind, String image) { @@ -646,7 +650,8 @@ DatasetDecl DatasetSpecification(Token startStmtToken) throws ParseException: { Pair<Identifier,Identifier> nameComponents = null; boolean ifNotExists = false; - Pair<Identifier,Identifier> typeComponents = null; + TypeExpression typeExpr = null; + TypeExpression metaTypeExpr = null; String adapterName = null; Map<String,String> properties = null; FunctionSignature appliedFunction = null; @@ -656,13 +661,12 @@ DatasetDecl DatasetSpecification(Token startStmtToken) throws ParseException: DatasetDecl stmt = null; boolean autogenerated = false; Pair<Integer, List<String>> filterField = null; - Pair<Identifier,Identifier> metaTypeComponents = new Pair<Identifier, Identifier>(null, null); RecordConstructor withRecord = null; } { ( <EXTERNAL> Dataset() nameComponents = QualifiedName() - <LEFTPAREN> typeComponents = TypeName() <RIGHTPAREN> + typeExpr = DatasetTypeSpecification() ifNotExists = IfNotExists() <USING> adapterName = AdapterName() properties = Configuration() ( <ON> nodeGroupName = Identifier() )? @@ -675,10 +679,8 @@ DatasetDecl DatasetSpecification(Token startStmtToken) throws ParseException: try{ stmt = new DatasetDecl(nameComponents.first, nameComponents.second, - typeComponents.first, - typeComponents.second, - metaTypeComponents.first, - metaTypeComponents.second, + typeExpr, + null, nodeGroupName != null? new Identifier(nodeGroupName): null, hints, DatasetType.EXTERNAL, @@ -692,7 +694,7 @@ DatasetDecl DatasetSpecification(Token startStmtToken) throws ParseException: | ( <INTERNAL> )? Dataset() nameComponents = QualifiedName() - <LEFTPAREN> typeComponents = TypeName() <RIGHTPAREN> + typeExpr = DatasetTypeSpecification() ( { String name; } <WITH> @@ -703,7 +705,7 @@ DatasetDecl DatasetSpecification(Token startStmtToken) throws ParseException: "We can only support one additional associated field called \"meta\"."); } } - <LEFTPAREN> metaTypeComponents = TypeName() <RIGHTPAREN> + metaTypeExpr = DatasetTypeSpecification() )? ifNotExists = IfNotExists() primaryKeyFields = PrimaryKey() @@ -724,10 +726,8 @@ DatasetDecl DatasetSpecification(Token startStmtToken) throws ParseException: try{ stmt = new DatasetDecl(nameComponents.first, nameComponents.second, - typeComponents.first, - typeComponents.second, - metaTypeComponents.first, - metaTypeComponents.second, + typeExpr, + metaTypeExpr, nodeGroupName != null ? new Identifier(nodeGroupName) : null, hints, DatasetType.INTERNAL, @@ -744,6 +744,76 @@ DatasetDecl DatasetSpecification(Token startStmtToken) throws ParseException: } } +TypeExpression DatasetTypeSpecification() throws ParseException: +{ + TypeExpression typeExpr = null; +} +{ + ( + LOOKAHEAD(3) typeExpr = DatasetRecordTypeSpecification(true) + | typeExpr = DatasetReferenceTypeSpecification() + ) + { + return typeExpr; + } +} + +TypeExpression DatasetReferenceTypeSpecification() throws ParseException: +{ + TypeExpression typeExpr = null; +} +{ + <LEFTPAREN> typeExpr = TypeReference() <RIGHTPAREN> + { + return typeExpr; + } +} + +TypeExpression DatasetRecordTypeSpecification(boolean allowRecordKindModifier) throws ParseException: +{ + RecordTypeDefinition recordTypeDef = null; + RecordTypeDefinition.RecordKind recordKind = null; + Token recordKindToken = null; +} +{ + <LEFTPAREN> recordTypeDef = DatasetRecordTypeDef() <RIGHTPAREN> + ( recordKind = RecordTypeKind() { recordKindToken = token; } <TYPE> )? + { + if (recordKind == null) { + recordKind = RecordTypeDefinition.RecordKind.CLOSED; + } else if (!allowRecordKindModifier) { + throw createUnexpectedTokenError(recordKindToken); + } + recordTypeDef.setRecordKind(recordKind); + return recordTypeDef; + } +} + +RecordTypeDefinition DatasetRecordTypeDef() throws ParseException: +{ + RecordTypeDefinition recType = new RecordTypeDefinition(); +} +{ + DatasetRecordField(recType) ( <COMMA> DatasetRecordField(recType) )* + { + return recType; + } +} + +void DatasetRecordField(RecordTypeDefinition recType) throws ParseException: +{ + String fieldName; + TypeExpression type = null; + boolean isUnknownable = true; +} +{ + fieldName = Identifier() + type = TypeReference() ( <NOT> <NULL> { isUnknownable = false; } )? + { + recType.addField(fieldName, type, isUnknownable); + } +} + RefreshExternalDatasetStatement RefreshExternalDatasetStatement() throws ParseException: { Token startToken = null; @@ -1575,15 +1645,28 @@ TypeExpression TypeExpr() throws ParseException: } } +RecordTypeDefinition.RecordKind RecordTypeKind() throws ParseException: +{ + RecordTypeDefinition.RecordKind recordKind = null; +} +{ + ( + <CLOSED> { recordKind = RecordTypeDefinition.RecordKind.CLOSED; } + | <OPEN> { recordKind = RecordTypeDefinition.RecordKind.OPEN; } + ) + { + return recordKind; + } +} + RecordTypeDefinition RecordTypeDef() throws ParseException: { Token startToken = null; RecordTypeDefinition recType = new RecordTypeDefinition(); - RecordTypeDefinition.RecordKind recordKind = null; + RecordTypeDefinition.RecordKind recordKind = RecordTypeDefinition.RecordKind.OPEN; } { - ( <CLOSED> { recordKind = RecordTypeDefinition.RecordKind.CLOSED; } - | <OPEN> { recordKind = RecordTypeDefinition.RecordKind.OPEN; } )? + ( recordKind = RecordTypeKind() )? <LEFTBRACE> { startToken = token; @@ -1610,9 +1693,6 @@ RecordTypeDefinition RecordTypeDef() throws ParseException: )? <RIGHTBRACE> { - if (recordKind == null) { - recordKind = RecordTypeDefinition.RecordKind.OPEN; - } recType.setRecordKind(recordKind); return addSourceLocation(recType, startToken); } diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataRecordTypes.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataRecordTypes.java index d9309d9..7fdbfcf 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataRecordTypes.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataRecordTypes.java @@ -74,7 +74,7 @@ public final class MetadataRecordTypes { public static final String FIELD_NAME_KIND = "Kind"; public static final String FIELD_NAME_LANGUAGE = "Language"; public static final String FIELD_NAME_LAST_REFRESH_TIME = "LastRefreshTime"; - public static final String FIELD_NAME_METADATA_DATAVERSE = "MetatypeDataverseName"; + public static final String FIELD_NAME_METATYPE_DATAVERSE = "MetatypeDataverseName"; public static final String FIELD_NAME_METATYPE_NAME = "MetatypeName"; public static final String FIELD_NAME_NAME = "Name"; public static final String FIELD_NAME_NODE_NAME = "NodeName"; diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java index 7f8b9bf..dae6152 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java @@ -250,7 +250,7 @@ public class DatasetTupleTranslator extends AbstractTupleTranslator<Dataset> { String metaTypeDataverseName = null; String metaTypeName = null; int metaTypeDataverseNameIndex = - datasetRecord.getType().getFieldIndex(MetadataRecordTypes.FIELD_NAME_METADATA_DATAVERSE); + datasetRecord.getType().getFieldIndex(MetadataRecordTypes.FIELD_NAME_METATYPE_DATAVERSE); if (metaTypeDataverseNameIndex >= 0) { metaTypeDataverseName = ((AString) datasetRecord.getValueByPos(metaTypeDataverseNameIndex)).getStringValue(); @@ -430,7 +430,7 @@ public class DatasetTupleTranslator extends AbstractTupleTranslator<Dataset> { if (dataset.hasMetaPart()) { // write open field 1, the meta item type Dataverse name. fieldName.reset(); - aString.setValue(MetadataRecordTypes.FIELD_NAME_METADATA_DATAVERSE); + aString.setValue(MetadataRecordTypes.FIELD_NAME_METATYPE_DATAVERSE); stringSerde.serialize(aString, fieldName.getDataOutput()); fieldValue.reset(); aString.setValue(dataset.getMetaItemTypeDataverseName()); diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java index 4b7d359..a5084e0 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java @@ -99,6 +99,8 @@ public class DatasetUtil { */ public static final byte OP_UPSERT = 0x03; + private static final String DATASET_INLINE_TYPE_PREFIX = "$d$t$"; + private DatasetUtil() { } @@ -207,9 +209,13 @@ public class DatasetUtil { * field is actually a key by making sure the field is coming from the right record (data record or meta record), * e.g. if the field name happens to be equal to the key name but the field is coming from the data record while * the key is coming from the meta record. - * @param keySourceIndicator indicates where the key is coming from, 1 from meta record, 0 from data record - * @param keyIndex the key index we're checking the field against - * @param fieldFromMeta whether the field is coming from the meta record or the data record + * + * @param keySourceIndicator + * indicates where the key is coming from, 1 from meta record, 0 from data record + * @param keyIndex + * the key index we're checking the field against + * @param fieldFromMeta + * whether the field is coming from the meta record or the data record * @return true if the key source matches the field source. Otherwise, false. */ private static boolean keySourceMatches(List<Integer> keySourceIndicator, int keyIndex, boolean fieldFromMeta) { @@ -603,4 +609,13 @@ public class DatasetUtil { } return new Pair<>(first, second); } + + public static String createInlineTypeName(String datasetName, boolean forMetaItemType) { + char typeChar = forMetaItemType ? 'm' : 'i'; + return DATASET_INLINE_TYPE_PREFIX + typeChar + '$' + datasetName; + } + + public static boolean isInlineTypeName(Dataset dataset, String typeDataverseName, String typeName) { + return dataset.getDataverseName().equals(typeDataverseName) && typeName.startsWith(DATASET_INLINE_TYPE_PREFIX); + } } diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataUtil.java index e5d4721..3436b44 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataUtil.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataUtil.java @@ -43,4 +43,5 @@ public class MetadataUtil { int idx = datasetName.indexOf('.'); return datasetName.substring(0, idx); } + }
