This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 5cc254a2014e601f1aa65e0a4fdf98d4ef538318 Merge: 2364a21 3e2623c Author: Dmitry Lychagin <[email protected]> AuthorDate: Fri May 8 17:12:47 2020 -0700 Merge commit '3e2623cf64827e891ecd8c6dea13d03f5599c6a5' into HEAD Change-Id: I89091d8bf8575c73d26f497d5fbffe780adbedf9 .../asterix/app/translator/QueryTranslator.java | 143 +++++++++++++++++---- .../create-dataset-inline-type-1.1.ddl.sqlpp | 141 ++++++++++++++++++++ .../create-dataset-inline-type-1.2.query.sqlpp | 22 ++++ .../create-dataset-inline-type-1.3.ddl.sqlpp | 30 +++++ .../create-dataset-inline-type-1.4.query.sqlpp | 26 ++++ .../create-dataset-inline-type-1.2.adm | 21 +++ .../create-dataset-inline-type-1.4.adm | 1 + .../results/feeds/feeds_01/feeds_01.1.adm | 2 +- .../cross-dataverse/cross-dv01/cross-dv01.1.ast | 6 +- .../test/resources/runtimets/testsuite_sqlpp.xml | 5 + .../adapter/factory/GenericAdapterFactory.java | 20 ++- .../util/ExternalDataCompatibilityUtils.java | 7 +- .../external/util/ExternalDataConstants.java | 10 -- asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj | 14 +- .../asterix/lang/common/statement/DatasetDecl.java | 32 ++--- .../lang/common/visitor/FormatPrintVisitor.java | 13 +- .../lang/common/visitor/QueryPrintVisitor.java | 15 ++- .../asterix-lang-sqlpp/src/main/javacc/SQLPP.jj | 136 ++++++++++++++++---- .../apache/asterix/metadata/utils/DatasetUtil.java | 20 ++- .../asterix/metadata/utils/MetadataLockUtil.java | 6 +- 20 files changed, 548 insertions(+), 122 deletions(-) diff --cc asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java index eefb7f3,a440ae4..cf14555 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java @@@ -99,9 -91,10 +99,11 @@@ import org.apache.asterix.lang.common.b import org.apache.asterix.lang.common.base.IStatementRewriter; import org.apache.asterix.lang.common.base.Statement; import org.apache.asterix.lang.common.expression.IndexedTypeExpression; + import org.apache.asterix.lang.common.expression.TypeExpression; + import org.apache.asterix.lang.common.expression.TypeReferenceExpression; import org.apache.asterix.lang.common.statement.CompactStatement; import org.apache.asterix.lang.common.statement.ConnectFeedStatement; +import org.apache.asterix.lang.common.statement.CreateAdapterStatement; import org.apache.asterix.lang.common.statement.CreateDataverseStatement; import org.apache.asterix.lang.common.statement.CreateFeedPolicyStatement; import org.apache.asterix.lang.common.statement.CreateFeedStatement; @@@ -581,44 -555,55 +583,74 @@@ public class QueryTranslator extends Ab } public void handleCreateDatasetStatement(MetadataProvider metadataProvider, Statement stmt, - IHyracksClientConnection hcc, IRequestParameters requestParameters) throws CompilationException, Exception { - MutableObject<ProgressState> progress = new MutableObject<>(ProgressState.NO_PROGRESS); + IHyracksClientConnection hcc, IRequestParameters requestParameters) throws Exception { DatasetDecl dd = (DatasetDecl) stmt; - SourceLocation sourceLoc = dd.getSourceLocation(); - String dataverseName = getActiveDataverse(dd.getDataverse()); + DataverseName dataverseName = getActiveDataverseName(dd.getDataverse()); String datasetName = dd.getName().getValue(); - DataverseName itemTypeDataverseName = getActiveDataverseName(dd.getItemTypeDataverse()); - String itemTypeName = dd.getItemTypeName().getValue(); - DatasetType dsType = dd.getDatasetType(); + TypeExpression itemTypeExpr = dd.getItemType(); - String itemTypeDataverseName = null, itemTypeName = null, itemTypeFullyQualifiedName = null; ++ DataverseName itemTypeDataverseName = null; ++ String itemTypeName = null; + switch (itemTypeExpr.getTypeKind()) { + case TYPEREFERENCE: + TypeReferenceExpression itemTypeRefExpr = (TypeReferenceExpression) itemTypeExpr; - Identifier itemTypeDataverseIdent = itemTypeRefExpr.getIdent().first; - itemTypeDataverseName = itemTypeDataverseIdent != null && itemTypeDataverseIdent.getValue() != null - ? itemTypeDataverseIdent.getValue() : dataverseName; ++ Pair<DataverseName, Identifier> itemTypeIdent = itemTypeRefExpr.getIdent(); ++ itemTypeDataverseName = itemTypeIdent.first != null ? itemTypeIdent.first : dataverseName; + itemTypeName = itemTypeRefExpr.getIdent().second.getValue(); - itemTypeFullyQualifiedName = itemTypeDataverseName + '.' + itemTypeName; + break; + case RECORD: + break; + default: - throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, sourceLoc, ++ throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, stmt.getSourceLocation(), + String.valueOf(itemTypeExpr.getTypeKind())); + } + + TypeExpression metaItemTypeExpr = dd.getMetaItemType(); - String metaItemTypeDataverseName = null, metaItemTypeName = null, metaItemTypeFullyQualifiedName = null; + DataverseName metaItemTypeDataverseName = null; + String metaItemTypeName = null; - Identifier metaItemTypeId = dd.getMetaItemTypeName(); - if (metaItemTypeId != null) { - metaItemTypeName = metaItemTypeId.getValue(); - metaItemTypeDataverseName = getActiveDataverseName(dd.getMetaItemTypeDataverse()); + if (metaItemTypeExpr != null) { + switch (metaItemTypeExpr.getTypeKind()) { + case TYPEREFERENCE: + TypeReferenceExpression metaItemTypeRefExpr = (TypeReferenceExpression) metaItemTypeExpr; - Identifier metaItemTypeDataverseIdent = metaItemTypeRefExpr.getIdent().first; ++ Pair<DataverseName, Identifier> metaItemTypeIdent = metaItemTypeRefExpr.getIdent(); + metaItemTypeDataverseName = - metaItemTypeDataverseIdent != null && metaItemTypeDataverseIdent.getValue() != null - ? metaItemTypeDataverseIdent.getValue() : dataverseName; ++ metaItemTypeIdent.first != null ? metaItemTypeIdent.first : dataverseName; + metaItemTypeName = metaItemTypeRefExpr.getIdent().second.getValue(); - metaItemTypeFullyQualifiedName = metaItemTypeDataverseName + '.' + metaItemTypeName; + break; + case RECORD: + break; + default: - throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, sourceLoc, ++ throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, stmt.getSourceLocation(), + String.valueOf(metaItemTypeExpr.getTypeKind())); + } } + Identifier ngNameId = dd.getNodegroupName(); String nodegroupName = ngNameId == null ? null : ngNameId.getValue(); String compactionPolicy = dd.getCompactionPolicy(); + boolean defaultCompactionPolicy = compactionPolicy == null; + + lockUtil.createDatasetBegin(lockManager, metadataProvider.getLocks(), dataverseName, datasetName, + itemTypeDataverseName, itemTypeName, metaItemTypeDataverseName, metaItemTypeName, nodegroupName, + compactionPolicy, defaultCompactionPolicy, dd.getDatasetDetailsDecl()); + try { + doCreateDatasetStatement(metadataProvider, dd, dataverseName, datasetName, itemTypeDataverseName, - itemTypeName, metaItemTypeDataverseName, metaItemTypeName, hcc, requestParameters); ++ itemTypeExpr, itemTypeName, metaItemTypeExpr, metaItemTypeDataverseName, metaItemTypeName, hcc, ++ requestParameters); + } finally { + metadataProvider.getLocks().unlock(); + } + } + + protected void doCreateDatasetStatement(MetadataProvider metadataProvider, DatasetDecl dd, - DataverseName dataverseName, String datasetName, DataverseName itemTypeDataverseName, String itemTypeName, ++ DataverseName dataverseName, String datasetName, DataverseName itemTypeDataverseName, ++ TypeExpression itemTypeExpr, String itemTypeName, TypeExpression metaItemTypeExpr, + DataverseName metaItemTypeDataverseName, String metaItemTypeName, IHyracksClientConnection hcc, + IRequestParameters requestParameters) throws Exception { + MutableObject<ProgressState> progress = new MutableObject<>(ProgressState.NO_PROGRESS); + SourceLocation sourceLoc = dd.getSourceLocation(); + DatasetType dsType = dd.getDatasetType(); + Identifier ngNameId = dd.getNodegroupName(); + String compactionPolicy = dd.getCompactionPolicy(); Map<String, String> compactionPolicyProperties = dd.getCompactionPolicyProperties(); String compressionScheme = metadataProvider.getCompressionManager() .getDdlOrDefaultCompressionScheme(dd.getDatasetCompressionScheme()); @@@ -637,10 -627,23 +669,23 @@@ throw new CompilationException(ErrorCode.DATASET_EXISTS, sourceLoc, datasetName, dataverseName); } } - Datatype dt = MetadataManager.INSTANCE.getDatatype(metadataProvider.getMetadataTxnContext(), - itemTypeDataverseName, itemTypeName); - if (dt == null) { - throw new CompilationException(ErrorCode.UNKNOWN_TYPE, sourceLoc, itemTypeName); + IAType itemType; + switch (itemTypeExpr.getTypeKind()) { + case TYPEREFERENCE: + itemType = metadataProvider.findType(itemTypeDataverseName, itemTypeName); + break; + case RECORD: + itemTypeDataverseName = dataverseName; + itemTypeName = DatasetUtil.createInlineTypeName(datasetName, false); - MetadataLockUtil.createTypeBegin(lockManager, metadataProvider.getLocks(), itemTypeDataverseName, - itemTypeDataverseName + "." + itemTypeName); ++ lockUtil.createTypeBegin(lockManager, metadataProvider.getLocks(), itemTypeDataverseName, ++ itemTypeName); + itemType = translateType(itemTypeDataverseName, itemTypeName, itemTypeExpr, mdTxnCtx); + MetadataManager.INSTANCE.addDatatype(mdTxnCtx, + new Datatype(itemTypeDataverseName, itemTypeName, itemType, true)); + break; + default: + throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, sourceLoc, + String.valueOf(itemTypeExpr.getTypeKind())); } String ngName = ngNameId != null ? ngNameId.getValue() : configureNodegroupForDataset(appCtx, dd.getHints(), dataverseName, datasetName, metadataProvider, @@@ -661,12 -663,29 +705,29 @@@ } IAType metaItemType = null; - if (metaItemTypeDataverseName != null && metaItemTypeName != null) { - metaItemType = metadataProvider.findType(metaItemTypeDataverseName, metaItemTypeName); - } - if (metaItemType != null && metaItemType.getTypeTag() != ATypeTag.OBJECT) { - throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, - "Dataset meta type has to be a record type."); + if (metaItemTypeExpr != null) { + switch (metaItemTypeExpr.getTypeKind()) { + case TYPEREFERENCE: + metaItemType = metadataProvider.findType(metaItemTypeDataverseName, metaItemTypeName); + if (metaItemType.getTypeTag() != ATypeTag.OBJECT) { + throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, + "Dataset meta type has to be a record type."); + } + break; + case RECORD: + metaItemTypeDataverseName = dataverseName; + metaItemTypeName = DatasetUtil.createInlineTypeName(datasetName, true); - MetadataLockUtil.createTypeBegin(lockManager, metadataProvider.getLocks(), - metaItemTypeDataverseName, metaItemTypeDataverseName + "." + metaItemTypeName); ++ lockUtil.createTypeBegin(lockManager, metadataProvider.getLocks(), ++ metaItemTypeDataverseName, metaItemTypeName); + metaItemType = translateType(metaItemTypeDataverseName, metaItemTypeName, + metaItemTypeExpr, mdTxnCtx); + MetadataManager.INSTANCE.addDatatype(mdTxnCtx, + new Datatype(metaItemTypeDataverseName, metaItemTypeName, metaItemType, true)); + break; + default: + throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, sourceLoc, + String.valueOf(metaItemTypeExpr.getTypeKind())); + } } ARecordType metaRecType = (ARecordType) metaItemType; @@@ -1290,19 -1313,25 +1348,27 @@@ } } - private IAType translateType(String dataverseName, String typeName, TypeExpression typeDef, ++ private IAType translateType(DataverseName dataverseName, String typeName, TypeExpression typeDef, + MetadataTransactionContext mdTxnCtx) throws AlgebricksException { - Map<TypeSignature, IAType> typeMap = TypeTranslator.computeTypes(mdTxnCtx, typeDef, typeName, dataverseName); ++ Map<TypeSignature, IAType> typeMap = ++ TypeTranslator.computeTypes(dataverseName, typeName, typeDef, dataverseName, mdTxnCtx); + TypeSignature typeSignature = new TypeSignature(dataverseName, typeName); + return typeMap.get(typeSignature); + } + protected void handleDataverseDropStatement(MetadataProvider metadataProvider, Statement stmt, IHyracksClientConnection hcc, IRequestParameters requestParameters) throws Exception { - DataverseDropStatement stmtDelete = (DataverseDropStatement) stmt; - SourceLocation sourceLoc = stmtDelete.getSourceLocation(); - String dataverseName = stmtDelete.getDataverseName().getValue(); - if (dataverseName.equals(MetadataBuiltinEntities.DEFAULT_DATAVERSE_NAME)) { + DataverseDropStatement stmtDropDataverse = (DataverseDropStatement) stmt; + SourceLocation sourceLoc = stmtDropDataverse.getSourceLocation(); + DataverseName dataverseName = stmtDropDataverse.getDataverseName(); + if (dataverseName.equals(MetadataBuiltinEntities.DEFAULT_DATAVERSE_NAME) + || dataverseName.equals(MetadataConstants.METADATA_DATAVERSE_NAME)) { throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, - MetadataBuiltinEntities.DEFAULT_DATAVERSE_NAME + " dataverse can't be dropped"); + dataverseName + " dataverse can't be dropped"); } - lockManager.acquireDataverseWriteLock(metadataProvider.getLocks(), dataverseName); + lockUtil.dropDataverseBegin(lockManager, metadataProvider.getLocks(), dataverseName); try { - doDropDataverse(stmtDelete, sourceLoc, metadataProvider, hcc); + doDropDataverse(stmtDropDataverse, metadataProvider, hcc, requestParameters); } finally { metadataProvider.getLocks().unlock(); ExternalDatasetsRegistry.INSTANCE.releaseAcquiredLocks(metadataProvider); @@@ -1515,8 -1529,41 +1581,39 @@@ } } validateDatasetState(metadataProvider, ds, sourceLoc); + + // prepare to drop item and meta types if they were created as inline types - String itemTypeDataverseName = ds.getItemTypeDataverseName(); ++ DataverseName itemTypeDataverseName = ds.getItemTypeDataverseName(); + String itemTypeName = ds.getItemTypeName(); + boolean isInlineItemType = DatasetUtil.isInlineTypeName(ds, itemTypeDataverseName, itemTypeName); + if (isInlineItemType) { - MetadataLockUtil.dropTypeBegin(lockManager, metadataProvider.getLocks(), itemTypeDataverseName, - itemTypeDataverseName + '.' + itemTypeName); ++ lockUtil.dropTypeBegin(lockManager, metadataProvider.getLocks(), itemTypeDataverseName, itemTypeName); + } - String metaTypeDataverseName = ds.getMetaItemTypeDataverseName(); ++ DataverseName metaTypeDataverseName = ds.getMetaItemTypeDataverseName(); + String metaTypeName = ds.getMetaItemTypeName(); + boolean isInlineMetaType = + metaTypeName != null && DatasetUtil.isInlineTypeName(ds, metaTypeDataverseName, metaTypeName); + if (isInlineMetaType) { - MetadataLockUtil.dropTypeBegin(lockManager, metadataProvider.getLocks(), metaTypeDataverseName, - metaTypeDataverseName + '.' + metaTypeName); ++ lockUtil.dropTypeBegin(lockManager, metadataProvider.getLocks(), metaTypeDataverseName, metaTypeName); + } + Datatype inlineItemType = isInlineItemType + ? MetadataManager.INSTANCE.getDatatype(mdTxnCtx.getValue(), itemTypeDataverseName, itemTypeName) + : null; + Datatype inlineMetaType = isInlineMetaType + ? MetadataManager.INSTANCE.getDatatype(mdTxnCtx.getValue(), metaTypeDataverseName, metaTypeName) + : null; + ds.drop(metadataProvider, mdTxnCtx, jobsToExecute, bActiveTxn, progress, hcc, dropCorrespondingNodeGroup, sourceLoc); + + // drop inline item and meta types + if (isInlineItemType && inlineItemType.getIsAnonymous()) { + MetadataManager.INSTANCE.dropDatatype(mdTxnCtx.getValue(), itemTypeDataverseName, itemTypeName); + } + if (isInlineMetaType && inlineMetaType.getIsAnonymous()) { + MetadataManager.INSTANCE.dropDatatype(mdTxnCtx.getValue(), metaTypeDataverseName, metaTypeName); + } + MetadataManager.INSTANCE.commitTransaction(mdTxnCtx.getValue()); } catch (Exception e) { if (bActiveTxn.booleanValue()) { diff --cc asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj index 88017db,9b54562..ca515e5 --- a/asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj +++ b/asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj @@@ -496,9 -464,9 +496,10 @@@ NodegroupDecl NodegroupSpecification() DatasetDecl DatasetSpecification() throws ParseException: { - Pair<Identifier,Identifier> nameComponents = null; + Pair<DataverseName,Identifier> nameComponents = null; boolean ifNotExists = false; - Pair<Identifier,Identifier> typeComponents = null; + Pair<DataverseName,Identifier> typeComponents = null; ++ Pair<DataverseName,Identifier> metaTypeComponents = null; String adapterName = null; Map<String,String> properties = null; FunctionSignature appliedFunction = null; @@@ -508,7 -476,7 +509,6 @@@ DatasetDecl dsetDecl = null; boolean autogenerated = false; Pair<Integer, List<String>> filterField = null; - Pair<DataverseName,Identifier> metaTypeComponents = new Pair<DataverseName, Identifier>(null, null); - Pair<Identifier,Identifier> metaTypeComponents = null; RecordConstructor withRecord = null; } { diff --cc asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DatasetDecl.java index 811a494,22753d0..c0ad1b8 --- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DatasetDecl.java +++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DatasetDecl.java @@@ -36,11 -36,9 +37,9 @@@ import org.apache.asterix.runtime.compr public class DatasetDecl extends AbstractStatement { protected final Identifier name; - protected final Identifier dataverse; + protected final DataverseName dataverse; - protected final DataverseName itemTypeDataverse; - protected final Identifier itemTypeName; - protected final DataverseName metaItemTypeDataverse; - protected final Identifier metaItemTypeName; + protected final TypeExpression itemType; + protected final TypeExpression metaItemType; protected final Identifier nodegroupName; protected final DatasetType datasetType; protected final IDatasetDetailsDecl datasetDetailsDecl; @@@ -48,8 -46,7 +47,7 @@@ private AdmObjectNode withObjectNode; protected final boolean ifNotExists; - public DatasetDecl(DataverseName dataverse, Identifier name, DataverseName itemTypeDataverse, - Identifier itemTypeName, DataverseName metaItemTypeDataverse, Identifier metaItemTypeName, - public DatasetDecl(Identifier dataverse, Identifier name, TypeExpression itemType, TypeExpression metaItemType, ++ public DatasetDecl(DataverseName dataverse, Identifier name, TypeExpression itemType, TypeExpression metaItemType, Identifier nodeGroupName, Map<String, String> hints, DatasetType datasetType, IDatasetDetailsDecl idd, RecordConstructor withRecord, boolean ifNotExists) throws CompilationException { this.dataverse = dataverse; @@@ -78,24 -73,12 +74,16 @@@ return name; } + public DataverseName getDataverse() { + return dataverse; + } + - public Identifier getItemTypeName() { - return itemTypeName; + public TypeExpression getItemType() { + return itemType; } - public DataverseName getItemTypeDataverse() { - return itemTypeDataverse; - } - - public Identifier getMetaItemTypeName() { - return metaItemTypeName; - } - - public DataverseName getMetaItemTypeDataverse() { - return metaItemTypeDataverse; + public TypeExpression getMetaItemType() { + return metaItemType; } public Identifier getNodegroupName() { diff --cc asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj index 53faff9,5aa45bc..17c94bd --- a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj +++ b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj @@@ -478,18 -428,11 +478,26 @@@ class SQLPPParser extends ScopeChecker } private SqlppParseException createUnexpectedTokenError() { - return createUnexpectedTokenError(null); - return createUnexpectedTokenError(token); ++ return createUnexpectedTokenError(token, null); + } + + private SqlppParseException createUnexpectedTokenError(Token t) { - return new SqlppParseException(getSourceLocation(t), "Unexpected token: " + LogRedactionUtil.userData(t.image)); ++ return createUnexpectedTokenError(t, null); + } + + private SqlppParseException createUnexpectedTokenError(String expected) { - String message = "Unexpected token: " + LogRedactionUtil.userData(token.image) + ++ return createUnexpectedTokenError(token, expected); ++ } ++ ++ private SqlppParseException createUnexpectedTokenError(Token t, String expected) { ++ String message = "Unexpected token: " + LogRedactionUtil.userData(t.image) + + (expected == null ? "" : ". Expected: " + LogRedactionUtil.userData(expected)); - return new SqlppParseException(getSourceLocation(token), message); ++ return new SqlppParseException(getSourceLocation(t), message); + } + + private boolean laToken(int idx, int kind) { + Token t = getToken(idx); + return t.kind == kind; } private boolean laToken(int idx, int kind, String image) { @@@ -749,12 -648,13 +757,10 @@@ DatasetDecl CreateDatasetStatement(Toke DatasetDecl DatasetSpecification(Token startStmtToken) throws ParseException: { - Pair<Identifier,Identifier> nameComponents = null; + Pair<DataverseName,Identifier> nameComponents = null; boolean ifNotExists = false; - Pair<DataverseName,Identifier> typeComponents = null; - String adapterName = null; - Map<String,String> properties = null; - FunctionSignature appliedFunction = null; + TypeExpression typeExpr = null; + TypeExpression metaTypeExpr = null; - String adapterName = null; - Map<String,String> properties = null; - FunctionSignature appliedFunction = null; Pair<List<Integer>, List<List<String>>> primaryKeyFields = null; String nodeGroupName = null; Map<String,String> hints = new HashMap<String,String>(); @@@ -764,84 -664,156 +770,150 @@@ RecordConstructor withRecord = null; } { + nameComponents = QualifiedName() - <LEFTPAREN> typeComponents = TypeName() <RIGHTPAREN> ++ typeExpr = DatasetTypeSpecification() ( - <EXTERNAL> Dataset() nameComponents = QualifiedName() - typeExpr = DatasetTypeSpecification() - ifNotExists = IfNotExists() - <USING> adapterName = AdapterName() properties = Configuration() - ( <ON> nodeGroupName = Identifier() )? - ( <HINTS> hints = Properties() )? - ( <WITH> withRecord = RecordConstructor() )? - { - ExternalDetailsDecl edd = new ExternalDetailsDecl(); - edd.setAdapter(adapterName); - edd.setProperties(properties); - try{ - stmt = new DatasetDecl(nameComponents.first, - nameComponents.second, - typeExpr, - null, - nodeGroupName != null? new Identifier(nodeGroupName): null, - hints, - DatasetType.EXTERNAL, - edd, - withRecord, - ifNotExists); - } catch (CompilationException e){ - throw new SqlppParseException(getSourceLocation(startStmtToken), e.getMessage()); + { String name; } + <WITH> + name = Identifier() + { + if (!name.equalsIgnoreCase("meta")){ + throw new SqlppParseException(getSourceLocation(startStmtToken), + "We can only support one additional associated field called \"meta\"."); } - } + } - <LEFTPAREN> metaTypeComponents = TypeName() <RIGHTPAREN> ++ metaTypeExpr = DatasetTypeSpecification() + )? + ifNotExists = IfNotExists() + primaryKeyFields = PrimaryKey() + (<AUTOGENERATED> { autogenerated = true; } )? + (<ON> nodeGroupName = Identifier() )? + ( <HINTS> hints = Properties() )? + ( LOOKAHEAD(2) <WITH> <FILTER> <ON> filterField = NestedField() )? + ( <WITH> withRecord = RecordConstructor() )? + { + if(filterField!=null && filterField.first!=0){ + throw new SqlppParseException(getSourceLocation(startStmtToken), + "A filter field can only be a field in the main record of the dataset."); + } + InternalDetailsDecl idd = new InternalDetailsDecl(primaryKeyFields.second, primaryKeyFields.first, autogenerated, + filterField == null? null : filterField.second); + try { - stmt = new DatasetDecl(nameComponents.first, nameComponents.second, typeComponents.first, typeComponents.second, - metaTypeComponents.first, metaTypeComponents.second, ++ stmt = new DatasetDecl(nameComponents.first, nameComponents.second, typeExpr, metaTypeExpr, + nodeGroupName != null ? new Identifier(nodeGroupName) : null, hints, DatasetType.INTERNAL, idd, withRecord, + ifNotExists); + return addSourceLocation(stmt, startStmtToken); + } catch (CompilationException e) { + throw new SqlppParseException(getSourceLocation(startStmtToken), e.getMessage()); + } + } +} - | ( <INTERNAL> )? - Dataset() nameComponents = QualifiedName() - typeExpr = DatasetTypeSpecification() - ( - { String name; } - <WITH> - name = Identifier() - { - if (!name.equalsIgnoreCase("meta")){ - throw new SqlppParseException(getSourceLocation(startStmtToken), - "We can only support one additional associated field called \"meta\"."); - } - } - metaTypeExpr = DatasetTypeSpecification() - )? - ifNotExists = IfNotExists() - primaryKeyFields = PrimaryKey() - (<AUTOGENERATED> { autogenerated = true; } )? - (<ON> nodeGroupName = Identifier() )? - ( <HINTS> hints = Properties() )? - ( LOOKAHEAD(2) <WITH> <FILTER> <ON> filterField = NestedField() )? - ( <WITH> withRecord = RecordConstructor() )? - { - if(filterField!=null && filterField.first!=0){ - throw new SqlppParseException(getSourceLocation(startStmtToken), - "A filter field can only be a field in the main record of the dataset."); - } - InternalDetailsDecl idd = new InternalDetailsDecl(primaryKeyFields.second, - primaryKeyFields.first, - autogenerated, - filterField == null? null : filterField.second); - try{ - stmt = new DatasetDecl(nameComponents.first, - nameComponents.second, - typeExpr, - metaTypeExpr, - nodeGroupName != null ? new Identifier(nodeGroupName) : null, - hints, - DatasetType.INTERNAL, - idd, - withRecord, - ifNotExists); - } catch (CompilationException e){ - throw new SqlppParseException(getSourceLocation(startStmtToken), e.getMessage()); - } - } - ) - { +DatasetDecl ExternalDatasetSpecification(Token startStmtToken) throws ParseException: +{ + Pair<DataverseName,Identifier> nameComponents = null; ++ TypeExpression typeExpr = null; + boolean ifNotExists = false; - Pair<DataverseName,Identifier> typeComponents = null; + String adapterName = null; + Map<String,String> properties = null; + String nodeGroupName = null; + Map<String,String> hints = new HashMap<String,String>(); + DatasetDecl stmt = null; - Pair<DataverseName,Identifier> metaTypeComponents = new Pair<DataverseName, Identifier>(null, null); + RecordConstructor withRecord = null; +} +{ + nameComponents = QualifiedName() - <LEFTPAREN> typeComponents = TypeName() <RIGHTPAREN> ++ typeExpr = DatasetTypeSpecification() + ifNotExists = IfNotExists() + <USING> adapterName = AdapterName() properties = Configuration() + ( <ON> nodeGroupName = Identifier() )? + ( <HINTS> hints = Properties() )? + ( <WITH> withRecord = RecordConstructor() )? + { + ExternalDetailsDecl edd = new ExternalDetailsDecl(); + edd.setAdapter(adapterName); + edd.setProperties(properties); + try { - stmt = new DatasetDecl(nameComponents.first, nameComponents.second, typeComponents.first, typeComponents.second, - metaTypeComponents.first, metaTypeComponents.second, ++ stmt = new DatasetDecl(nameComponents.first, nameComponents.second, typeExpr, null, + nodeGroupName != null? new Identifier(nodeGroupName): null, hints, DatasetType.EXTERNAL, edd, withRecord, + ifNotExists); return addSourceLocation(stmt, startStmtToken); + } catch (CompilationException e) { + throw new SqlppParseException(getSourceLocation(startStmtToken), e.getMessage()); } + } } + TypeExpression DatasetTypeSpecification() throws ParseException: + { + TypeExpression typeExpr = null; + } + { + ( + LOOKAHEAD(3) typeExpr = DatasetRecordTypeSpecification(true) + | typeExpr = DatasetReferenceTypeSpecification() + ) + { + return typeExpr; + } + } + + TypeExpression DatasetReferenceTypeSpecification() throws ParseException: + { + TypeExpression typeExpr = null; + } + { + <LEFTPAREN> typeExpr = TypeReference() <RIGHTPAREN> + { + return typeExpr; + } + } + + TypeExpression DatasetRecordTypeSpecification(boolean allowRecordKindModifier) throws ParseException: + { + RecordTypeDefinition recordTypeDef = null; + RecordTypeDefinition.RecordKind recordKind = null; + Token recordKindToken = null; + } + { + <LEFTPAREN> recordTypeDef = DatasetRecordTypeDef() <RIGHTPAREN> + ( recordKind = RecordTypeKind() { recordKindToken = token; } <TYPE> )? + { + if (recordKind == null) { + recordKind = RecordTypeDefinition.RecordKind.CLOSED; + } else if (!allowRecordKindModifier) { + throw createUnexpectedTokenError(recordKindToken); + } + recordTypeDef.setRecordKind(recordKind); + return recordTypeDef; + } + } + + RecordTypeDefinition DatasetRecordTypeDef() throws ParseException: + { + RecordTypeDefinition recType = new RecordTypeDefinition(); + } + { + DatasetRecordField(recType) ( <COMMA> DatasetRecordField(recType) )* + { + return recType; + } + } + + void DatasetRecordField(RecordTypeDefinition recType) throws ParseException: + { + String fieldName; + TypeExpression type = null; + boolean isUnknownable = true; + } + { + fieldName = Identifier() + type = TypeReference() ( <NOT> <NULL> { isUnknownable = false; } )? + { + recType.addField(fieldName, type, isUnknownable); + } + } + RefreshExternalDatasetStatement RefreshExternalDatasetStatement() throws ParseException: { Token startToken = null; diff --cc asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java index 84166ce,a5084e0..78c026d --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java @@@ -590,4 -595,27 +596,12 @@@ public class DatasetUtil return nodeGroup; } - // This doesn't work if the dataset or the dataverse name contains a '.' - public static Pair<String, String> getDatasetInfo(MetadataProvider metadata, String datasetArg) { - String first; - String second; - int i = datasetArg.indexOf('.'); - if (i > 0 && i < datasetArg.length() - 1) { - first = datasetArg.substring(0, i); - second = datasetArg.substring(i + 1); - } else { - first = metadata.getDefaultDataverse() == null ? null : metadata.getDefaultDataverse().getDataverseName(); - second = datasetArg; - } - return new Pair<>(first, second); - } - + public static String createInlineTypeName(String datasetName, boolean forMetaItemType) { + char typeChar = forMetaItemType ? 'm' : 'i'; + return DATASET_INLINE_TYPE_PREFIX + typeChar + '$' + datasetName; + } + - public static boolean isInlineTypeName(Dataset dataset, String typeDataverseName, String typeName) { ++ public static boolean isInlineTypeName(Dataset dataset, DataverseName typeDataverseName, String typeName) { + return dataset.getDataverseName().equals(typeDataverseName) && typeName.startsWith(DATASET_INLINE_TYPE_PREFIX); + } } diff --cc asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataLockUtil.java index 88d292b,baac712..1b88a06 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataLockUtil.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataLockUtil.java @@@ -24,48 -22,27 +24,50 @@@ import org.apache.asterix.common.metada import org.apache.asterix.common.metadata.LockList; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; -public class MetadataLockUtil { +public class MetadataLockUtil implements IMetadataLockUtil { - private MetadataLockUtil() { + @Override + public void createDataverseBegin(IMetadataLockManager lockMgr, LockList locks, DataverseName dataverseName) + throws AlgebricksException { + lockMgr.acquireDataverseReadLock(locks, dataverseName); + } + + @Override + public void dropDataverseBegin(IMetadataLockManager lockMgr, LockList locks, DataverseName dataverseName) + throws AlgebricksException { + lockMgr.acquireDataverseWriteLock(locks, dataverseName); + } + + @Override + public void createDatasetBegin(IMetadataLockManager lockMgr, LockList locks, DataverseName dataverseName, + String datasetName, DataverseName itemTypeDataverseName, String itemTypeName, + DataverseName metaItemTypeDataverseName, String metaItemTypeName, String nodeGroupName, + String compactionPolicyName, boolean isDefaultCompactionPolicy, Object datasetDetails) + throws AlgebricksException { + createDatasetBeginPre(lockMgr, locks, dataverseName, itemTypeDataverseName, itemTypeName, + metaItemTypeDataverseName, metaItemTypeName, nodeGroupName, compactionPolicyName, + isDefaultCompactionPolicy); + lockMgr.acquireDatasetWriteLock(locks, dataverseName, datasetName); } - public static void createDatasetBegin(IMetadataLockManager lockMgr, LockList locks, String dataverseName, - String itemTypeDataverseName, String itemTypeFullyQualifiedName, String metaItemTypeDataverseName, - String metaItemTypeFullyQualifiedName, String nodeGroupName, String compactionPolicyName, - String datasetFullyQualifiedName, boolean isDefaultCompactionPolicy) throws AlgebricksException { + protected final void createDatasetBeginPre(IMetadataLockManager lockMgr, LockList locks, + DataverseName dataverseName, DataverseName itemTypeDataverseName, String itemTypeName, + DataverseName metaItemTypeDataverseName, String metaItemTypeName, String nodeGroupName, + String compactionPolicyName, boolean isDefaultCompactionPolicy) throws AlgebricksException { lockMgr.acquireDataverseReadLock(locks, dataverseName); -- if (!dataverseName.equals(itemTypeDataverseName)) { ++ if (itemTypeDataverseName != null && !dataverseName.equals(itemTypeDataverseName)) { lockMgr.acquireDataverseReadLock(locks, itemTypeDataverseName); } if (metaItemTypeDataverseName != null && !metaItemTypeDataverseName.equals(dataverseName) && !metaItemTypeDataverseName.equals(itemTypeDataverseName)) { lockMgr.acquireDataverseReadLock(locks, metaItemTypeDataverseName); } - lockMgr.acquireDataTypeReadLock(locks, itemTypeDataverseName, itemTypeName); - lockMgr.acquireDataTypeReadLock(locks, itemTypeFullyQualifiedName); - if (metaItemTypeFullyQualifiedName != null - && !metaItemTypeFullyQualifiedName.equals(itemTypeFullyQualifiedName)) { - lockMgr.acquireDataTypeReadLock(locks, metaItemTypeFullyQualifiedName); ++ if (itemTypeName != null) { ++ lockMgr.acquireDataTypeReadLock(locks, itemTypeDataverseName, itemTypeName); ++ } + if (metaItemTypeDataverseName != null && !metaItemTypeDataverseName.equals(itemTypeDataverseName) + && !metaItemTypeName.equals(itemTypeName)) { + lockMgr.acquireDataTypeReadLock(locks, metaItemTypeDataverseName, metaItemTypeName); } if (nodeGroupName != null) { lockMgr.acquireNodeGroupReadLock(locks, nodeGroupName);
