This is an automated email from the ASF dual-hosted git repository. dlych pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit b939313fc095b5aadce152e339f6cbfcf361fbb2 Merge: 881578e 13a40e1 Author: Dmitry Lychagin <[email protected]> AuthorDate: Mon Apr 12 17:21:22 2021 -0700 Merge branch 'cheshire-cat' Change-Id: Iaa3ab431d3225c9f95ec530b8099bf211684c34f .../asterix/optimizer/base/AnalysisUtil.java | 3 +- ...ceMaterializationForInsertWithSelfScanRule.java | 3 +- .../rules/PushLimitIntoPrimarySearchRule.java | 5 +- .../rules/am/AccessMethodJobGenParams.java | 3 +- .../optimizer/rules/am/BTreeJobGenParams.java | 3 +- .../rules/am/InvertedIndexJobGenParams.java | 3 +- .../rules/am/OptimizableOperatorSubTree.java | 2 +- .../optimizer/rules/am/RTreeJobGenParams.java | 3 +- .../api/http/server/AbstractNCUdfServlet.java | 5 +- .../api/http/server/RebalanceApiServlet.java | 9 +- .../asterix/api/http/server/ServletUtil.java | 4 +- .../asterix/app/translator/QueryTranslator.java | 68 +++++-- .../dataflow/CheckpointInSecondaryIndexTest.java | 7 +- .../dataflow/GlobalVirtualBufferCacheTest.java | 11 +- .../test/dataflow/MultiPartitionLSMIndexTest.java | 7 +- .../dataflow/SearchCursorComponentSwitchTest.java | 5 +- .../asterix/test/dataflow/StorageTestUtils.java | 15 +- .../asterix/test/metadata/MetadataManagerTest.java | 17 +- .../metadata/MetadataManagerWindowsOsTest.java | 18 +- .../storage/IndexDropOperatorNodePushableTest.java | 5 +- .../invalid-dataset-name.3.ddl.sqlpp | 5 + .../invalid-dataset-name.4.ddl.sqlpp | 9 + ....ddl.sqlpp => invalid-dataset-name.5.ddl.sqlpp} | 11 +- .../invalid-dataverse-name.6.ddl.sqlpp} | 6 +- .../invalid-dataverse-name.7.ddl.sqlpp} | 7 +- .../invalid-feed-name.4.ddl.sqlpp | 20 ++ .../invalid-feed-name.5.ddl.sqlpp} | 22 ++- .../invalid-feed-policy-name.3.ddl.sqlpp | 10 + .../invalid-feed-policy-name.4.ddl.sqlpp} | 12 +- .../invalid-index-name.3.ddl.sqlpp | 5 + .../invalid-index-name.4.ddl.sqlpp | 9 + .../invalid-index-name.5.ddl.sqlpp} | 13 +- .../invalid-type-name.3.ddl.sqlpp | 5 + .../invalid-type-name.4.ddl.sqlpp | 11 ++ ...e.3.ddl.sqlpp => invalid-type-name.5.ddl.sqlpp} | 13 +- .../invalid-udf-name/invalid-udf-name.3.ddl.sqlpp | 5 + .../invalid-udf-name/invalid-udf-name.4.ddl.sqlpp | 11 ++ ...me.3.ddl.sqlpp => invalid-udf-name.5.ddl.sqlpp} | 13 +- .../test/resources/runtimets/testsuite_sqlpp.xml | 30 ++- .../common/functions/FunctionSignature.java | 7 +- .../asterix/common/metadata/DataverseName.java | 209 +++++++++++---------- .../asterix/common/storage/ResourceReference.java | 21 ++- .../asterix/common/metadata/DataverseNameTest.java | 108 ++++++++--- .../external/library/ExternalLibraryManager.java | 8 +- .../asterix/external/util/ExternalDataUtils.java | 2 +- .../external/feed/test/InputHandlerTest.java | 8 +- .../asterix/lang/common/util/FunctionUtil.java | 9 +- .../visitor/VariableCheckAndRewriteVisitor.java | 17 +- .../asterix-lang-sqlpp/src/main/javacc/SQLPP.jj | 105 +++++++---- .../metadata/declared/LoadableDataSource.java | 5 +- .../metadata/declared/MetadataProvider.java | 2 +- .../CompactionPolicyTupleTranslator.java | 4 +- .../DatasetTupleTranslator.java | 3 +- .../DatasourceAdapterTupleTranslator.java | 3 +- .../DataverseTupleTranslator.java | 3 +- .../ExternalFileTupleTranslator.java | 3 +- .../FeedConnectionTupleTranslator.java | 3 +- .../FeedPolicyTupleTranslator.java | 3 +- .../FeedTupleTranslator.java | 3 +- ...ullTextFilterMetadataEntityTupleTranslator.java | 2 +- .../FunctionTupleTranslator.java | 7 +- .../LibraryTupleTranslator.java | 3 +- .../SynonymTupleTranslator.java | 3 +- .../DecodeDataverseDisplayNameDescriptor.java | 16 +- .../functions/DecodeDataverseNameDescriptor.java | 10 +- 65 files changed, 698 insertions(+), 282 deletions(-) diff --cc asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj index d8fd123,0aa9b58..c4c86f4 --- a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj +++ b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj @@@ -1093,145 -1100,29 +1113,145 @@@ CreateIndexStatement IndexSpecification ( indexName = Identifier() ifNotExists = IfNotExists() <ON> nameComponents = QualifiedName() - <LEFTPAREN> ( fieldPair = OpenField() + <LEFTPAREN> { startElementToken = token; } + indexedElement = IndexedElement(startElementToken) { + indexedElementList.add(indexedElement); + hasUnnest |= indexedElement.hasUnnest(); + } + (<COMMA> { startElementToken = token; } + indexedElement = IndexedElement(startElementToken) { + indexedElementList.add(indexedElement); + hasUnnest |= indexedElement.hasUnnest(); + } + )* + <RIGHTPAREN> + ( <TYPE> indexParams = IndexType() )? ( <ENFORCED> { enforced = true; } )? + ) + { + IndexType indexType; + int gramLength; + if (indexParams != null) { + indexType = indexParams.type; + gramLength = indexParams.gramLength; + fullTextConfigName = indexParams.fullTextConfig; + } else { + indexType = hasUnnest ? IndexType.ARRAY : IndexType.BTREE; + gramLength = -1; + fullTextConfigName = null; + } + CreateIndexStatement stmt = new CreateIndexStatement(nameComponents.first, nameComponents.second, + new Identifier(indexName), indexType, indexedElementList, enforced, gramLength, fullTextConfigName, ifNotExists); + return addSourceLocation(stmt, startStmtToken); + } +} + +CreateIndexStatement.IndexedElement IndexedElement(Token startElementToken) throws ParseException: +{ + Triple<Integer, List<List<String>>, List<Pair<List<String>, IndexedTypeExpression>>> element = null; + Pair<List<String>, IndexedTypeExpression> elementSimple = null; + int elementSimpleSource = 0; +} +{ + ( + element = IndexedElementUnnestSelect() + | ( + LOOKAHEAD({ laIdentifier(META) && laToken(2, LEFTPAREN) && laToken(3, RIGHTPAREN) }) + <IDENTIFIER> { expectToken(META); } <LEFTPAREN> <RIGHTPAREN> + <DOT> elementSimple = IndexedField() + { elementSimpleSource = 1; } + ) + | elementSimple = IndexedField() + | <LEFTPAREN> ( element = IndexedElementUnnestSelect() | elementSimple = IndexedField() ) <RIGHTPAREN> + ) + { + int source; + List<List<String>> unnestList; + List<Pair<List<String>, IndexedTypeExpression>> projectList; + if (elementSimple != null) { + source = elementSimpleSource; + unnestList = null; + projectList = Collections.singletonList(elementSimple); + } else { + source = element.first; + unnestList = element.second; + projectList = element.third; + } + CreateIndexStatement.IndexedElement ie = new CreateIndexStatement.IndexedElement(source, unnestList, projectList); + ie.setSourceLocation(getSourceLocation(startElementToken)); + return ie; + } +} + +Triple<Integer, List<List<String>>, List<Pair<List<String>, IndexedTypeExpression>>> IndexedElementUnnestSelect() + throws ParseException: +{ + int source = 0; + Pair<List<List<String>>, List<Pair<List<String>, IndexedTypeExpression>>> element = null; +} +{ + <UNNEST> + ( + ( + LOOKAHEAD({ laIdentifier(META) && laToken(2, LEFTPAREN) && laToken(3, RIGHTPAREN) }) + <IDENTIFIER> { expectToken(META); } <LEFTPAREN> <RIGHTPAREN> + <DOT> element = IndexedElementUnnestSelectBody() { source = 1; } + ) | element = IndexedElementUnnestSelectBody() + ) + { + return new Triple<Integer, List<List<String>>, List<Pair<List<String>, IndexedTypeExpression>>>( + source, element.first, element.second + ); + } +} + +Pair<List<List<String>>, List<Pair<List<String>, IndexedTypeExpression>>> IndexedElementUnnestSelectBody() + throws ParseException: +{ - List<String> path = null; ++ Triple<List<String>, Token, Token> path = null; + IndexedTypeExpression type = null; + List<List<String>> unnestList = new ArrayList(); + List<Pair<List<String>, IndexedTypeExpression>> projectList = new ArrayList(); +} +{ - path = MultipartIdentifier() { unnestList.add(path); } - ( <UNNEST> path = MultipartIdentifier() { unnestList.add(path); })* ++ path = MultipartIdentifier() { unnestList.add(path.first); } ++ ( <UNNEST> path = MultipartIdentifier() { unnestList.add(path.first); })* + ( + ( <COLON> type = IndexedTypeExpr(false) { - stmt.addFieldExprPair(fieldPair.second); - stmt.addFieldIndexIndicator(fieldPair.first); + projectList.add(new Pair<List<String>, IndexedTypeExpression>(null, type)); } - ) (<COMMA> fieldPair = OpenField() + ) | + ( + <SELECT> path = MultipartIdentifier() ( <COLON> type = IndexedTypeExpr(false) )? { - projectList.add(new Pair<List<String>, IndexedTypeExpression>(path, type)); - stmt.addFieldExprPair(fieldPair.second); - stmt.addFieldIndexIndicator(fieldPair.first); ++ projectList.add(new Pair<List<String>, IndexedTypeExpression>(path.first, type)); } - )* <RIGHTPAREN> ( <TYPE> indexType = IndexType() )? ( <ENFORCED> { enforced = true; } )? - ) + ( <COMMA> path = MultipartIdentifier() ( <COLON> type = IndexedTypeExpr(false) )? + { - projectList.add(new Pair<List<String>, IndexedTypeExpression>(path, type)); ++ projectList.add(new Pair<List<String>, IndexedTypeExpression>(path.first, type)); + } + )* + ) + )? { - stmt.setIndexName(new Identifier(indexName)); - stmt.setIfNotExists(ifNotExists); - stmt.setDataverseName(nameComponents.first); - stmt.setDatasetName(nameComponents.second); - if (indexType != null) { - stmt.setIndexType(indexType.type); - stmt.setGramLength(indexType.gramLength); + if (projectList.isEmpty()) { + // To support the case (<UNNEST> IDENTIFIER)* IDENTIFIER w/o any type specification. + projectList.add(new Pair<List<String>, IndexedTypeExpression>(null, null)); } - stmt.setEnforced(enforced); - return addSourceLocation(stmt, startStmtToken); + + return new Pair<List<List<String>>, List<Pair<List<String>, IndexedTypeExpression>>>(unnestList, projectList); + } +} + +Pair<List<String>, IndexedTypeExpression> IndexedField() throws ParseException: +{ - List<String> path = null; ++ Triple<List<String>, Token, Token> path = null; + IndexedTypeExpression type = null; +} +{ + path = MultipartIdentifier() ( <COLON> type = IndexedTypeExpr(true) )? + { - return new Pair<List<String>, IndexedTypeExpression>(path, type); ++ return new Pair<List<String>, IndexedTypeExpression>(path.first, type); } } diff --cc asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FullTextFilterMetadataEntityTupleTranslator.java index 4845310,0000000..2212e82 mode 100644,000000..100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FullTextFilterMetadataEntityTupleTranslator.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FullTextFilterMetadataEntityTupleTranslator.java @@@ -1,201 -1,0 +1,201 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.asterix.metadata.entitytupletranslators; + +import static org.apache.asterix.metadata.bootstrap.MetadataRecordTypes.FIELD_NAME_FULL_TEXT_STOPWORD_LIST; +import static org.apache.asterix.metadata.bootstrap.MetadataRecordTypes.FULL_TEXT_ARECORD_DATAVERSE_NAME_FIELD_INDEX; +import static org.apache.asterix.metadata.bootstrap.MetadataRecordTypes.FULL_TEXT_ARECORD_FILTER_NAME_FIELD_INDEX; +import static org.apache.asterix.metadata.bootstrap.MetadataRecordTypes.FULL_TEXT_ARECORD_FILTER_TYPE_FIELD_INDEX; + +import java.util.List; + +import org.apache.asterix.builders.OrderedListBuilder; +import org.apache.asterix.common.exceptions.AsterixException; +import org.apache.asterix.common.exceptions.ErrorCode; +import org.apache.asterix.common.metadata.DataverseName; +import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider; +import org.apache.asterix.metadata.bootstrap.MetadataPrimaryIndexes; +import org.apache.asterix.metadata.bootstrap.MetadataRecordTypes; +import org.apache.asterix.metadata.entities.FullTextFilterMetadataEntity; +import org.apache.asterix.om.base.AInt8; +import org.apache.asterix.om.base.AOrderedList; +import org.apache.asterix.om.base.ARecord; +import org.apache.asterix.om.base.AString; +import org.apache.asterix.om.base.IACursor; +import org.apache.asterix.om.types.AOrderedListType; +import org.apache.asterix.om.types.BuiltinType; +import org.apache.asterix.runtime.fulltext.AbstractFullTextFilterDescriptor; +import org.apache.asterix.runtime.fulltext.StopwordsFullTextFilterDescriptor; +import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; +import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer; +import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.data.std.util.ArrayBackedValueStorage; +import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder; +import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference; +import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; +import org.apache.hyracks.storage.am.lsm.invertedindex.fulltext.FullTextFilterType; + +import com.google.common.collect.ImmutableList; + +public class FullTextFilterMetadataEntityTupleTranslator extends AbstractTupleTranslator<FullTextFilterMetadataEntity> { + + private static final int FULLTEXT_FILTER_PAYLOAD_TUPLE_FIELD_INDEX = 2; + protected final ArrayTupleReference tuple; + protected final ISerializerDeserializer<AInt8> int8Serde = + SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.AINT8); + + protected FullTextFilterMetadataEntityTupleTranslator(boolean getTuple) { + super(getTuple, MetadataPrimaryIndexes.FULL_TEXT_FILTER_DATASET, FULLTEXT_FILTER_PAYLOAD_TUPLE_FIELD_INDEX); + if (getTuple) { + tuple = new ArrayTupleReference(); + } else { + tuple = null; + } + } + + @Override + protected FullTextFilterMetadataEntity createMetadataEntityFromARecord(ARecord aRecord) throws AlgebricksException { + AString dataverseName = (AString) aRecord.getValueByPos(FULL_TEXT_ARECORD_DATAVERSE_NAME_FIELD_INDEX); + AString filterName = (AString) aRecord.getValueByPos(FULL_TEXT_ARECORD_FILTER_NAME_FIELD_INDEX); + AString filterTypeAString = (AString) aRecord.getValueByPos(FULL_TEXT_ARECORD_FILTER_TYPE_FIELD_INDEX); + + FullTextFilterType filterType = FullTextFilterType.getEnumIgnoreCase(filterTypeAString.getStringValue()); + AbstractFullTextFilterDescriptor filterDescriptor; + switch (filterType) { + case STOPWORDS: + return createStopwordsFilterDescriptorFromARecord(dataverseName, filterName, aRecord); + case STEMMER: + case SYNONYM: + default: + throw new AsterixException(ErrorCode.METADATA_ERROR, "Not supported yet"); + } + } + + public FullTextFilterMetadataEntity createStopwordsFilterDescriptorFromARecord(AString dataverseName, AString name, - ARecord aRecord) { ++ ARecord aRecord) throws AlgebricksException { + ImmutableList.Builder<String> stopwordsBuilder = ImmutableList.<String> builder(); + IACursor stopwordsCursor = ((AOrderedList) (aRecord + .getValueByPos(MetadataRecordTypes.FULLTEXT_ENTITY_ARECORD_STOPWORD_LIST_FIELD_INDEX))).getCursor(); + while (stopwordsCursor.next()) { + stopwordsBuilder.add(((AString) stopwordsCursor.get()).getStringValue()); + } + + StopwordsFullTextFilterDescriptor filterDescriptor = new StopwordsFullTextFilterDescriptor( + DataverseName.createFromCanonicalForm(dataverseName.getStringValue()), name.getStringValue(), + stopwordsBuilder.build()); + return new FullTextFilterMetadataEntity(filterDescriptor); + } + + private void writeKeyAndValue2FieldVariables(String key, String value) throws HyracksDataException { + fieldName.reset(); + aString.setValue(key); + stringSerde.serialize(aString, fieldName.getDataOutput()); + + fieldValue.reset(); + aString.setValue(value); + stringSerde.serialize(aString, fieldValue.getDataOutput()); + } + + private void writeOrderedList2RecordBuilder(String strFieldName, List<String> list) throws HyracksDataException { + fieldName.reset(); + aString.setValue(strFieldName); + stringSerde.serialize(aString, fieldName.getDataOutput()); + + OrderedListBuilder listBuilder = new OrderedListBuilder(); + listBuilder.reset(new AOrderedListType(BuiltinType.ASTRING, null)); + ArrayBackedValueStorage itemValue = new ArrayBackedValueStorage(); + for (String s : list) { + itemValue.reset(); + aString.setValue(s); + stringSerde.serialize(aString, itemValue.getDataOutput()); + listBuilder.addItem(itemValue); + } + + fieldValue.reset(); + listBuilder.write(fieldValue.getDataOutput(), true); + + recordBuilder.addField(fieldName, fieldValue); + } + + private void writeStopwordFilterDescriptor(StopwordsFullTextFilterDescriptor stopwordsFullTextFilterDescriptor) + throws HyracksDataException { + writeOrderedList2RecordBuilder(FIELD_NAME_FULL_TEXT_STOPWORD_LIST, + stopwordsFullTextFilterDescriptor.getStopwordList()); + } + + private void writeFulltextFilter(AbstractFullTextFilterDescriptor filterDescriptor) + throws AsterixException, HyracksDataException { + fieldValue.reset(); + aString.setValue(filterDescriptor.getDataverseName().getCanonicalForm()); + stringSerde.serialize(aString, fieldValue.getDataOutput()); + recordBuilder.addField(FULL_TEXT_ARECORD_DATAVERSE_NAME_FIELD_INDEX, fieldValue); + + fieldValue.reset(); + aString.setValue(filterDescriptor.getName()); + stringSerde.serialize(aString, fieldValue.getDataOutput()); + recordBuilder.addField(FULL_TEXT_ARECORD_FILTER_NAME_FIELD_INDEX, fieldValue); + + fieldValue.reset(); + aString.setValue(filterDescriptor.getFilterType().getValue()); + stringSerde.serialize(aString, fieldValue.getDataOutput()); + recordBuilder.addField(FULL_TEXT_ARECORD_FILTER_TYPE_FIELD_INDEX, fieldValue); + + switch (filterDescriptor.getFilterType()) { + case STOPWORDS: + writeStopwordFilterDescriptor((StopwordsFullTextFilterDescriptor) filterDescriptor); + break; + case STEMMER: + case SYNONYM: + default: + throw new AsterixException(ErrorCode.METADATA_ERROR, "Not supported yet"); + } + } + + private void writeIndex(String dataverseName, String filterName, ArrayTupleBuilder tupleBuilder) + throws HyracksDataException { + aString.setValue(dataverseName); + stringSerde.serialize(aString, tupleBuilder.getDataOutput()); + tupleBuilder.addFieldEndOffset(); + + aString.setValue(filterName); + stringSerde.serialize(aString, tupleBuilder.getDataOutput()); + tupleBuilder.addFieldEndOffset(); + } + + @Override + public ITupleReference getTupleFromMetadataEntity(FullTextFilterMetadataEntity filterMetadataEntity) + throws HyracksDataException, AsterixException { + tupleBuilder.reset(); + + writeIndex(filterMetadataEntity.getFullTextFilter().getDataverseName().getCanonicalForm(), + filterMetadataEntity.getFullTextFilter().getName(), tupleBuilder); + + // Write the record + recordBuilder.reset(MetadataRecordTypes.FULL_TEXT_FILTER_RECORDTYPE); + + writeFulltextFilter(filterMetadataEntity.getFullTextFilter()); + + recordBuilder.write(tupleBuilder.getDataOutput(), true); + tupleBuilder.addFieldEndOffset(); + + tuple.reset(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray()); + return tuple; + } +}
