This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit e956d8d6948577825747bad118c27dd379319132 Merge: 7e7960c090 3f20e39b24 Author: Hussain Towaileb <[email protected]> AuthorDate: Sat Oct 26 23:02:17 2024 +0300 Merge branch gerrit/trinity into gerrit/goldfish Ext-ref: MB-63819 Change-Id: I14251ec5e66e4ee48505b10783e1a7bd6ebf49c2 .../app/function/ActiveRequestsDatasource.java | 7 ++++ .../app/function/CompletedRequestsDatasource.java | 7 ++++ .../app/function/DatasetResourcesDatasource.java | 15 +++++++++ .../asterix/app/function/DumpIndexDatasource.java | 21 +++++++++++- .../asterix/app/function/DumpIndexRewriter.java | 2 +- .../app/function/JobSummariesDatasource.java | 7 ++++ .../asterix/app/function/PingDatasource.java | 6 ++++ .../asterix/app/function/QueryIndexDatasource.java | 18 ++++++++++ .../app/function/QueryPartitionDatasource.java | 19 +++++++++++ .../app/function/StorageComponentsDatasource.java | 15 +++++++++ .../TPCDSAllTablesDataGeneratorDatasource.java | 15 +++++++++ .../TPCDSSingleTableDataGeneratorDatasource.java | 20 ++++++++++++ .../test.000.ddl.sqlpp | 34 +++++++++++++++++++ .../test.010.update.sqlpp | 21 ++++++++++++ .../test.020.query.sqlpp | 23 +++++++++++++ .../test.030.query.sqlpp | 24 ++++++++++++++ .../test.999.ddl.sqlpp | 20 ++++++++++++ .../test.020.unorderedtxt | 2 ++ .../test.030.plan | 38 ++++++++++++++++++++++ .../src/test/resources/runtimets/sqlpp_queries.xml | 5 +++ .../resources/runtimets/testsuite_it_python.xml | 2 ++ .../asterix/metadata/declared/DataSource.java | 24 ++++++++++++++ .../metadata/declared/DatasetDataSource.java | 5 --- .../metadata/declared/FunctionDataSource.java | 16 +++++++++ .../core/algebra/metadata/IDataSource.java | 4 +-- .../visitors/IsomorphismOperatorVisitor.java | 17 +++++----- 26 files changed, 369 insertions(+), 18 deletions(-) diff --cc asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java index 1eac011a6a,e5d78a4721..23fcf8c1df --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java @@@ -39,18 -42,22 +42,25 @@@ public class DumpIndexDatasource extend private final IndexDataflowHelperFactory indexDataflowHelperFactory; private final RecordDescriptor recDesc; private final IBinaryComparatorFactory[] comparatorFactories; - private final AlgebricksAbsolutePartitionConstraint storageLocations; + private final AlgebricksAbsolutePartitionConstraint constraint; + private final int[][] partitionsMap; + private final Index index; public DumpIndexDatasource(INodeDomain domain, IndexDataflowHelperFactory indexDataflowHelperFactory, RecordDescriptor recDesc, IBinaryComparatorFactory[] comparatorFactories, - AlgebricksAbsolutePartitionConstraint constraint, int[][] partitionsMap) throws AlgebricksException { - AlgebricksAbsolutePartitionConstraint storageLocations, Index index) throws AlgebricksException { ++ AlgebricksAbsolutePartitionConstraint constraint, int[][] partitionsMap, Index index) ++ throws AlgebricksException { super(DUMP_INDEX_DATASOURCE_ID, DumpIndexRewriter.DUMP_INDEX, domain); this.indexDataflowHelperFactory = indexDataflowHelperFactory; this.recDesc = recDesc; this.comparatorFactories = comparatorFactories; - this.storageLocations = storageLocations; + this.constraint = constraint; + this.partitionsMap = partitionsMap; + this.index = index; + } + + public Index getIndex() { + return index; } @Override diff --cc asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java index d50315b3d4,b5ea73fac9..825d5718a0 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java @@@ -80,20 -66,13 +80,20 @@@ public class DumpIndexRewriter extends } ISecondaryIndexOperationsHelper secondaryIndexHelper = SecondaryIndexOperationsHelper.createIndexOperationsHelper(dataset, index, metadataProvider, loc); + PartitioningProperties partitioningProperties = + metadataProvider.getPartitioningProperties(dataset, index.getIndexName()); IndexDataflowHelperFactory indexDataflowHelperFactory = new IndexDataflowHelperFactory(metadataProvider.getStorageComponentProvider().getStorageManager(), - secondaryIndexHelper.getSecondaryFileSplitProvider()); + partitioningProperties.getSplitsProvider()); AlgebricksAbsolutePartitionConstraint secondaryPartitionConstraint = - (AlgebricksAbsolutePartitionConstraint) secondaryIndexHelper.getSecondaryPartitionConstraint(); + (AlgebricksAbsolutePartitionConstraint) partitioningProperties.getConstraints(); return new DumpIndexDatasource(context.getComputationNodeDomain(), indexDataflowHelperFactory, secondaryIndexHelper.getSecondaryRecDesc(), secondaryIndexHelper.getSecondaryComparatorFactories(), - secondaryPartitionConstraint, partitioningProperties.getComputeStorageMap()); - secondaryPartitionConstraint, index); ++ secondaryPartitionConstraint, partitioningProperties.getComputeStorageMap(), index); + } + + @Override + protected boolean invalidArgs(List<Mutable<ILogicalExpression>> args) { + return args.size() < 3; } } diff --cc asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryIndexDatasource.java index 52054d6d1d,1c036f87e4..d6a45af34e --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryIndexDatasource.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryIndexDatasource.java @@@ -135,7 -134,16 +144,16 @@@ public class QueryIndexDatasource exten } private static DataSourceId createQueryIndexDataSourceId(Dataset dataset, String indexName) { - return new DataSourceId(dataset.getDataverseName(), dataset.getDatasetName(), + return new DataSourceId(dataset.getDatabaseName(), dataset.getDataverseName(), dataset.getDatasetName(), new String[] { indexName, QueryIndexRewriter.QUERY_INDEX.getName() }); } + + @Override + public boolean sameFunctionDatasource(FunctionDataSource other) { + if (!Objects.equals(this.functionId, other.getFunctionId())) { + return false; + } + QueryIndexDatasource that = (QueryIndexDatasource) other; + return Objects.equals(this.ds, that.getDataset()) && Objects.equals(this.indexName, that.getIndexName()); + } } diff --cc asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryPartitionDatasource.java index 4d40ba6913,0000000000..4917b33cd6 mode 100644,000000..100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryPartitionDatasource.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryPartitionDatasource.java @@@ -1,131 -1,0 +1,150 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.app.function; + +import java.util.ArrayList; +import java.util.List; ++import java.util.Objects; + +import org.apache.asterix.common.cluster.IClusterStateManager; +import org.apache.asterix.metadata.api.IDatasourceFunction; +import org.apache.asterix.metadata.declared.DataSourceId; +import org.apache.asterix.metadata.declared.FunctionDataSource; +import org.apache.asterix.metadata.declared.MetadataProvider; +import org.apache.asterix.metadata.entities.Dataset; +import org.apache.asterix.om.types.ARecordType; +import org.apache.asterix.om.types.IAType; +import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint; +import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint; +import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; +import org.apache.hyracks.algebricks.common.utils.Pair; +import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext; +import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable; +import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment; +import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSource; +import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSourcePropertiesProvider; +import org.apache.hyracks.algebricks.core.algebra.metadata.IProjectionFiltrationInfo; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema; +import org.apache.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty; +import org.apache.hyracks.algebricks.core.algebra.properties.INodeDomain; +import org.apache.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector; +import org.apache.hyracks.algebricks.core.algebra.properties.RandomPartitioningProperty; +import org.apache.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector; +import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenContext; +import org.apache.hyracks.api.dataflow.IOperatorDescriptor; +import org.apache.hyracks.api.job.JobSpecification; +import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory; + +public class QueryPartitionDatasource extends FunctionDataSource { + + private final Dataset ds; + private final AlgebricksAbsolutePartitionConstraint storageLocations; + private final int partitionNum; + + public QueryPartitionDatasource(Dataset ds, INodeDomain domain, + AlgebricksAbsolutePartitionConstraint storageLocations, ARecordType recType, int partitionNum) + throws AlgebricksException { + super(createQueryPartitionDataSourceId(ds), QueryPartitionRewriter.QUERY_PARTITION, domain, recType); + if (partitionNum < 0) { + throw new IllegalArgumentException("partition must be >= 0"); + } + this.partitionNum = partitionNum; + this.ds = ds; + this.storageLocations = storageLocations; + } + ++ public Dataset getDatasource() { ++ return ds; ++ } ++ ++ public int getPartitionNumber() { ++ return partitionNum; ++ } ++ + @Override + protected void initSchemaType(IAType iType) { + ARecordType type = (ARecordType) iType; + IAType[] fieldTypes = type.getFieldTypes(); + schemaTypes = new IAType[fieldTypes.length]; + System.arraycopy(fieldTypes, 0, schemaTypes, 0, schemaTypes.length); + } + + @Override + protected AlgebricksAbsolutePartitionConstraint getLocations(IClusterStateManager csm, MetadataProvider md) { + return storageLocations; + } + + @Override + public boolean isScanAccessPathALeaf() { + // the index scan op is not a leaf op. the ETS op will start the scan of the index. we need the ETS op below + // the index scan to be still generated + return false; + } + + @Override + protected IDatasourceFunction createFunction(MetadataProvider metadataProvider, + AlgebricksAbsolutePartitionConstraint locations) { + throw new UnsupportedOperationException("query-partition() does not use record reader adapter"); + } + + @Override + public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> buildDatasourceScanRuntime( + MetadataProvider metadataProvider, IDataSource<DataSourceId> dataSource, + List<LogicalVariable> scanVariables, List<LogicalVariable> projectVariables, boolean projectPushed, + List<LogicalVariable> minFilterVars, List<LogicalVariable> maxFilterVars, + ITupleFilterFactory tupleFilterFactory, long outputLimit, IOperatorSchema opSchema, + IVariableTypeEnvironment typeEnv, JobGenContext context, JobSpecification jobSpec, Object implConfig, + IProjectionFiltrationInfo projectionInfo) throws AlgebricksException { + return metadataProvider.getBtreePartitionSearchRuntime(jobSpec, opSchema, typeEnv, context, ds, + tupleFilterFactory, outputLimit, partitionNum); + } + + @Override + public IDataSourcePropertiesProvider getPropertiesProvider() { + return new IDataSourcePropertiesProvider() { + @Override + public IPhysicalPropertiesVector computeRequiredProperties(List<LogicalVariable> scanVariables, + IOptimizationContext ctx) { + return StructuralPropertiesVector.EMPTY_PROPERTIES_VECTOR; + } + + @Override + public IPhysicalPropertiesVector computeDeliveredProperties(List<LogicalVariable> scanVariables, + IOptimizationContext ctx) { + List<ILocalStructuralProperty> propsLocal = new ArrayList<>(1); + return new StructuralPropertiesVector(new RandomPartitioningProperty(domain), propsLocal); + } + }; + } + + private static DataSourceId createQueryPartitionDataSourceId(Dataset dataset) { + return new DataSourceId(dataset.getDatabaseName(), dataset.getDataverseName(), dataset.getDatasetName(), + new String[] { dataset.getDatasetName(), QueryPartitionRewriter.QUERY_PARTITION.getName() }); + } ++ ++ @Override ++ public boolean sameFunctionDatasource(FunctionDataSource other) { ++ if (!Objects.equals(this.functionId, other.getFunctionId())) { ++ return false; ++ } ++ QueryPartitionDatasource that = (QueryPartitionDatasource) other; ++ return Objects.equals(this.ds, that.getDatasource()) ++ && Objects.equals(this.partitionNum, that.getPartitionNumber()); ++ } +} diff --cc asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSAllTablesDataGeneratorDatasource.java index b4d1b03ab2,489b052795..ac19b0057e --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSAllTablesDataGeneratorDatasource.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSAllTablesDataGeneratorDatasource.java @@@ -63,7 -69,16 +69,16 @@@ public class TPCDSAllTablesDataGenerato } @Override - protected AlgebricksAbsolutePartitionConstraint getLocations(IClusterStateManager csm) { - return csm.getSortedClusterLocations(); + protected AlgebricksAbsolutePartitionConstraint getLocations(IClusterStateManager csm, MetadataProvider md) { + return md.getDataPartitioningProvider().getClusterLocations(); } + + @Override + public boolean sameFunctionDatasource(FunctionDataSource other) { + if (!Objects.equals(this.functionId, other.getFunctionId())) { + return false; + } + TPCDSAllTablesDataGeneratorDatasource that = (TPCDSAllTablesDataGeneratorDatasource) other; + return Objects.equals(this.scalingFactor, that.getScalingFactor()); + } } diff --cc asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSSingleTableDataGeneratorDatasource.java index 3bbaee2079,c90dff173a..6551346ed5 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSSingleTableDataGeneratorDatasource.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSSingleTableDataGeneratorDatasource.java @@@ -67,7 -77,17 +77,17 @@@ public class TPCDSSingleTableDataGenera } @Override - protected AlgebricksAbsolutePartitionConstraint getLocations(IClusterStateManager csm) { - return csm.getSortedClusterLocations(); + protected AlgebricksAbsolutePartitionConstraint getLocations(IClusterStateManager csm, MetadataProvider md) { + return md.getDataPartitioningProvider().getClusterLocations(); } + + @Override + public boolean sameFunctionDatasource(FunctionDataSource other) { + if (!Objects.equals(this.functionId, other.getFunctionId())) { + return false; + } + TPCDSSingleTableDataGeneratorDatasource that = (TPCDSSingleTableDataGeneratorDatasource) other; + return Objects.equals(this.tableName, that.getTableName()) + && Objects.equals(this.scalingFactor, that.getScalingFactor()); + } } diff --cc asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DataSource.java index 1fa84f6aca,36273d2d3e..77620419ce --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DataSource.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DataSource.java @@@ -166,5 -166,28 +167,28 @@@ public abstract class DataSource implem List<LogicalVariable> minFilterVars, List<LogicalVariable> maxFilterVars, ITupleFilterFactory tupleFilterFactory, long outputLimit, IOperatorSchema opSchema, IVariableTypeEnvironment typeEnv, JobGenContext context, JobSpecification jobSpec, Object implConfig, - IProjectionInfo<?> projectionInfo) throws AlgebricksException; + IProjectionFiltrationInfo projectionFiltrationInfo) throws AlgebricksException; + + @Override + public boolean sameAs(IDataSource<?> other) { + if (this == other) { + return true; + } + + if (!(other instanceof DataSource)) { + return false; + } + + DataSource that = (DataSource) other; + if (!Objects.equals(this.id, that.getId()) || !Objects.equals(this.datasourceType, that.getDatasourceType())) { + return false; + } + + if (this.datasourceType == Type.EXTERNAL_DATASET && that.getDatasourceType() == Type.EXTERNAL_DATASET + && !Objects.equals(this.getProperties(), other.getProperties())) { + return false; + } + + return true; + } } diff --cc asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java index 91f7615cdc,bce22c1acd..2322e53ac9 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java @@@ -137,6 -124,22 +137,22 @@@ public abstract class FunctionDataSourc } protected static DataSourceId createDataSourceId(FunctionIdentifier fid, String... parameters) { - return new DataSourceId(FunctionSignature.getDataverseName(fid), fid.getName(), parameters); + return new DataSourceId(fid.getDatabase(), FunctionSignature.getDataverseName(fid), fid.getName(), parameters); } + + protected abstract boolean sameFunctionDatasource(FunctionDataSource other); + + @Override + public boolean sameAs(IDataSource<?> other) { + if (!super.sameAs(other)) { + return false; + } + + if (!(other instanceof FunctionDataSource)) { + return false; + } + + FunctionDataSource that = (FunctionDataSource) other; + return sameFunctionDatasource(that); + } } diff --cc hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java index 05f41a0535,7ee3c53e9d..b985d09ce5 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java @@@ -491,12 -485,13 +491,13 @@@ public class IsomorphismOperatorVisito if (!isomorphic) { return Boolean.FALSE; } - IDataSource<?> dataSource = op.getDataSource(); - IDataSource<?> argDataSource = argScan.getDataSource(); - if (dataSource.compareProperties() && argDataSource.compareProperties() - && !Objects.equals(dataSource.getProperties(), argDataSource.getProperties())) { + + isomorphic = op.getOutputLimit() == argScan.getOutputLimit() - && Objects.equals(op.getProjectionInfo(), argScan.getProjectionInfo()); ++ && Objects.equals(op.getProjectionFiltrationInfo(), argScan.getProjectionFiltrationInfo()); + if (!isomorphic) { return Boolean.FALSE; } + DataSourceScanOperator scanOpArg = (DataSourceScanOperator) copyAndSubstituteVar(op, arg); ILogicalExpression opCondition = op.getSelectCondition() != null ? op.getSelectCondition().getValue() : null; ILogicalExpression argCondition =
