This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit ce064bdf040149c7811bc8bb83c4b820af24b452 Author: Hussain Towaileb <[email protected]> AuthorDate: Thu Oct 24 08:40:51 2024 +0300 [ASTERIXDB-3521][FUN]: Verify datasource function args before replicating Ext-ref: MB-63994 Change-Id: I94f86e0f1e949e94a161d72be5531233192dd0c8 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19005 Tested-by: Jenkins <[email protected]> Reviewed-by: Hussain Towaileb <[email protected]> Reviewed-by: Ali Alsuliman <[email protected]> --- .../app/function/ActiveRequestsDatasource.java | 7 ++++ .../app/function/CompletedRequestsDatasource.java | 7 ++++ .../app/function/DatasetResourcesDatasource.java | 15 +++++++++ .../asterix/app/function/DumpIndexDatasource.java | 20 +++++++++++- .../asterix/app/function/DumpIndexRewriter.java | 2 +- .../app/function/JobSummariesDatasource.java | 7 ++++ .../asterix/app/function/PingDatasource.java | 6 ++++ .../asterix/app/function/QueryIndexDatasource.java | 18 ++++++++++ .../app/function/StorageComponentsDatasource.java | 15 +++++++++ .../TPCDSAllTablesDataGeneratorDatasource.java | 15 +++++++++ .../TPCDSSingleTableDataGeneratorDatasource.java | 20 ++++++++++++ .../test.000.ddl.sqlpp | 34 +++++++++++++++++++ .../test.010.update.sqlpp | 21 ++++++++++++ .../test.020.query.sqlpp | 23 +++++++++++++ .../test.030.query.sqlpp | 24 ++++++++++++++ .../test.999.ddl.sqlpp | 20 ++++++++++++ .../test.020.unorderedtxt | 2 ++ .../test.030.plan | 38 ++++++++++++++++++++++ .../src/test/resources/runtimets/sqlpp_queries.xml | 5 +++ .../resources/runtimets/testsuite_it_python.xml | 2 ++ .../asterix/metadata/declared/DataSource.java | 24 ++++++++++++++ .../metadata/declared/DatasetDataSource.java | 5 --- .../metadata/declared/FunctionDataSource.java | 16 +++++++++ .../core/algebra/metadata/IDataSource.java | 4 +-- .../visitors/IsomorphismOperatorVisitor.java | 17 +++++----- 25 files changed, 349 insertions(+), 18 deletions(-) diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/ActiveRequestsDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/ActiveRequestsDatasource.java index 5239d0146f..acb2ca5a8d 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/ActiveRequestsDatasource.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/ActiveRequestsDatasource.java @@ -18,6 +18,8 @@ */ package org.apache.asterix.app.function; +import java.util.Objects; + import org.apache.asterix.app.message.ClientRequestsRequest; import org.apache.asterix.metadata.api.IDatasourceFunction; import org.apache.asterix.metadata.declared.DataSourceId; @@ -48,4 +50,9 @@ public class ActiveRequestsDatasource extends FunctionDataSource { public boolean skipJobCapacityAssignment() { return true; } + + @Override + protected boolean sameFunctionDatasource(FunctionDataSource other) { + return Objects.equals(this.functionId, other.getFunctionId()); + } } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/CompletedRequestsDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/CompletedRequestsDatasource.java index f02af21955..4d48eef01c 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/CompletedRequestsDatasource.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/CompletedRequestsDatasource.java @@ -18,6 +18,8 @@ */ package org.apache.asterix.app.function; +import java.util.Objects; + import org.apache.asterix.app.message.ClientRequestsRequest; import org.apache.asterix.metadata.api.IDatasourceFunction; import org.apache.asterix.metadata.declared.DataSourceId; @@ -48,4 +50,9 @@ public class CompletedRequestsDatasource extends FunctionDataSource { public boolean skipJobCapacityAssignment() { return true; } + + @Override + protected boolean sameFunctionDatasource(FunctionDataSource other) { + return Objects.equals(this.functionId, other.getFunctionId()); + } } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DatasetResourcesDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DatasetResourcesDatasource.java index 3e5033b7a8..d1c3c01e07 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DatasetResourcesDatasource.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DatasetResourcesDatasource.java @@ -18,6 +18,8 @@ */ package org.apache.asterix.app.function; +import java.util.Objects; + import org.apache.asterix.metadata.api.IDatasourceFunction; import org.apache.asterix.metadata.declared.DataSourceId; import org.apache.asterix.metadata.declared.FunctionDataSource; @@ -43,4 +45,17 @@ public class DatasetResourcesDatasource extends FunctionDataSource { AlgebricksAbsolutePartitionConstraint locations) { return new DatasetResourcesFunction(locations, datasetId); } + + public int getDatasetId() { + return datasetId; + } + + @Override + public boolean sameFunctionDatasource(FunctionDataSource other) { + if (!Objects.equals(this.functionId, other.getFunctionId())) { + return false; + } + DatasetResourcesDatasource that = (DatasetResourcesDatasource) other; + return Objects.equals(this.datasetId, that.getDatasetId()); + } } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java index 691be4706c..e5d78a4721 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java @@ -18,6 +18,8 @@ */ package org.apache.asterix.app.function; +import java.util.Objects; + import org.apache.asterix.common.cluster.IClusterStateManager; import org.apache.asterix.external.api.IDataParserFactory; import org.apache.asterix.external.parser.factory.JSONDataParserFactory; @@ -25,6 +27,7 @@ import org.apache.asterix.metadata.api.IDatasourceFunction; import org.apache.asterix.metadata.declared.DataSourceId; import org.apache.asterix.metadata.declared.FunctionDataSource; import org.apache.asterix.metadata.declared.MetadataProvider; +import org.apache.asterix.metadata.entities.Index; import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.algebricks.core.algebra.properties.INodeDomain; @@ -40,15 +43,21 @@ public class DumpIndexDatasource extends FunctionDataSource { private final RecordDescriptor recDesc; private final IBinaryComparatorFactory[] comparatorFactories; private final AlgebricksAbsolutePartitionConstraint storageLocations; + private final Index index; public DumpIndexDatasource(INodeDomain domain, IndexDataflowHelperFactory indexDataflowHelperFactory, RecordDescriptor recDesc, IBinaryComparatorFactory[] comparatorFactories, - AlgebricksAbsolutePartitionConstraint storageLocations) throws AlgebricksException { + AlgebricksAbsolutePartitionConstraint storageLocations, Index index) throws AlgebricksException { super(DUMP_INDEX_DATASOURCE_ID, DumpIndexRewriter.DUMP_INDEX, domain); this.indexDataflowHelperFactory = indexDataflowHelperFactory; this.recDesc = recDesc; this.comparatorFactories = comparatorFactories; this.storageLocations = storageLocations; + this.index = index; + } + + public Index getIndex() { + return index; } @Override @@ -66,4 +75,13 @@ public class DumpIndexDatasource extends FunctionDataSource { protected IDataParserFactory createDataParserFactory() { return new JSONDataParserFactory(); } + + @Override + public boolean sameFunctionDatasource(FunctionDataSource other) { + if (!Objects.equals(this.functionId, other.getFunctionId())) { + return false; + } + DumpIndexDatasource that = (DumpIndexDatasource) other; + return Objects.equals(this.index, that.getIndex()); + } } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java index 6c0382d52b..b5ea73fac9 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java @@ -73,6 +73,6 @@ public class DumpIndexRewriter extends FunctionRewriter { (AlgebricksAbsolutePartitionConstraint) secondaryIndexHelper.getSecondaryPartitionConstraint(); return new DumpIndexDatasource(context.getComputationNodeDomain(), indexDataflowHelperFactory, secondaryIndexHelper.getSecondaryRecDesc(), secondaryIndexHelper.getSecondaryComparatorFactories(), - secondaryPartitionConstraint); + secondaryPartitionConstraint, index); } } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/JobSummariesDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/JobSummariesDatasource.java index 6bab0cd609..1a23db3690 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/JobSummariesDatasource.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/JobSummariesDatasource.java @@ -18,6 +18,8 @@ */ package org.apache.asterix.app.function; +import java.util.Objects; + import org.apache.asterix.metadata.api.IDatasourceFunction; import org.apache.asterix.metadata.declared.DataSourceId; import org.apache.asterix.metadata.declared.FunctionDataSource; @@ -40,4 +42,9 @@ public class JobSummariesDatasource extends FunctionDataSource { AlgebricksAbsolutePartitionConstraint locations) { return new JobSummariesFunction(AlgebricksAbsolutePartitionConstraint.randomLocation(locations.getLocations())); } + + @Override + protected boolean sameFunctionDatasource(FunctionDataSource other) { + return Objects.equals(this.functionId, other.getFunctionId()); + } } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/PingDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/PingDatasource.java index 1b6d807c2c..6eabda9439 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/PingDatasource.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/PingDatasource.java @@ -18,6 +18,8 @@ */ package org.apache.asterix.app.function; +import java.util.Objects; + import org.apache.asterix.metadata.api.IDatasourceFunction; import org.apache.asterix.metadata.declared.DataSourceId; import org.apache.asterix.metadata.declared.FunctionDataSource; @@ -40,4 +42,8 @@ public class PingDatasource extends FunctionDataSource { return new PingFunction(locations); } + @Override + protected boolean sameFunctionDatasource(FunctionDataSource other) { + return Objects.equals(this.functionId, other.getFunctionId()); + } } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryIndexDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryIndexDatasource.java index cf2b891357..d7c0e6db18 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryIndexDatasource.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryIndexDatasource.java @@ -20,6 +20,7 @@ package org.apache.asterix.app.function; import java.util.ArrayList; import java.util.List; +import java.util.Objects; import org.apache.asterix.common.cluster.IClusterStateManager; import org.apache.asterix.metadata.api.IDatasourceFunction; @@ -68,6 +69,14 @@ public class QueryIndexDatasource extends FunctionDataSource { this.numSecKeys = numSecKeys; } + public Dataset getDataset() { + return ds; + } + + public String getIndexName() { + return indexName; + } + @Override protected void initSchemaType(IAType iType) { ARecordType type = (ARecordType) iType; @@ -126,4 +135,13 @@ public class QueryIndexDatasource extends FunctionDataSource { return new DataSourceId(dataset.getDataverseName(), dataset.getDatasetName(), new String[] { indexName, QueryIndexRewriter.QUERY_INDEX.getName() }); } + + @Override + public boolean sameFunctionDatasource(FunctionDataSource other) { + if (!Objects.equals(this.functionId, other.getFunctionId())) { + return false; + } + QueryIndexDatasource that = (QueryIndexDatasource) other; + return Objects.equals(this.ds, that.getDataset()) && Objects.equals(this.indexName, that.getIndexName()); + } } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/StorageComponentsDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/StorageComponentsDatasource.java index 6157412ad3..c470a20110 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/StorageComponentsDatasource.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/StorageComponentsDatasource.java @@ -18,6 +18,8 @@ */ package org.apache.asterix.app.function; +import java.util.Objects; + import org.apache.asterix.metadata.api.IDatasourceFunction; import org.apache.asterix.metadata.declared.DataSourceId; import org.apache.asterix.metadata.declared.FunctionDataSource; @@ -38,9 +40,22 @@ public class StorageComponentsDatasource extends FunctionDataSource { this.datasetId = datasetId; } + public int getDatasetId() { + return datasetId; + } + @Override protected IDatasourceFunction createFunction(MetadataProvider metadataProvider, AlgebricksAbsolutePartitionConstraint locations) { return new StorageComponentsFunction(locations, datasetId); } + + @Override + public boolean sameFunctionDatasource(FunctionDataSource other) { + if (!Objects.equals(this.functionId, other.getFunctionId())) { + return false; + } + StorageComponentsDatasource that = (StorageComponentsDatasource) other; + return Objects.equals(this.datasetId, that.getDatasetId()); + } } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSAllTablesDataGeneratorDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSAllTablesDataGeneratorDatasource.java index 430ba36106..489b052795 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSAllTablesDataGeneratorDatasource.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSAllTablesDataGeneratorDatasource.java @@ -18,6 +18,8 @@ */ package org.apache.asterix.app.function; +import java.util.Objects; + import org.apache.asterix.common.cluster.IClusterStateManager; import org.apache.asterix.metadata.api.IDatasourceFunction; import org.apache.asterix.metadata.declared.DataSourceId; @@ -42,6 +44,10 @@ public class TPCDSAllTablesDataGeneratorDatasource extends FunctionDataSource { this.scalingFactor = scalingFactor; } + public double getScalingFactor() { + return scalingFactor; + } + /** * This ensures that each function will have a unique DataSourceId by passing the table name as part of the * DataSourceId. This eliminates the issue of creating a single function even though multiple functions calls @@ -66,4 +72,13 @@ public class TPCDSAllTablesDataGeneratorDatasource extends FunctionDataSource { protected AlgebricksAbsolutePartitionConstraint getLocations(IClusterStateManager csm) { return csm.getSortedClusterLocations(); } + + @Override + public boolean sameFunctionDatasource(FunctionDataSource other) { + if (!Objects.equals(this.functionId, other.getFunctionId())) { + return false; + } + TPCDSAllTablesDataGeneratorDatasource that = (TPCDSAllTablesDataGeneratorDatasource) other; + return Objects.equals(this.scalingFactor, that.getScalingFactor()); + } } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSSingleTableDataGeneratorDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSSingleTableDataGeneratorDatasource.java index 67319e332a..c90dff173a 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSSingleTableDataGeneratorDatasource.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSSingleTableDataGeneratorDatasource.java @@ -18,6 +18,8 @@ */ package org.apache.asterix.app.function; +import java.util.Objects; + import org.apache.asterix.common.cluster.IClusterStateManager; import org.apache.asterix.metadata.api.IDatasourceFunction; import org.apache.asterix.metadata.declared.DataSourceId; @@ -44,6 +46,14 @@ public class TPCDSSingleTableDataGeneratorDatasource extends FunctionDataSource this.scalingFactor = scalingFactor; } + public String getTableName() { + return tableName; + } + + public double getScalingFactor() { + return scalingFactor; + } + /** * This ensures that each function will have a unique DataSourceId by passing the table name as part of the * DataSourceId. This eliminates the issue of creating a single function even though multiple functions calls @@ -70,4 +80,14 @@ public class TPCDSSingleTableDataGeneratorDatasource extends FunctionDataSource protected AlgebricksAbsolutePartitionConstraint getLocations(IClusterStateManager csm) { return csm.getSortedClusterLocations(); } + + @Override + public boolean sameFunctionDatasource(FunctionDataSource other) { + if (!Objects.equals(this.functionId, other.getFunctionId())) { + return false; + } + TPCDSSingleTableDataGeneratorDatasource that = (TPCDSSingleTableDataGeneratorDatasource) other; + return Objects.equals(this.tableName, that.getTableName()) + && Objects.equals(this.scalingFactor, that.getScalingFactor()); + } } diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.000.ddl.sqlpp new file mode 100644 index 0000000000..737c344605 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.000.ddl.sqlpp @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +DROP DATAVERSE test IF EXISTS; +CREATE DATAVERSE test; +USE test; + +DROP TYPE test IF EXISTS; +CREATE TYPE test AS OPEN {id: int}; + +DROP DATASET test IF EXISTS; +CREATE DATASET test(test) PRIMARY KEY id; + +DROP INDEX test.test_int_idx IF EXISTS; +CREATE INDEX test_int_idx ON test(intField: int); + +DROP INDEX test.test_string_idx IF EXISTS; +CREATE INDEX test_string_idx ON test(stringField: string); diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.010.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.010.update.sqlpp new file mode 100644 index 0000000000..aeadafc06d --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.010.update.sqlpp @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +USE test; +INSERT INTO test({"id": 1, "intField": 15, "stringField": "foo"}); diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.020.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.020.query.sqlpp new file mode 100644 index 0000000000..9752c75cb0 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.020.query.sqlpp @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +SET `import-private-functions` `true`; + +SELECT dump_index("test", "test", "test_int_idx") as intIndex +UNION ALL +SELECT dump_index("test", "test", "test_string_idx") as stringIndex; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.030.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.030.query.sqlpp new file mode 100644 index 0000000000..b4209fc814 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.030.query.sqlpp @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +SET `import-private-functions` `true`; + +explain +SELECT dump_index("test", "test", "test_int_idx") as intIndex +UNION ALL +SELECT dump_index("test", "test", "test_string_idx") as stringIndex; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.999.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.999.ddl.sqlpp new file mode 100644 index 0000000000..20dc6fde5e --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.999.ddl.sqlpp @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +DROP DATAVERSE test IF EXISTS; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/explain/explain_same_datasource_function_different_arguments/test.020.unorderedtxt b/asterixdb/asterix-app/src/test/resources/runtimets/results/explain/explain_same_datasource_function_different_arguments/test.020.unorderedtxt new file mode 100644 index 0000000000..24e0e67981 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/explain/explain_same_datasource_function_different_arguments/test.020.unorderedtxt @@ -0,0 +1,2 @@ +{ "intIndex": [ { "values": [ 15, 1 ] } ] } +{ "stringIndex": [ { "values": [ "foo", 1 ] } ] } diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/explain/explain_same_datasource_function_different_arguments/test.030.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results/explain/explain_same_datasource_function_different_arguments/test.030.plan new file mode 100644 index 0000000000..825f43a131 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/explain/explain_same_datasource_function_different_arguments/test.030.plan @@ -0,0 +1,38 @@ +distribute result [$$11] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] +-- DISTRIBUTE_RESULT |UNPARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED| + union ($$16, $$17, $$11) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- UNION_ALL |UNPARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED| + project ([$$16]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- STREAM_PROJECT |UNPARTITIONED| + assign [$$16] <- [cast({"intIndex": $$13})] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ASSIGN |UNPARTITIONED| + aggregate [$$13] <- [listify($$12)] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- AGGREGATE |UNPARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- RANDOM_MERGE_EXCHANGE |PARTITIONED| + data-scan []<-[$$12] <- asterix.dump-index. [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- DATASOURCE_SCAN |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- EMPTY_TUPLE_SOURCE |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED| + project ([$$17]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- STREAM_PROJECT |UNPARTITIONED| + assign [$$17] <- [cast({"stringIndex": $$15})] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ASSIGN |UNPARTITIONED| + aggregate [$$15] <- [listify($$14)] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- AGGREGATE |UNPARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- RANDOM_MERGE_EXCHANGE |PARTITIONED| + data-scan []<-[$$14] <- asterix.dump-index. [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- DATASOURCE_SCAN |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- EMPTY_TUPLE_SOURCE |PARTITIONED| \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml b/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml index 09b9e2360e..b0c4c44adf 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml @@ -337,6 +337,11 @@ <expected-error>ASX1001: Syntax error: EXPLAIN is not supported for this kind of statement</expected-error> </compilation-unit> </test-case> + <test-case FilePath="explain"> + <compilation-unit name="explain_same_datasource_function_different_arguments"> + <output-dir compare="Text">explain_same_datasource_function_different_arguments</output-dir> + </compilation-unit> + </test-case> </test-group> <test-group name="aggregate"> <test-case FilePath="aggregate"> diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_python.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_python.xml index 686ede22e9..50795b7f07 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_python.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_python.xml @@ -49,6 +49,7 @@ <output-dir compare="Clean-JSON">python_open_type_validation</output-dir> </compilation-unit> </test-case> + <!-- TODO(Ian): disabling this test as it is intermittently failing as the output is dependent on the python version running on the test machine <test-case FilePath="external-library" check-warnings="true"> <compilation-unit name="py_function_error"> <output-dir compare="Clean-JSON">py_function_error</output-dir> @@ -63,6 +64,7 @@ ArithmeticError: oof (in line 28, at column 1)</expected-warn> </compilation-unit> </test-case> + --> <test-case FilePath="external-library"> <compilation-unit name="mysentiment_twitter"> <output-dir compare="Text">mysentiment_twitter</output-dir> diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DataSource.java index 95b090658d..36273d2d3e 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DataSource.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DataSource.java @@ -24,6 +24,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import org.apache.asterix.om.types.IAType; import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint; @@ -166,4 +167,27 @@ public abstract class DataSource implements IDataSource<DataSourceId> { ITupleFilterFactory tupleFilterFactory, long outputLimit, IOperatorSchema opSchema, IVariableTypeEnvironment typeEnv, JobGenContext context, JobSpecification jobSpec, Object implConfig, IProjectionInfo<?> projectionInfo) throws AlgebricksException; + + @Override + public boolean sameAs(IDataSource<?> other) { + if (this == other) { + return true; + } + + if (!(other instanceof DataSource)) { + return false; + } + + DataSource that = (DataSource) other; + if (!Objects.equals(this.id, that.getId()) || !Objects.equals(this.datasourceType, that.getDatasourceType())) { + return false; + } + + if (this.datasourceType == Type.EXTERNAL_DATASET && that.getDatasourceType() == Type.EXTERNAL_DATASET + && !Objects.equals(this.getProperties(), other.getProperties())) { + return false; + } + + return true; + } } diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java index 66ea5a7313..6b42ff2b53 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java @@ -198,9 +198,4 @@ public class DatasetDataSource extends DataSource { public boolean isScanAccessPathALeaf() { return dataset.getDatasetType() == DatasetType.EXTERNAL; } - - @Override - public boolean compareProperties() { - return dataset.getDatasetType() == DatasetType.EXTERNAL; - } } diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java index 3b6bd9d55a..bce22c1acd 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java @@ -126,4 +126,20 @@ public abstract class FunctionDataSource extends DataSource { protected static DataSourceId createDataSourceId(FunctionIdentifier fid, String... parameters) { return new DataSourceId(FunctionSignature.getDataverseName(fid), fid.getName(), parameters); } + + protected abstract boolean sameFunctionDatasource(FunctionDataSource other); + + @Override + public boolean sameAs(IDataSource<?> other) { + if (!super.sameAs(other)) { + return false; + } + + if (!(other instanceof FunctionDataSource)) { + return false; + } + + FunctionDataSource that = (FunctionDataSource) other; + return sameFunctionDatasource(that); + } } diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IDataSource.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IDataSource.java index 5b75cd9c22..f7cd6da634 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IDataSource.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IDataSource.java @@ -48,7 +48,5 @@ public interface IDataSource<T> { public Map<String, Serializable> getProperties(); - default boolean compareProperties() { - return false; - } + boolean sameAs(IDataSource<?> other); } diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java index 9e2e87c197..d81550c370 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java @@ -469,20 +469,21 @@ public class IsomorphismOperatorVisitor implements ILogicalOperatorVisitor<Boole if (aop.getOperatorTag() != LogicalOperatorTag.DATASOURCESCAN) { return Boolean.FALSE; } - DataSourceScanOperator argScan = (DataSourceScanOperator) arg; - boolean isomorphic = op.getDataSource().getId().equals(argScan.getDataSource().getId()) - && op.getOutputLimit() == argScan.getOutputLimit() - && Objects.equals(op.getProjectionInfo(), argScan.getProjectionInfo()); + DataSourceScanOperator argScan = (DataSourceScanOperator) arg; + IDataSource<?> dataSource = op.getDataSource(); + IDataSource<?> argDataSource = argScan.getDataSource(); + boolean isomorphic = dataSource.sameAs(argDataSource); if (!isomorphic) { return Boolean.FALSE; } - IDataSource<?> dataSource = op.getDataSource(); - IDataSource<?> argDataSource = argScan.getDataSource(); - if (dataSource.compareProperties() && argDataSource.compareProperties() - && !Objects.equals(dataSource.getProperties(), argDataSource.getProperties())) { + + isomorphic = op.getOutputLimit() == argScan.getOutputLimit() + && Objects.equals(op.getProjectionInfo(), argScan.getProjectionInfo()); + if (!isomorphic) { return Boolean.FALSE; } + DataSourceScanOperator scanOpArg = (DataSourceScanOperator) copyAndSubstituteVar(op, arg); ILogicalExpression opCondition = op.getSelectCondition() != null ? op.getSelectCondition().getValue() : null; ILogicalExpression argCondition =
