>From Hussain Towaileb <[email protected]>: Hussain Towaileb has submitted this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19005 )
Change subject: [ASTERIXDB-3521][FUN]: Verify datasource function args before replicating ...................................................................... [ASTERIXDB-3521][FUN]: Verify datasource function args before replicating Ext-ref: MB-63994 Change-Id: I94f86e0f1e949e94a161d72be5531233192dd0c8 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19005 Tested-by: Jenkins <[email protected]> Reviewed-by: Hussain Towaileb <[email protected]> Reviewed-by: Ali Alsuliman <[email protected]> --- M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSSingleTableDataGeneratorDatasource.java M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/JobSummariesDatasource.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSAllTablesDataGeneratorDatasource.java A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.010.update.sqlpp M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/StorageComponentsDatasource.java A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.030.query.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.999.ddl.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.020.query.sqlpp A asterixdb/asterix-app/src/test/resources/runtimets/results/explain/explain_same_datasource_function_different_arguments/test.030.plan M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java A asterixdb/asterix-app/src/test/resources/runtimets/results/explain/explain_same_datasource_function_different_arguments/test.020.unorderedtxt M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DataSource.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/CompletedRequestsDatasource.java A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.000.ddl.sqlpp M asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java M hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IDataSource.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/PingDatasource.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DatasetResourcesDatasource.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/ActiveRequestsDatasource.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryIndexDatasource.java M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_python.xml 25 files changed, 363 insertions(+), 18 deletions(-) Approvals: Ali Alsuliman: Looks good to me, approved Hussain Towaileb: Looks good to me, but someone else must approve Jenkins: Verified diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/ActiveRequestsDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/ActiveRequestsDatasource.java index 5239d01..acb2ca5 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/ActiveRequestsDatasource.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/ActiveRequestsDatasource.java @@ -18,6 +18,8 @@ */ package org.apache.asterix.app.function; +import java.util.Objects; + import org.apache.asterix.app.message.ClientRequestsRequest; import org.apache.asterix.metadata.api.IDatasourceFunction; import org.apache.asterix.metadata.declared.DataSourceId; @@ -48,4 +50,9 @@ public boolean skipJobCapacityAssignment() { return true; } + + @Override + protected boolean sameFunctionDatasource(FunctionDataSource other) { + return Objects.equals(this.functionId, other.getFunctionId()); + } } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/CompletedRequestsDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/CompletedRequestsDatasource.java index f02af21..4d48eef 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/CompletedRequestsDatasource.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/CompletedRequestsDatasource.java @@ -18,6 +18,8 @@ */ package org.apache.asterix.app.function; +import java.util.Objects; + import org.apache.asterix.app.message.ClientRequestsRequest; import org.apache.asterix.metadata.api.IDatasourceFunction; import org.apache.asterix.metadata.declared.DataSourceId; @@ -48,4 +50,9 @@ public boolean skipJobCapacityAssignment() { return true; } + + @Override + protected boolean sameFunctionDatasource(FunctionDataSource other) { + return Objects.equals(this.functionId, other.getFunctionId()); + } } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DatasetResourcesDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DatasetResourcesDatasource.java index 3e5033b..d1c3c01 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DatasetResourcesDatasource.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DatasetResourcesDatasource.java @@ -18,6 +18,8 @@ */ package org.apache.asterix.app.function; +import java.util.Objects; + import org.apache.asterix.metadata.api.IDatasourceFunction; import org.apache.asterix.metadata.declared.DataSourceId; import org.apache.asterix.metadata.declared.FunctionDataSource; @@ -43,4 +45,17 @@ AlgebricksAbsolutePartitionConstraint locations) { return new DatasetResourcesFunction(locations, datasetId); } + + public int getDatasetId() { + return datasetId; + } + + @Override + public boolean sameFunctionDatasource(FunctionDataSource other) { + if (!Objects.equals(this.functionId, other.getFunctionId())) { + return false; + } + DatasetResourcesDatasource that = (DatasetResourcesDatasource) other; + return Objects.equals(this.datasetId, that.getDatasetId()); + } } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java index 691be47..e5d78a4 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java @@ -18,6 +18,8 @@ */ package org.apache.asterix.app.function; +import java.util.Objects; + import org.apache.asterix.common.cluster.IClusterStateManager; import org.apache.asterix.external.api.IDataParserFactory; import org.apache.asterix.external.parser.factory.JSONDataParserFactory; @@ -25,6 +27,7 @@ import org.apache.asterix.metadata.declared.DataSourceId; import org.apache.asterix.metadata.declared.FunctionDataSource; import org.apache.asterix.metadata.declared.MetadataProvider; +import org.apache.asterix.metadata.entities.Index; import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.algebricks.core.algebra.properties.INodeDomain; @@ -40,15 +43,21 @@ private final RecordDescriptor recDesc; private final IBinaryComparatorFactory[] comparatorFactories; private final AlgebricksAbsolutePartitionConstraint storageLocations; + private final Index index; public DumpIndexDatasource(INodeDomain domain, IndexDataflowHelperFactory indexDataflowHelperFactory, RecordDescriptor recDesc, IBinaryComparatorFactory[] comparatorFactories, - AlgebricksAbsolutePartitionConstraint storageLocations) throws AlgebricksException { + AlgebricksAbsolutePartitionConstraint storageLocations, Index index) throws AlgebricksException { super(DUMP_INDEX_DATASOURCE_ID, DumpIndexRewriter.DUMP_INDEX, domain); this.indexDataflowHelperFactory = indexDataflowHelperFactory; this.recDesc = recDesc; this.comparatorFactories = comparatorFactories; this.storageLocations = storageLocations; + this.index = index; + } + + public Index getIndex() { + return index; } @Override @@ -66,4 +75,13 @@ protected IDataParserFactory createDataParserFactory() { return new JSONDataParserFactory(); } + + @Override + public boolean sameFunctionDatasource(FunctionDataSource other) { + if (!Objects.equals(this.functionId, other.getFunctionId())) { + return false; + } + DumpIndexDatasource that = (DumpIndexDatasource) other; + return Objects.equals(this.index, that.getIndex()); + } } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java index 6c0382d..b5ea73f 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java @@ -73,6 +73,6 @@ (AlgebricksAbsolutePartitionConstraint) secondaryIndexHelper.getSecondaryPartitionConstraint(); return new DumpIndexDatasource(context.getComputationNodeDomain(), indexDataflowHelperFactory, secondaryIndexHelper.getSecondaryRecDesc(), secondaryIndexHelper.getSecondaryComparatorFactories(), - secondaryPartitionConstraint); + secondaryPartitionConstraint, index); } } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/JobSummariesDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/JobSummariesDatasource.java index 6bab0cd..1a23db3 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/JobSummariesDatasource.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/JobSummariesDatasource.java @@ -18,6 +18,8 @@ */ package org.apache.asterix.app.function; +import java.util.Objects; + import org.apache.asterix.metadata.api.IDatasourceFunction; import org.apache.asterix.metadata.declared.DataSourceId; import org.apache.asterix.metadata.declared.FunctionDataSource; @@ -40,4 +42,9 @@ AlgebricksAbsolutePartitionConstraint locations) { return new JobSummariesFunction(AlgebricksAbsolutePartitionConstraint.randomLocation(locations.getLocations())); } + + @Override + protected boolean sameFunctionDatasource(FunctionDataSource other) { + return Objects.equals(this.functionId, other.getFunctionId()); + } } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/PingDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/PingDatasource.java index 1b6d807..6eabda9 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/PingDatasource.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/PingDatasource.java @@ -18,6 +18,8 @@ */ package org.apache.asterix.app.function; +import java.util.Objects; + import org.apache.asterix.metadata.api.IDatasourceFunction; import org.apache.asterix.metadata.declared.DataSourceId; import org.apache.asterix.metadata.declared.FunctionDataSource; @@ -40,4 +42,8 @@ return new PingFunction(locations); } + @Override + protected boolean sameFunctionDatasource(FunctionDataSource other) { + return Objects.equals(this.functionId, other.getFunctionId()); + } } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryIndexDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryIndexDatasource.java index cf2b891..d7c0e6d 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryIndexDatasource.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryIndexDatasource.java @@ -20,6 +20,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Objects; import org.apache.asterix.common.cluster.IClusterStateManager; import org.apache.asterix.metadata.api.IDatasourceFunction; @@ -68,6 +69,14 @@ this.numSecKeys = numSecKeys; } + public Dataset getDataset() { + return ds; + } + + public String getIndexName() { + return indexName; + } + @Override protected void initSchemaType(IAType iType) { ARecordType type = (ARecordType) iType; @@ -126,4 +135,13 @@ return new DataSourceId(dataset.getDataverseName(), dataset.getDatasetName(), new String[] { indexName, QueryIndexRewriter.QUERY_INDEX.getName() }); } + + @Override + public boolean sameFunctionDatasource(FunctionDataSource other) { + if (!Objects.equals(this.functionId, other.getFunctionId())) { + return false; + } + QueryIndexDatasource that = (QueryIndexDatasource) other; + return Objects.equals(this.ds, that.getDataset()) && Objects.equals(this.indexName, that.getIndexName()); + } } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/StorageComponentsDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/StorageComponentsDatasource.java index 6157412..c470a20 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/StorageComponentsDatasource.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/StorageComponentsDatasource.java @@ -18,6 +18,8 @@ */ package org.apache.asterix.app.function; +import java.util.Objects; + import org.apache.asterix.metadata.api.IDatasourceFunction; import org.apache.asterix.metadata.declared.DataSourceId; import org.apache.asterix.metadata.declared.FunctionDataSource; @@ -38,9 +40,22 @@ this.datasetId = datasetId; } + public int getDatasetId() { + return datasetId; + } + @Override protected IDatasourceFunction createFunction(MetadataProvider metadataProvider, AlgebricksAbsolutePartitionConstraint locations) { return new StorageComponentsFunction(locations, datasetId); } + + @Override + public boolean sameFunctionDatasource(FunctionDataSource other) { + if (!Objects.equals(this.functionId, other.getFunctionId())) { + return false; + } + StorageComponentsDatasource that = (StorageComponentsDatasource) other; + return Objects.equals(this.datasetId, that.getDatasetId()); + } } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSAllTablesDataGeneratorDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSAllTablesDataGeneratorDatasource.java index 430ba36..489b052 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSAllTablesDataGeneratorDatasource.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSAllTablesDataGeneratorDatasource.java @@ -18,6 +18,8 @@ */ package org.apache.asterix.app.function; +import java.util.Objects; + import org.apache.asterix.common.cluster.IClusterStateManager; import org.apache.asterix.metadata.api.IDatasourceFunction; import org.apache.asterix.metadata.declared.DataSourceId; @@ -42,6 +44,10 @@ this.scalingFactor = scalingFactor; } + public double getScalingFactor() { + return scalingFactor; + } + /** * This ensures that each function will have a unique DataSourceId by passing the table name as part of the * DataSourceId. This eliminates the issue of creating a single function even though multiple functions calls @@ -66,4 +72,13 @@ protected AlgebricksAbsolutePartitionConstraint getLocations(IClusterStateManager csm) { return csm.getSortedClusterLocations(); } + + @Override + public boolean sameFunctionDatasource(FunctionDataSource other) { + if (!Objects.equals(this.functionId, other.getFunctionId())) { + return false; + } + TPCDSAllTablesDataGeneratorDatasource that = (TPCDSAllTablesDataGeneratorDatasource) other; + return Objects.equals(this.scalingFactor, that.getScalingFactor()); + } } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSSingleTableDataGeneratorDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSSingleTableDataGeneratorDatasource.java index 67319e3..c90dff1 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSSingleTableDataGeneratorDatasource.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSSingleTableDataGeneratorDatasource.java @@ -18,6 +18,8 @@ */ package org.apache.asterix.app.function; +import java.util.Objects; + import org.apache.asterix.common.cluster.IClusterStateManager; import org.apache.asterix.metadata.api.IDatasourceFunction; import org.apache.asterix.metadata.declared.DataSourceId; @@ -44,6 +46,14 @@ this.scalingFactor = scalingFactor; } + public String getTableName() { + return tableName; + } + + public double getScalingFactor() { + return scalingFactor; + } + /** * This ensures that each function will have a unique DataSourceId by passing the table name as part of the * DataSourceId. This eliminates the issue of creating a single function even though multiple functions calls @@ -70,4 +80,14 @@ protected AlgebricksAbsolutePartitionConstraint getLocations(IClusterStateManager csm) { return csm.getSortedClusterLocations(); } + + @Override + public boolean sameFunctionDatasource(FunctionDataSource other) { + if (!Objects.equals(this.functionId, other.getFunctionId())) { + return false; + } + TPCDSSingleTableDataGeneratorDatasource that = (TPCDSSingleTableDataGeneratorDatasource) other; + return Objects.equals(this.tableName, that.getTableName()) + && Objects.equals(this.scalingFactor, that.getScalingFactor()); + } } diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.000.ddl.sqlpp new file mode 100644 index 0000000..737c344 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.000.ddl.sqlpp @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +DROP DATAVERSE test IF EXISTS; +CREATE DATAVERSE test; +USE test; + +DROP TYPE test IF EXISTS; +CREATE TYPE test AS OPEN {id: int}; + +DROP DATASET test IF EXISTS; +CREATE DATASET test(test) PRIMARY KEY id; + +DROP INDEX test.test_int_idx IF EXISTS; +CREATE INDEX test_int_idx ON test(intField: int); + +DROP INDEX test.test_string_idx IF EXISTS; +CREATE INDEX test_string_idx ON test(stringField: string); diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.010.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.010.update.sqlpp new file mode 100644 index 0000000..aeadafc --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.010.update.sqlpp @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +USE test; +INSERT INTO test({"id": 1, "intField": 15, "stringField": "foo"}); diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.020.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.020.query.sqlpp new file mode 100644 index 0000000..9752c75 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.020.query.sqlpp @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +SET `import-private-functions` `true`; + +SELECT dump_index("test", "test", "test_int_idx") as intIndex +UNION ALL +SELECT dump_index("test", "test", "test_string_idx") as stringIndex; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.030.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.030.query.sqlpp new file mode 100644 index 0000000..b4209fc --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.030.query.sqlpp @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +SET `import-private-functions` `true`; + +explain +SELECT dump_index("test", "test", "test_int_idx") as intIndex +UNION ALL +SELECT dump_index("test", "test", "test_string_idx") as stringIndex; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.999.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.999.ddl.sqlpp new file mode 100644 index 0000000..20dc6fd --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.999.ddl.sqlpp @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +DROP DATAVERSE test IF EXISTS; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/explain/explain_same_datasource_function_different_arguments/test.020.unorderedtxt b/asterixdb/asterix-app/src/test/resources/runtimets/results/explain/explain_same_datasource_function_different_arguments/test.020.unorderedtxt new file mode 100644 index 0000000..24e0e67 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/explain/explain_same_datasource_function_different_arguments/test.020.unorderedtxt @@ -0,0 +1,2 @@ +{ "intIndex": [ { "values": [ 15, 1 ] } ] } +{ "stringIndex": [ { "values": [ "foo", 1 ] } ] } diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/explain/explain_same_datasource_function_different_arguments/test.030.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results/explain/explain_same_datasource_function_different_arguments/test.030.plan new file mode 100644 index 0000000..825f43a --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/explain/explain_same_datasource_function_different_arguments/test.030.plan @@ -0,0 +1,38 @@ +distribute result [$$11] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] +-- DISTRIBUTE_RESULT |UNPARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED| + union ($$16, $$17, $$11) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- UNION_ALL |UNPARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED| + project ([$$16]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- STREAM_PROJECT |UNPARTITIONED| + assign [$$16] <- [cast({"intIndex": $$13})] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ASSIGN |UNPARTITIONED| + aggregate [$$13] <- [listify($$12)] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- AGGREGATE |UNPARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- RANDOM_MERGE_EXCHANGE |PARTITIONED| + data-scan []<-[$$12] <- asterix.dump-index. [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- DATASOURCE_SCAN |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- EMPTY_TUPLE_SOURCE |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED| + project ([$$17]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- STREAM_PROJECT |UNPARTITIONED| + assign [$$17] <- [cast({"stringIndex": $$15})] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ASSIGN |UNPARTITIONED| + aggregate [$$15] <- [listify($$14)] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- AGGREGATE |UNPARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- RANDOM_MERGE_EXCHANGE |PARTITIONED| + data-scan []<-[$$14] <- asterix.dump-index. [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- DATASOURCE_SCAN |PARTITIONED| + exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- ONE_TO_ONE_EXCHANGE |PARTITIONED| + empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0] + -- EMPTY_TUPLE_SOURCE |PARTITIONED| \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml b/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml index 09b9e23..b0c4c44 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml @@ -337,6 +337,11 @@ <expected-error>ASX1001: Syntax error: EXPLAIN is not supported for this kind of statement</expected-error> </compilation-unit> </test-case> + <test-case FilePath="explain"> + <compilation-unit name="explain_same_datasource_function_different_arguments"> + <output-dir compare="Text">explain_same_datasource_function_different_arguments</output-dir> + </compilation-unit> + </test-case> </test-group> <test-group name="aggregate"> <test-case FilePath="aggregate"> diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_python.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_python.xml index 686ede2..50795b7 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_python.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_python.xml @@ -49,6 +49,7 @@ <output-dir compare="Clean-JSON">python_open_type_validation</output-dir> </compilation-unit> </test-case> + <!-- TODO(Ian): disabling this test as it is intermittently failing as the output is dependent on the python version running on the test machine <test-case FilePath="external-library" check-warnings="true"> <compilation-unit name="py_function_error"> <output-dir compare="Clean-JSON">py_function_error</output-dir> @@ -63,6 +64,7 @@ (in line 28, at column 1)</expected-warn> </compilation-unit> </test-case> + --> <test-case FilePath="external-library"> <compilation-unit name="mysentiment_twitter"> <output-dir compare="Text">mysentiment_twitter</output-dir> diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DataSource.java index 95b0906..36273d2 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DataSource.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DataSource.java @@ -24,6 +24,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import org.apache.asterix.om.types.IAType; import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint; @@ -166,4 +167,27 @@ ITupleFilterFactory tupleFilterFactory, long outputLimit, IOperatorSchema opSchema, IVariableTypeEnvironment typeEnv, JobGenContext context, JobSpecification jobSpec, Object implConfig, IProjectionInfo<?> projectionInfo) throws AlgebricksException; + + @Override + public boolean sameAs(IDataSource<?> other) { + if (this == other) { + return true; + } + + if (!(other instanceof DataSource)) { + return false; + } + + DataSource that = (DataSource) other; + if (!Objects.equals(this.id, that.getId()) || !Objects.equals(this.datasourceType, that.getDatasourceType())) { + return false; + } + + if (this.datasourceType == Type.EXTERNAL_DATASET && that.getDatasourceType() == Type.EXTERNAL_DATASET + && !Objects.equals(this.getProperties(), other.getProperties())) { + return false; + } + + return true; + } } diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java index 66ea5a7..6b42ff2 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java @@ -198,9 +198,4 @@ public boolean isScanAccessPathALeaf() { return dataset.getDatasetType() == DatasetType.EXTERNAL; } - - @Override - public boolean compareProperties() { - return dataset.getDatasetType() == DatasetType.EXTERNAL; - } } diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java index 3b6bd9d..bce22c1 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java @@ -126,4 +126,20 @@ protected static DataSourceId createDataSourceId(FunctionIdentifier fid, String... parameters) { return new DataSourceId(FunctionSignature.getDataverseName(fid), fid.getName(), parameters); } + + protected abstract boolean sameFunctionDatasource(FunctionDataSource other); + + @Override + public boolean sameAs(IDataSource<?> other) { + if (!super.sameAs(other)) { + return false; + } + + if (!(other instanceof FunctionDataSource)) { + return false; + } + + FunctionDataSource that = (FunctionDataSource) other; + return sameFunctionDatasource(that); + } } diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IDataSource.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IDataSource.java index 5b75cd9..f7cd6da 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IDataSource.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IDataSource.java @@ -48,7 +48,5 @@ public Map<String, Serializable> getProperties(); - default boolean compareProperties() { - return false; - } + boolean sameAs(IDataSource<?> other); } diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java index 9e2e87c..d81550c 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java @@ -469,20 +469,21 @@ if (aop.getOperatorTag() != LogicalOperatorTag.DATASOURCESCAN) { return Boolean.FALSE; } - DataSourceScanOperator argScan = (DataSourceScanOperator) arg; - boolean isomorphic = op.getDataSource().getId().equals(argScan.getDataSource().getId()) - && op.getOutputLimit() == argScan.getOutputLimit() - && Objects.equals(op.getProjectionInfo(), argScan.getProjectionInfo()); + DataSourceScanOperator argScan = (DataSourceScanOperator) arg; + IDataSource<?> dataSource = op.getDataSource(); + IDataSource<?> argDataSource = argScan.getDataSource(); + boolean isomorphic = dataSource.sameAs(argDataSource); if (!isomorphic) { return Boolean.FALSE; } - IDataSource<?> dataSource = op.getDataSource(); - IDataSource<?> argDataSource = argScan.getDataSource(); - if (dataSource.compareProperties() && argDataSource.compareProperties() - && !Objects.equals(dataSource.getProperties(), argDataSource.getProperties())) { + + isomorphic = op.getOutputLimit() == argScan.getOutputLimit() + && Objects.equals(op.getProjectionInfo(), argScan.getProjectionInfo()); + if (!isomorphic) { return Boolean.FALSE; } + DataSourceScanOperator scanOpArg = (DataSourceScanOperator) copyAndSubstituteVar(op, arg); ILogicalExpression opCondition = op.getSelectCondition() != null ? op.getSelectCondition().getValue() : null; ILogicalExpression argCondition = -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19005 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: neo Gerrit-Change-Id: I94f86e0f1e949e94a161d72be5531233192dd0c8 Gerrit-Change-Number: 19005 Gerrit-PatchSet: 6 Gerrit-Owner: Hussain Towaileb <[email protected]> Gerrit-Reviewer: Ali Alsuliman <[email protected]> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Hussain Towaileb <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-MessageType: merged
