Repository: incubator-quickstep Updated Branches: refs/heads/master 39c6214a3 -> 8a84039e1
Get the list of referenced base relations - Find the base relations that are referenced in the query. - The referenced relations are stored in the QueryHandle. - Separate method in QueryProcessor class to just get this list of relations, without needing to fully optimize the query. Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/8a84039e Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/8a84039e Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/8a84039e Branch: refs/heads/master Commit: 8a84039e140ed80376505b6da950c08a6112470e Parents: 39c6214 Author: Harshad Deshmukh <hbdeshm...@apache.org> Authored: Tue Nov 28 16:16:59 2017 -0600 Committer: Harshad Deshmukh <hbdeshm...@apache.org> Committed: Fri Dec 1 14:26:19 2017 -0600 ---------------------------------------------------------------------- query_optimizer/CMakeLists.txt | 2 + query_optimizer/LogicalGenerator.cpp | 1 - query_optimizer/Optimizer.cpp | 11 +++ query_optimizer/Optimizer.hpp | 15 ++++ query_optimizer/QueryHandle.hpp | 21 +++++ query_optimizer/QueryProcessor.cpp | 7 ++ query_optimizer/QueryProcessor.hpp | 11 +++ query_optimizer/resolver/CMakeLists.txt | 1 + query_optimizer/resolver/Resolver.cpp | 7 ++ query_optimizer/resolver/Resolver.hpp | 5 ++ query_optimizer/rules/CMakeLists.txt | 13 ++++ .../rules/ReferencedBaseRelations.cpp | 66 ++++++++++++++++ .../rules/ReferencedBaseRelations.hpp | 80 ++++++++++++++++++++ 13 files changed, 239 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8a84039e/query_optimizer/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/query_optimizer/CMakeLists.txt b/query_optimizer/CMakeLists.txt index 1e4e346..9128c63 100644 --- a/query_optimizer/CMakeLists.txt +++ b/query_optimizer/CMakeLists.txt @@ -206,7 +206,9 @@ target_link_libraries(quickstep_queryoptimizer_LogicalToPhysicalMapper target_link_libraries(quickstep_queryoptimizer_Optimizer quickstep_queryoptimizer_ExecutionGenerator quickstep_queryoptimizer_LogicalGenerator + quickstep_queryoptimizer_OptimizerContext quickstep_queryoptimizer_PhysicalGenerator + quickstep_queryoptimizer_resolver_Resolver quickstep_utility_Macros) target_link_libraries(quickstep_queryoptimizer_OptimizerContext quickstep_queryoptimizer_expressions_ExprId http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8a84039e/query_optimizer/LogicalGenerator.cpp ---------------------------------------------------------------------- diff --git a/query_optimizer/LogicalGenerator.cpp b/query_optimizer/LogicalGenerator.cpp index abeca53..111fa1f 100644 --- a/query_optimizer/LogicalGenerator.cpp +++ b/query_optimizer/LogicalGenerator.cpp @@ -23,7 +23,6 @@ #include <vector> #include "parser/ParseStatement.hpp" - #include "query_optimizer/OptimizerContext.hpp" #include "query_optimizer/Validator.hpp" #include "query_optimizer/logical/Logical.hpp" http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8a84039e/query_optimizer/Optimizer.cpp ---------------------------------------------------------------------- diff --git a/query_optimizer/Optimizer.cpp b/query_optimizer/Optimizer.cpp index 1b91574..d002ca1 100644 --- a/query_optimizer/Optimizer.cpp +++ b/query_optimizer/Optimizer.cpp @@ -21,6 +21,8 @@ #include "query_optimizer/ExecutionGenerator.hpp" #include "query_optimizer/LogicalGenerator.hpp" +#include "query_optimizer/OptimizerContext.hpp" +#include "query_optimizer/resolver/Resolver.hpp" namespace quickstep { namespace optimizer { @@ -38,5 +40,14 @@ void Optimizer::generateQueryHandle(const ParseStatement &parse_statement, logical_generator.generatePlan(*catalog_database, parse_statement))); } +void Optimizer::findReferencedBaseRelations(const ParseStatement &parse_statement, + CatalogDatabase *catalog_database, + QueryHandle *query_handle) { + OptimizerContext optimizer_context; + resolver::Resolver resolver(*catalog_database, &optimizer_context); + resolver.resolve(parse_statement); + query_handle->setReferencedBaseRelations(resolver.getReferencedBaseRelations()); +} + } // namespace optimizer } // namespace quickstep http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8a84039e/query_optimizer/Optimizer.hpp ---------------------------------------------------------------------- diff --git a/query_optimizer/Optimizer.hpp b/query_optimizer/Optimizer.hpp index 227dd04..16f4122 100644 --- a/query_optimizer/Optimizer.hpp +++ b/query_optimizer/Optimizer.hpp @@ -69,6 +69,21 @@ class Optimizer { OptimizerContext *optimizer_context, QueryHandle *query_handle); + /** + * @brief Find the reference base relations in the query and set them in the + * QueryHandle. + * + * @note This function is useful if the objective is to only find the + * referenced relations in the query and not to optimize the query. + * + * @param parse_statement The parse tree of the input query. + * @param catalog_database The database that the query is executed on. + * @param query_handle The QueryHandle for this query. + */ + void findReferencedBaseRelations(const ParseStatement &parse_statement, + CatalogDatabase *catalog_database, + QueryHandle *query_handle); + private: DISALLOW_COPY_AND_ASSIGN(Optimizer); }; http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8a84039e/query_optimizer/QueryHandle.hpp ---------------------------------------------------------------------- diff --git a/query_optimizer/QueryHandle.hpp b/query_optimizer/QueryHandle.hpp index 6feeb4c..3e76463 100644 --- a/query_optimizer/QueryHandle.hpp +++ b/query_optimizer/QueryHandle.hpp @@ -23,6 +23,7 @@ #include <cstddef> #include <cstdint> #include <memory> +#include <vector> #include "catalog/Catalog.pb.h" #include "catalog/CatalogTypedefs.hpp" @@ -178,6 +179,23 @@ class QueryHandle { query_result_relation_ = relation; } + /** + * @brief Get the referenced base relations in this query. + **/ + const std::vector<const CatalogRelation*>& getReferencedBaseRelations() const { + return referenced_base_relations_; + } + + /** + * @brief Set the referenced base relations in this query. + * @param referenced_base_relations_ The list of referenced base relations + * in this query. + **/ + void setReferencedBaseRelations( + const std::vector<const CatalogRelation*> &referenced_base_relations) { + referenced_base_relations_ = referenced_base_relations; + } + #ifdef QUICKSTEP_DISTRIBUTED /** * @brief Whether the query will be executed in the single node. @@ -214,6 +232,9 @@ class QueryHandle { // and deleted by the Cli shell. const CatalogRelation *query_result_relation_; + std::vector<const CatalogRelation*> referenced_base_relations_; + + private: #ifdef QUICKSTEP_DISTRIBUTED // Indicate whether the query should be executed on the default Shiftboss for // correctness purpose. http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8a84039e/query_optimizer/QueryProcessor.cpp ---------------------------------------------------------------------- diff --git a/query_optimizer/QueryProcessor.cpp b/query_optimizer/QueryProcessor.cpp index d74eeba..2b5011e 100644 --- a/query_optimizer/QueryProcessor.cpp +++ b/query_optimizer/QueryProcessor.cpp @@ -33,6 +33,8 @@ using std::ofstream; namespace quickstep { +class QueryHandle; + void QueryProcessor::generateQueryHandle(const ParseStatement &statement, QueryHandle *query_handle) { optimizer::OptimizerContext optimizer_context; @@ -76,4 +78,9 @@ void QueryProcessor::loadCatalog() { catalog_altered_ = false; } +void QueryProcessor::findReferencedBaseRelationsInQuery( + const ParseStatement &statement, QueryHandle *query_handle) { + optimizer_.findReferencedBaseRelations(statement, getDefaultDatabase(), query_handle); +} + } // namespace quickstep http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8a84039e/query_optimizer/QueryProcessor.hpp ---------------------------------------------------------------------- diff --git a/query_optimizer/QueryProcessor.hpp b/query_optimizer/QueryProcessor.hpp index 2b39b84..1e16acc 100644 --- a/query_optimizer/QueryProcessor.hpp +++ b/query_optimizer/QueryProcessor.hpp @@ -161,6 +161,17 @@ class QueryProcessor { QueryHandle *query_handle); /** + * @brief Find the reference base relations in the query and set them in the + * QueryHandle. + * + * @note This function does not fully optimize the query plan. + * + * @param statement The parsed SQL statement of the query. + * @param query_handle The QueryHandle of the given query. + */ + void findReferencedBaseRelationsInQuery(const ParseStatement &statement, QueryHandle *query_handle); + + /** * @brief Save the catalog back to disk. **/ void saveCatalog(); http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8a84039e/query_optimizer/resolver/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/query_optimizer/resolver/CMakeLists.txt b/query_optimizer/resolver/CMakeLists.txt index 6feb1e8..0746275 100644 --- a/query_optimizer/resolver/CMakeLists.txt +++ b/query_optimizer/resolver/CMakeLists.txt @@ -119,6 +119,7 @@ target_link_libraries(quickstep_queryoptimizer_resolver_Resolver quickstep_queryoptimizer_logical_UpdateTable quickstep_queryoptimizer_logical_WindowAggregate quickstep_queryoptimizer_resolver_NameResolver + quickstep_queryoptimizer_rules_ReferencedBaseRelations quickstep_storage_StorageBlockLayout_proto quickstep_storage_StorageConstants quickstep_types_IntType http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8a84039e/query_optimizer/resolver/Resolver.cpp ---------------------------------------------------------------------- diff --git a/query_optimizer/resolver/Resolver.cpp b/query_optimizer/resolver/Resolver.cpp index 0b6dc22..b07cf10 100644 --- a/query_optimizer/resolver/Resolver.cpp +++ b/query_optimizer/resolver/Resolver.cpp @@ -114,6 +114,7 @@ #include "query_optimizer/logical/UpdateTable.hpp" #include "query_optimizer/logical/WindowAggregate.hpp" #include "query_optimizer/resolver/NameResolver.hpp" +#include "query_optimizer/rules/ReferencedBaseRelations.hpp" #include "storage/StorageBlockLayout.pb.h" #include "storage/StorageConstants.hpp" #include "types/IntType.hpp" @@ -454,6 +455,12 @@ L::LogicalPtr Resolver::resolve(const ParseStatement &parse_query) { return logical_plan_; } +std::vector<const CatalogRelation*> Resolver::getReferencedBaseRelations() { + std::unique_ptr<ReferencedBaseRelations> base_relations(new ReferencedBaseRelations(context_)); + base_relations->apply(logical_plan_); + return base_relations->getReferencedBaseRelations(); +} + L::LogicalPtr Resolver::resolveCopyFrom( const ParseStatementCopy ©_from_statement) { DCHECK(copy_from_statement.getCopyDirection() == ParseStatementCopy::kFrom); http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8a84039e/query_optimizer/resolver/Resolver.hpp ---------------------------------------------------------------------- diff --git a/query_optimizer/resolver/Resolver.hpp b/query_optimizer/resolver/Resolver.hpp index 1784782..be5eebf 100644 --- a/query_optimizer/resolver/Resolver.hpp +++ b/query_optimizer/resolver/Resolver.hpp @@ -114,6 +114,11 @@ class Resolver { */ logical::LogicalPtr resolve(const ParseStatement &parse_query); + /** + * @brief Get the referenced base relations in the query. + */ + std::vector<const CatalogRelation*> getReferencedBaseRelations(); + private: /** * @brief Expression-scoped info that contains both constant and non-constant http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8a84039e/query_optimizer/rules/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/query_optimizer/rules/CMakeLists.txt b/query_optimizer/rules/CMakeLists.txt index d6c043f..f9cd51c 100644 --- a/query_optimizer/rules/CMakeLists.txt +++ b/query_optimizer/rules/CMakeLists.txt @@ -39,6 +39,9 @@ add_library(quickstep_queryoptimizer_rules_PushDownSemiAntiJoin PushDownSemiAnti add_library(quickstep_queryoptimizer_rules_ReduceGroupByAttributes ReduceGroupByAttributes.cpp ReduceGroupByAttributes.hpp) +add_library(quickstep_queryoptimizer_rules_ReferencedBaseRelations + ReferencedBaseRelations.cpp + ReferencedBaseRelations.hpp) add_library(quickstep_queryoptimizer_rules_ReorderColumns ReorderColumns.cpp ReorderColumns.hpp) add_library(quickstep_queryoptimizer_rules_ReuseAggregateExpressions ReuseAggregateExpressions.cpp @@ -270,6 +273,15 @@ target_link_libraries(quickstep_queryoptimizer_rules_ReduceGroupByAttributes quickstep_queryoptimizer_rules_PruneColumns quickstep_queryoptimizer_rules_Rule quickstep_utility_Macros) +target_link_libraries(quickstep_queryoptimizer_rules_ReferencedBaseRelations + quickstep_catalog_CatalogRelation + quickstep_catalog_CatalogTypedefs + quickstep_queryoptimizer_logical_Logical + quickstep_queryoptimizer_logical_PatternMatcher + quickstep_queryoptimizer_logical_TableReference + quickstep_queryoptimizer_rules_Rule + quickstep_queryoptimizer_rules_UnnestSubqueries + quickstep_utility_Macros) target_link_libraries(quickstep_queryoptimizer_rules_ReorderColumns quickstep_queryoptimizer_expressions_AttributeReference quickstep_queryoptimizer_expressions_ExprId @@ -433,6 +445,7 @@ target_link_libraries(quickstep_queryoptimizer_rules quickstep_queryoptimizer_rules_PushDownLowCostDisjunctivePredicate quickstep_queryoptimizer_rules_PushDownSemiAntiJoin quickstep_queryoptimizer_rules_ReduceGroupByAttributes + quickstep_queryoptimizer_rules_ReferencedBaseRelations quickstep_queryoptimizer_rules_ReorderColumns quickstep_queryoptimizer_rules_ReuseAggregateExpressions quickstep_queryoptimizer_rules_Rule http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8a84039e/query_optimizer/rules/ReferencedBaseRelations.cpp ---------------------------------------------------------------------- diff --git a/query_optimizer/rules/ReferencedBaseRelations.cpp b/query_optimizer/rules/ReferencedBaseRelations.cpp new file mode 100644 index 0000000..8af3295 --- /dev/null +++ b/query_optimizer/rules/ReferencedBaseRelations.cpp @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + **/ + +#include <stack> + +#include "query_optimizer/rules/ReferencedBaseRelations.hpp" + +#include "catalog/CatalogRelation.hpp" +#include "query_optimizer/logical/Logical.hpp" +#include "query_optimizer/logical/PatternMatcher.hpp" +#include "query_optimizer/logical/TableReference.hpp" +#include "query_optimizer/rules/UnnestSubqueries.hpp" + +namespace quickstep { +namespace optimizer { + +class OptimizerContext; + +namespace L = ::quickstep::optimizer::logical; + +void ReferencedBaseRelations::applyToNode(const L::LogicalPtr &input) { + L::TableReferencePtr table_reference; + const CatalogRelation *input_relation = nullptr; + if (L::SomeTableReference::MatchesWithConditionalCast(input, &table_reference)) { + input_relation = table_reference->catalog_relation(); + referenced_base_relations_[input_relation->getID()] = input_relation; + } +} + +L::LogicalPtr ReferencedBaseRelations::apply(const L::LogicalPtr &input) { + L::LogicalPtr unnested_tree = UnnestSubqueries(optimizer_context_).apply(input); + + std::stack<L::LogicalPtr> to_be_visited; + to_be_visited.push(unnested_tree); + + while (!to_be_visited.empty()) { + const L::LogicalPtr &curr_node = to_be_visited.top(); + to_be_visited.pop(); + + applyToNode(curr_node); + + for (auto child : curr_node->children()) { + to_be_visited.push(child); + } + } + return input; +} + +} // namespace optimizer +} // namespace quickstep http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/8a84039e/query_optimizer/rules/ReferencedBaseRelations.hpp ---------------------------------------------------------------------- diff --git a/query_optimizer/rules/ReferencedBaseRelations.hpp b/query_optimizer/rules/ReferencedBaseRelations.hpp new file mode 100644 index 0000000..b24a148 --- /dev/null +++ b/query_optimizer/rules/ReferencedBaseRelations.hpp @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + **/ + +#ifndef QUICKSTEP_QUERY_OPTIMIZER_RULES_REFERENCED_BASE_RELATIONS_HPP_ +#define QUICKSTEP_QUERY_OPTIMIZER_RULES_REFERENCED_BASE_RELATIONS_HPP_ + +#include <string> +#include <unordered_map> +#include <vector> + +#include "catalog/CatalogTypedefs.hpp" +#include "query_optimizer/logical/Logical.hpp" +#include "query_optimizer/rules/Rule.hpp" +#include "utility/Macros.hpp" + +namespace quickstep { + +class CatalogRelation; + +namespace optimizer { + +class OptimizerContext; + +class ReferencedBaseRelations : public Rule<logical::Logical> { + public: + /** + * @brief Constructor + * @param optimizer_context The optimizer context. + */ + explicit ReferencedBaseRelations(OptimizerContext *optimizer_context) + : optimizer_context_(optimizer_context) { + } + + std::string getName() const override { return "ReferencedBaseRelations"; } + + logical::LogicalPtr apply(const logical::LogicalPtr &input) override; + + /** + * @brief Get the base relations referenced in a query. + */ + std::vector<const CatalogRelation*> getReferencedBaseRelations() const { + std::vector<const CatalogRelation*> ret; + for (auto it = referenced_base_relations_.begin(); + it != referenced_base_relations_.end(); + ++it) { + ret.push_back(it->second); + } + return ret; + } + + private: + void applyToNode(const logical::LogicalPtr &input); + + OptimizerContext *optimizer_context_; + + std::unordered_map<relation_id, const CatalogRelation*> referenced_base_relations_; + + DISALLOW_COPY_AND_ASSIGN(ReferencedBaseRelations); +}; + +} // namespace optimizer +} // namespace quickstep + +#endif // QUICKSTEP_QUERY_OPTIMIZER_RULES_REFERENCED_BASE_RELATIONS_HPP_
