Repository: incubator-quickstep Updated Branches: refs/heads/lip-refactor-optimizer 72b349d5f -> 10eff2c1d
Updates Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/10eff2c1 Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/10eff2c1 Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/10eff2c1 Branch: refs/heads/lip-refactor-optimizer Commit: 10eff2c1d6aff4d29db469e64897be9e1d984779 Parents: 72b349d Author: Jianqiao Zhu <jianq...@cs.wisc.edu> Authored: Tue Oct 11 11:04:27 2016 -0500 Committer: Jianqiao Zhu <jianq...@cs.wisc.edu> Committed: Tue Oct 11 11:04:27 2016 -0500 ---------------------------------------------------------------------- query_optimizer/CMakeLists.txt | 20 --- query_optimizer/ExecutionGenerator.cpp | 51 +++--- query_optimizer/ExecutionGenerator.hpp | 10 +- query_optimizer/LIPFilterGenerator.cpp | 190 ---------------------- query_optimizer/LIPFilterGenerator.hpp | 129 --------------- relational_operators/AggregationOperator.cpp | 21 +-- relational_operators/AggregationOperator.hpp | 9 +- relational_operators/BuildHashOperator.cpp | 23 +-- relational_operators/BuildHashOperator.hpp | 18 +- relational_operators/HashJoinOperator.cpp | 52 +----- relational_operators/HashJoinOperator.hpp | 43 ++--- relational_operators/RelationalOperator.hpp | 12 +- relational_operators/SelectOperator.cpp | 67 +------- relational_operators/SelectOperator.hpp | 18 +- 14 files changed, 63 insertions(+), 600 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/10eff2c1/query_optimizer/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/query_optimizer/CMakeLists.txt b/query_optimizer/CMakeLists.txt index e1f36d1..fa9141c 100644 --- a/query_optimizer/CMakeLists.txt +++ b/query_optimizer/CMakeLists.txt @@ -41,7 +41,6 @@ add_subdirectory(tests) # Declare micro-libs: add_library(quickstep_queryoptimizer_ExecutionGenerator ExecutionGenerator.cpp ExecutionGenerator.hpp) -add_library(quickstep_queryoptimizer_LIPFilterGenerator LIPFilterGenerator.cpp LIPFilterGenerator.hpp) add_library(quickstep_queryoptimizer_LogicalGenerator LogicalGenerator.cpp LogicalGenerator.hpp) add_library(quickstep_queryoptimizer_LogicalToPhysicalMapper ../empty_src.cpp @@ -73,7 +72,6 @@ target_link_libraries(quickstep_queryoptimizer_ExecutionGenerator quickstep_expressions_windowaggregation_WindowAggregateFunction_proto quickstep_queryexecution_QueryContext quickstep_queryexecution_QueryContext_proto - quickstep_queryoptimizer_LIPFilterGenerator quickstep_queryoptimizer_OptimizerContext quickstep_queryoptimizer_QueryHandle quickstep_queryoptimizer_QueryPlan @@ -153,23 +151,6 @@ if (ENABLE_DISTRIBUTED) target_link_libraries(quickstep_queryoptimizer_ExecutionGenerator quickstep_catalog_Catalog_proto) endif() -target_link_libraries(quickstep_queryoptimizer_LIPFilterGenerator - glog - quickstep_catalog_CatalogAttribute - quickstep_catalog_CatalogTypedefs - quickstep_queryexecution_QueryContext - quickstep_queryexecution_QueryContext_proto - quickstep_queryoptimizer_QueryPlan - quickstep_queryoptimizer_physical_Aggregate - quickstep_queryoptimizer_physical_HashJoin - quickstep_queryoptimizer_physical_LIPFilterConfiguration - quickstep_queryoptimizer_physical_Physical - quickstep_queryoptimizer_physical_Selection - quickstep_relationaloperators_RelationalOperator - quickstep_types_Type - quickstep_utility_lipfilter_LIPFilter - quickstep_utility_lipfilter_LIPFilterDeployment - quickstep_utility_lipfilter_LIPFilter_proto) target_link_libraries(quickstep_queryoptimizer_LogicalGenerator glog quickstep_parser_ParseStatement @@ -243,7 +224,6 @@ target_link_libraries(quickstep_queryoptimizer_Validator add_library(quickstep_queryoptimizer ../empty_src.cpp QueryOptimizerModule.hpp) target_link_libraries(quickstep_queryoptimizer quickstep_queryoptimizer_ExecutionGenerator - quickstep_queryoptimizer_LIPFilterGenerator quickstep_queryoptimizer_LogicalGenerator quickstep_queryoptimizer_LogicalToPhysicalMapper quickstep_queryoptimizer_Optimizer http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/10eff2c1/query_optimizer/ExecutionGenerator.cpp ---------------------------------------------------------------------- diff --git a/query_optimizer/ExecutionGenerator.cpp b/query_optimizer/ExecutionGenerator.cpp index 23d9a53..9347c9c 100644 --- a/query_optimizer/ExecutionGenerator.cpp +++ b/query_optimizer/ExecutionGenerator.cpp @@ -54,11 +54,10 @@ #include "expressions/window_aggregation/WindowAggregateFunction.pb.h" #include "query_execution/QueryContext.hpp" #include "query_execution/QueryContext.pb.h" -#include "query_optimizer/LIPFilterGenerator.hpp" +#include "query_optimizer/ExecutionHeuristics.hpp" #include "query_optimizer/OptimizerContext.hpp" #include "query_optimizer/QueryHandle.hpp" #include "query_optimizer/QueryPlan.hpp" -#include "query_optimizer/cost_model/SimpleCostModel.hpp" #include "query_optimizer/cost_model/StarSchemaSimpleCostModel.hpp" #include "query_optimizer/expressions/AggregateFunction.hpp" #include "query_optimizer/expressions/Alias.hpp" @@ -77,7 +76,6 @@ #include "query_optimizer/physical/HashJoin.hpp" #include "query_optimizer/physical/InsertSelection.hpp" #include "query_optimizer/physical/InsertTuple.hpp" -#include "query_optimizer/physical/LIPFilterConfiguration.hpp" #include "query_optimizer/physical/NestedLoopsJoin.hpp" #include "query_optimizer/physical/PatternMatcher.hpp" #include "query_optimizer/physical/Physical.hpp" @@ -155,6 +153,9 @@ static const volatile bool aggregate_hashtable_type_dummy DEFINE_bool(parallelize_load, true, "Parallelize loading data files."); +DEFINE_bool(optimize_joins, false, + "Enable post execution plan generation optimizations for joins."); + namespace E = ::quickstep::optimizer::expressions; namespace P = ::quickstep::optimizer::physical; namespace S = ::quickstep::serialization; @@ -167,13 +168,6 @@ void ExecutionGenerator::generatePlan(const P::PhysicalPtr &physical_plan) { cost_model_.reset( new cost::StarSchemaSimpleCostModel(top_level_physical_plan_->shared_subplans())); - simple_cost_model_.reset( - new cost::SimpleCostModel(top_level_physical_plan_->shared_subplans())); - const P::LIPFilterConfigurationPtr &lip_filter_configuration = - top_level_physical_plan_->lip_filter_configuration(); - if (lip_filter_configuration != nullptr) { - lip_filter_generator_.reset(new LIPFilterGenerator(lip_filter_configuration)); - } const CatalogRelation *result_relation = nullptr; @@ -183,10 +177,6 @@ void ExecutionGenerator::generatePlan(const P::PhysicalPtr &physical_plan) { } generatePlanInternal(top_level_physical_plan_->plan()); - if (lip_filter_generator_ != nullptr) { - lip_filter_generator_->deployLIPFilters(execution_plan_, query_context_proto_); - } - // Set the query result relation if the input plan exists in physical_to_execution_map_, // which indicates the plan is the result of a SELECT query. const std::unordered_map<P::PhysicalPtr, CatalogRelationInfo>::const_iterator it = @@ -221,6 +211,11 @@ void ExecutionGenerator::generatePlan(const P::PhysicalPtr &physical_plan) { temporary_relation_info.producer_operator_index); } + // Optimize execution plan based on heuristics captured during execution plan generation, if enabled. + if (FLAGS_optimize_joins) { + execution_heuristics_->optimizeExecutionPlan(execution_plan_, query_context_proto_); + } + #ifdef QUICKSTEP_DISTRIBUTED catalog_database_cache_proto_->set_name(catalog_database_->getName()); @@ -243,10 +238,6 @@ void ExecutionGenerator::generatePlanInternal( generatePlanInternal(child); } - if (lip_filter_generator_ != nullptr) { - lip_filter_generator_->registerAttributeMap(physical_plan, attribute_substitution_map_); - } - switch (physical_plan->getPhysicalType()) { case P::PhysicalType::kAggregate: return convertAggregate( @@ -578,10 +569,6 @@ void ExecutionGenerator::convertSelection( std::forward_as_tuple(select_index, output_relation)); temporary_relation_info_vec_.emplace_back(select_index, output_relation); - - if (lip_filter_generator_ != nullptr) { - lip_filter_generator_->addSelectionInfo(physical_selection, select_index); - } } void ExecutionGenerator::convertSharedSubplanReference(const physical::SharedSubplanReferencePtr &physical_plan) { @@ -619,7 +606,7 @@ void ExecutionGenerator::convertHashJoin(const P::HashJoinPtr &physical_plan) { const CatalogRelation *referenced_stored_probe_relation = nullptr; const CatalogRelation *referenced_stored_build_relation = nullptr; - std::size_t build_cardinality = simple_cost_model_->estimateCardinality(build_physical); + std::size_t build_cardinality = cost_model_->estimateCardinality(build_physical); bool any_probe_attributes_nullable = false; bool any_build_attributes_nullable = false; @@ -842,10 +829,15 @@ void ExecutionGenerator::convertHashJoin(const P::HashJoinPtr &physical_plan) { output_relation)); temporary_relation_info_vec_.emplace_back(join_operator_index, output_relation); - if (lip_filter_generator_ != nullptr) { - lip_filter_generator_->addHashJoinInfo(physical_plan, - build_operator_index, - join_operator_index); + // Add heuristics for the Hash Join, if enabled. + if (FLAGS_optimize_joins && !skip_hash_join_optimization) { + execution_heuristics_->addHashJoinInfo(build_operator_index, + join_operator_index, + referenced_stored_build_relation, + referenced_stored_probe_relation, + std::move(build_original_attribute_ids), + std::move(probe_original_attribute_ids), + join_hash_table_index); } } @@ -1474,11 +1466,6 @@ void ExecutionGenerator::convertAggregate( execution_plan_->addDirectDependency(destroy_aggregation_state_operator_index, finalize_aggregation_operator_index, true); - - if (lip_filter_generator_ != nullptr) { - lip_filter_generator_->addAggregateInfo(physical_plan, - aggregation_operator_index); - } } void ExecutionGenerator::convertSort(const P::SortPtr &physical_sort) { http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/10eff2c1/query_optimizer/ExecutionGenerator.hpp ---------------------------------------------------------------------- diff --git a/query_optimizer/ExecutionGenerator.hpp b/query_optimizer/ExecutionGenerator.hpp index 8890296..2aaf5ab 100644 --- a/query_optimizer/ExecutionGenerator.hpp +++ b/query_optimizer/ExecutionGenerator.hpp @@ -33,7 +33,7 @@ #include "catalog/CatalogTypedefs.hpp" #include "query_execution/QueryContext.hpp" #include "query_execution/QueryContext.pb.h" -#include "query_optimizer/LIPFilterGenerator.hpp" +#include "query_optimizer/ExecutionHeuristics.hpp" #include "query_optimizer/QueryHandle.hpp" #include "query_optimizer/QueryPlan.hpp" #include "query_optimizer/cost_model/CostModel.hpp" @@ -102,7 +102,8 @@ class ExecutionGenerator { : catalog_database_(DCHECK_NOTNULL(catalog_database)), query_handle_(DCHECK_NOTNULL(query_handle)), execution_plan_(DCHECK_NOTNULL(query_handle->getQueryPlanMutable())), - query_context_proto_(DCHECK_NOTNULL(query_handle->getQueryContextProtoMutable())) { + query_context_proto_(DCHECK_NOTNULL(query_handle->getQueryContextProtoMutable())), + execution_heuristics_(new ExecutionHeuristics()) { query_context_proto_->set_query_id(query_handle_->query_id()); #ifdef QUICKSTEP_DISTRIBUTED catalog_database_cache_proto_ = DCHECK_NOTNULL(query_handle->getCatalogDatabaseCacheProtoMutable()); @@ -385,6 +386,7 @@ class ExecutionGenerator { QueryHandle *query_handle_; QueryPlan *execution_plan_; // A part of QueryHandle. serialization::QueryContext *query_context_proto_; // A part of QueryHandle. + std::unique_ptr<ExecutionHeuristics> execution_heuristics_; #ifdef QUICKSTEP_DISTRIBUTED serialization::CatalogDatabase *catalog_database_cache_proto_; // A part of QueryHandle. @@ -420,13 +422,9 @@ class ExecutionGenerator { * @brief The cost model to use for creating the execution plan. */ std::unique_ptr<cost::CostModel> cost_model_; - std::unique_ptr<cost::CostModel> simple_cost_model_; physical::TopLevelPlanPtr top_level_physical_plan_; - // Sub-generator for deploying LIP (lookahead information passing) filters. - std::unique_ptr<LIPFilterGenerator> lip_filter_generator_; - DISALLOW_COPY_AND_ASSIGN(ExecutionGenerator); }; http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/10eff2c1/query_optimizer/LIPFilterGenerator.cpp ---------------------------------------------------------------------- diff --git a/query_optimizer/LIPFilterGenerator.cpp b/query_optimizer/LIPFilterGenerator.cpp deleted file mode 100644 index ef10400..0000000 --- a/query_optimizer/LIPFilterGenerator.cpp +++ /dev/null @@ -1,190 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - **/ - -#include "query_optimizer/LIPFilterGenerator.hpp" - -#include <map> -#include <utility> - -#include "catalog/CatalogAttribute.hpp" -#include "catalog/CatalogTypedefs.hpp" -#include "query_execution/QueryContext.pb.h" -#include "relational_operators/RelationalOperator.hpp" -#include "types/Type.hpp" -#include "utility/lip_filter/LIPFilter.hpp" -#include "utility/lip_filter/LIPFilter.pb.h" - -#include "glog/logging.h" - -namespace quickstep { -namespace optimizer { - -namespace E = ::quickstep::optimizer::expressions; -namespace P = ::quickstep::optimizer::physical; - -void LIPFilterGenerator::registerAttributeMap( - const P::PhysicalPtr &node, - const std::unordered_map<E::ExprId, const CatalogAttribute *> &attribute_substitution_map) { - const auto &build_info_map = lip_filter_configuration_->getBuildInfoMap(); - const auto build_it = build_info_map.find(node); - if (build_it != build_info_map.end()) { - auto &map_entry = attribute_map_[node]; - for (const auto &info : build_it->second) { - E::ExprId attr_id = info.build_attribute->id(); - map_entry.emplace(attr_id, attribute_substitution_map.at(attr_id)); - } - } - const auto &probe_info_map = lip_filter_configuration_->getProbeInfoMap(); - const auto probe_it = probe_info_map.find(node); - if (probe_it != probe_info_map.end()) { - auto &map_entry = attribute_map_[node]; - for (const auto &info : probe_it->second) { - E::ExprId attr_id = info.probe_attribute->id(); - map_entry.emplace(attr_id, attribute_substitution_map.at(attr_id)); - } - } -} - -void LIPFilterGenerator::deployLIPFilters(QueryPlan *execution_plan, - serialization::QueryContext *query_context_proto) const { - LIPFilterBuilderMap lip_filter_builder_map; - - // Deploy builders - const auto &build_info_map = lip_filter_configuration_->getBuildInfoMap(); - for (const auto &info : builder_infos_) { - const auto build_it = build_info_map.find(info.builder_node); - if (build_it != build_info_map.end()) { - deployBuilderInternal(execution_plan, - query_context_proto, - info.builder_node, - info.builder_operator_index, - build_it->second, - &lip_filter_builder_map); - } - } - - // Deploy probers - const auto &probe_info_map = lip_filter_configuration_->getProbeInfoMap(); - for (const auto &info : prober_infos_) { - const auto probe_it = probe_info_map.find(info.prober_node); - if (probe_it != probe_info_map.end()) { - deployProberInteral(execution_plan, - query_context_proto, - info.prober_node, - info.prober_operator_index, - probe_it->second, - lip_filter_builder_map); - } - } -} - -void LIPFilterGenerator::deployBuilderInternal( - QueryPlan *execution_plan, - serialization::QueryContext *query_context_proto, - const physical::PhysicalPtr &builder_node, - const QueryPlan::DAGNodeIndex builder_operator_index, - const std::vector<physical::LIPFilterBuildInfo> &build_info_vec, - LIPFilterBuilderMap *lip_filter_builder_map) const { - const auto lip_deployment_index = query_context_proto->lip_filter_deployments_size(); - auto *lip_filter_deployment_info_proto = - query_context_proto->add_lip_filter_deployments(); - lip_filter_deployment_info_proto->set_action_type(serialization::LIPFilterActionType::BUILD); - - const auto &builder_attribute_map = attribute_map_.at(builder_node); - for (const auto &info : build_info_vec) { - const QueryContext::lip_filter_id lip_filter_id = query_context_proto->lip_filters_size(); - serialization::LIPFilter *lip_filter_proto = query_context_proto->add_lip_filters(); - const CatalogAttribute *target_attr = builder_attribute_map.at(info.build_attribute->id()); - const Type &attr_type = target_attr->getType(); - - switch (info.filter_type) { - case LIPFilterType::kSingleIdentityHashFilter: { - DCHECK(!attr_type.isVariableLength()); - lip_filter_proto->set_lip_filter_type( - serialization::LIPFilterType::SINGLE_IDENTITY_HASH_FILTER); - lip_filter_proto->SetExtension( - serialization::SingleIdentityHashFilter::filter_cardinality, info.filter_cardinality); - lip_filter_proto->SetExtension( - serialization::SingleIdentityHashFilter::attribute_size, attr_type.minimumByteLength()); - break; - } - default: - LOG(FATAL) << "Unsupported LIPFilter type"; - break; - } - - lip_filter_builder_map->emplace( - std::make_pair(info.build_attribute->id(), builder_node), - std::make_pair(lip_filter_id, builder_operator_index)); - - auto *lip_filter_entry_proto = lip_filter_deployment_info_proto->add_entries(); - lip_filter_entry_proto->set_lip_filter_id(lip_filter_id); - lip_filter_entry_proto->set_attribute_id(target_attr->getID()); - lip_filter_entry_proto->mutable_attribute_type()->CopyFrom(attr_type.getProto()); - - std::cerr << "Build " << info.build_attribute->toString() - << " @" << builder_node - << " size = " << info.filter_cardinality << "\n"; - } - - RelationalOperator *relop = - execution_plan->getQueryPlanDAGMutable()->getNodePayloadMutable(builder_operator_index); - relop->deployLIPFilter(lip_deployment_index); -} - -void LIPFilterGenerator::deployProberInteral( - QueryPlan *execution_plan, - serialization::QueryContext *query_context_proto, - const physical::PhysicalPtr &prober_node, - const QueryPlan::DAGNodeIndex prober_operator_index, - const std::vector<physical::LIPFilterProbeInfo> &probe_info_vec, - const LIPFilterBuilderMap &lip_filter_builder_map) const { - const auto lip_deployment_index = query_context_proto->lip_filter_deployments_size(); - auto *lip_filter_deployment_info_proto = - query_context_proto->add_lip_filter_deployments(); - lip_filter_deployment_info_proto->set_action_type(serialization::LIPFilterActionType::PROBE); - - const auto &prober_attribute_map = attribute_map_.at(prober_node); - for (const auto &info : probe_info_vec) { - const auto &builder_info = - lip_filter_builder_map.at( - std::make_pair(info.build_attribute->id(), info.builder)); - const CatalogAttribute *target_attr = prober_attribute_map.at(info.probe_attribute->id()); - - auto *lip_filter_entry_proto = lip_filter_deployment_info_proto->add_entries(); - lip_filter_entry_proto->set_lip_filter_id(builder_info.first); - lip_filter_entry_proto->set_attribute_id(target_attr->getID()); - lip_filter_entry_proto->mutable_attribute_type()->CopyFrom( - target_attr->getType().getProto()); - - execution_plan->addDirectDependency(prober_operator_index, - builder_info.second, - true /* is_pipeline_breaker */); - - std::cerr << "Probe " << info.probe_attribute->toString() - << " @" << prober_node << "\n"; - } - - RelationalOperator *relop = - execution_plan->getQueryPlanDAGMutable()->getNodePayloadMutable(prober_operator_index); - relop->deployLIPFilter(lip_deployment_index); -} - -} // namespace optimizer -} // namespace quickstep http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/10eff2c1/query_optimizer/LIPFilterGenerator.hpp ---------------------------------------------------------------------- diff --git a/query_optimizer/LIPFilterGenerator.hpp b/query_optimizer/LIPFilterGenerator.hpp deleted file mode 100644 index 4b597cf..0000000 --- a/query_optimizer/LIPFilterGenerator.hpp +++ /dev/null @@ -1,129 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - **/ - -#ifndef QUICKSTEP_QUERY_OPTIMIZER_LIP_FILTER_GENERATOR_HPP_ -#define QUICKSTEP_QUERY_OPTIMIZER_LIP_FILTER_GENERATOR_HPP_ - -#include <map> -#include <unordered_map> - -#include "catalog/CatalogTypedefs.hpp" -#include "query_execution/QueryContext.hpp" -#include "query_execution/QueryContext.pb.h" -#include "query_optimizer/QueryPlan.hpp" -#include "query_optimizer/physical/LIPFilterConfiguration.hpp" -#include "query_optimizer/physical/Aggregate.hpp" -#include "query_optimizer/physical/HashJoin.hpp" -#include "query_optimizer/physical/Physical.hpp" -#include "query_optimizer/physical/Selection.hpp" - -#include "glog/logging.h" - -namespace quickstep { - -class CatalogAttribute; - -namespace optimizer { - -/** \addtogroup QueryOptimizer - * @{ - */ - -class LIPFilterGenerator { - public: - LIPFilterGenerator(const physical::LIPFilterConfigurationPtr &lip_filter_configuration) - : lip_filter_configuration_(lip_filter_configuration) { - DCHECK(lip_filter_configuration_ != nullptr); - } - - void registerAttributeMap( - const physical::PhysicalPtr &node, - const std::unordered_map<expressions::ExprId, const CatalogAttribute *> &attribute_substitution_map); - - void addAggregateInfo(const physical::AggregatePtr &aggregate, - const QueryPlan::DAGNodeIndex aggregate_operator_index) { - prober_infos_.emplace_back(aggregate, aggregate_operator_index); - } - - void addHashJoinInfo(const physical::HashJoinPtr &hash_join, - const QueryPlan::DAGNodeIndex build_operator_index, - const QueryPlan::DAGNodeIndex join_operator_index) { - builder_infos_.emplace_back(hash_join, build_operator_index); - prober_infos_.emplace_back(hash_join, join_operator_index); - } - - void addSelectionInfo(const physical::SelectionPtr &selection, - const QueryPlan::DAGNodeIndex select_operator_index) { - prober_infos_.emplace_back(selection, select_operator_index); - } - - void deployLIPFilters(QueryPlan *execution_plan, - serialization::QueryContext *query_context_proto) const; - - private: - struct BuilderInfo { - BuilderInfo(const physical::PhysicalPtr &builder_node_in, - const QueryPlan::DAGNodeIndex builder_operator_index_in) - : builder_node(builder_node_in), - builder_operator_index(builder_operator_index_in) { - } - const physical::PhysicalPtr builder_node; - const QueryPlan::DAGNodeIndex builder_operator_index; - }; - - struct ProberInfo { - ProberInfo(const physical::PhysicalPtr &prober_node_in, - const QueryPlan::DAGNodeIndex prober_operator_index_in) - : prober_node(prober_node_in), - prober_operator_index(prober_operator_index_in) { - } - const physical::PhysicalPtr prober_node; - const QueryPlan::DAGNodeIndex prober_operator_index; - }; - - typedef std::map<std::pair<expressions::ExprId, physical::PhysicalPtr>, - std::pair<QueryContext::lip_filter_id, QueryPlan::DAGNodeIndex>> LIPFilterBuilderMap; - - void deployBuilderInternal(QueryPlan *execution_plan, - serialization::QueryContext *query_context_proto, - const physical::PhysicalPtr &builder_node, - const QueryPlan::DAGNodeIndex builder_operator_index, - const std::vector<physical::LIPFilterBuildInfo> &build_info_vec, - LIPFilterBuilderMap *lip_filter_builder_map) const; - - void deployProberInteral(QueryPlan *execution_plan, - serialization::QueryContext *query_context_proto, - const physical::PhysicalPtr &prober_node, - const QueryPlan::DAGNodeIndex prober_operator_index, - const std::vector<physical::LIPFilterProbeInfo> &probe_info_vec, - const LIPFilterBuilderMap &lip_filter_builder_map) const; - - const physical::LIPFilterConfigurationPtr lip_filter_configuration_; - std::map<physical::PhysicalPtr, std::map<expressions::ExprId, const CatalogAttribute *>> attribute_map_; - std::vector<BuilderInfo> builder_infos_; - std::vector<ProberInfo> prober_infos_; -}; - - -/** @} */ - -} // namespace optimizer -} // namespace quickstep - -#endif /* QUICKSTEP_QUERY_OPTIMIZER_LIP_FILTER_GENERATOR_HPP_ */ http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/10eff2c1/relational_operators/AggregationOperator.cpp ---------------------------------------------------------------------- diff --git a/relational_operators/AggregationOperator.cpp b/relational_operators/AggregationOperator.cpp index 71baa53..056e76d 100644 --- a/relational_operators/AggregationOperator.cpp +++ b/relational_operators/AggregationOperator.cpp @@ -38,24 +38,14 @@ bool AggregationOperator::getAllWorkOrders( StorageManager *storage_manager, const tmb::client_id scheduler_client_id, tmb::MessageBus *bus) { - const LIPFilterDeployment *lip_filter_deployment = nullptr; - if (lip_deployment_index_ != QueryContext::kInvalidILIPDeploymentId) { - lip_filter_deployment = query_context->getLIPDeployment(lip_deployment_index_); - } - if (input_relation_is_stored_) { if (!started_) { for (const block_id input_block_id : input_relation_block_ids_) { - LIPFilterAdaptiveProber *lip_filter_adaptive_prober = nullptr; - if (lip_filter_deployment != nullptr) { - lip_filter_adaptive_prober = lip_filter_deployment->createLIPFilterAdaptiveProber(); - } container->addNormalWorkOrder( new AggregationWorkOrder( query_id_, input_block_id, - query_context->getAggregationState(aggr_state_index_), - lip_filter_adaptive_prober), + query_context->getAggregationState(aggr_state_index_)), op_index_); } started_ = true; @@ -63,16 +53,11 @@ bool AggregationOperator::getAllWorkOrders( return started_; } else { while (num_workorders_generated_ < input_relation_block_ids_.size()) { - LIPFilterAdaptiveProber *lip_filter_adaptive_prober = nullptr; - if (lip_filter_deployment != nullptr) { - lip_filter_adaptive_prober = lip_filter_deployment->createLIPFilterAdaptiveProber(); - } container->addNormalWorkOrder( new AggregationWorkOrder( query_id_, input_relation_block_ids_[num_workorders_generated_], - query_context->getAggregationState(aggr_state_index_), - lip_filter_adaptive_prober), + query_context->getAggregationState(aggr_state_index_)), op_index_); ++num_workorders_generated_; } @@ -113,7 +98,7 @@ serialization::WorkOrder* AggregationOperator::createWorkOrderProto(const block_ void AggregationWorkOrder::execute() { - state_->aggregateBlock(input_block_id_, lip_filter_adaptive_prober_.get()); + state_->aggregateBlock(input_block_id_); } } // namespace quickstep http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/10eff2c1/relational_operators/AggregationOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/AggregationOperator.hpp b/relational_operators/AggregationOperator.hpp index da36d57..31c1da4 100644 --- a/relational_operators/AggregationOperator.hpp +++ b/relational_operators/AggregationOperator.hpp @@ -30,7 +30,6 @@ #include "relational_operators/WorkOrder.hpp" #include "storage/StorageBlockInfo.hpp" #include "utility/Macros.hpp" -#include "utility/lip_filter/LIPFilterAdaptiveProber.hpp" #include "glog/logging.h" @@ -141,12 +140,10 @@ class AggregationWorkOrder : public WorkOrder { **/ AggregationWorkOrder(const std::size_t query_id, const block_id input_block_id, - AggregationOperationState *state, - LIPFilterAdaptiveProber *lip_filter_adaptive_prober) + AggregationOperationState *state) : WorkOrder(query_id), input_block_id_(input_block_id), - state_(DCHECK_NOTNULL(state)), - lip_filter_adaptive_prober_(lip_filter_adaptive_prober) {} + state_(DCHECK_NOTNULL(state)) {} ~AggregationWorkOrder() override {} @@ -156,8 +153,6 @@ class AggregationWorkOrder : public WorkOrder { const block_id input_block_id_; AggregationOperationState *state_; - std::unique_ptr<LIPFilterAdaptiveProber> lip_filter_adaptive_prober_; - DISALLOW_COPY_AND_ASSIGN(AggregationWorkOrder); }; http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/10eff2c1/relational_operators/BuildHashOperator.cpp ---------------------------------------------------------------------- diff --git a/relational_operators/BuildHashOperator.cpp b/relational_operators/BuildHashOperator.cpp index 35bb5cf..465621c 100644 --- a/relational_operators/BuildHashOperator.cpp +++ b/relational_operators/BuildHashOperator.cpp @@ -34,7 +34,6 @@ #include "storage/TupleReference.hpp" #include "storage/TupleStorageSubBlock.hpp" #include "storage/ValueAccessor.hpp" -#include "utility/lip_filter/LIPFilterBuilder.hpp" #include "glog/logging.h" @@ -69,14 +68,6 @@ bool BuildHashOperator::getAllWorkOrders( tmb::MessageBus *bus) { DCHECK(query_context != nullptr); - LIPFilterBuilderPtr lip_filter_builder = nullptr; - if (lip_deployment_index_ != QueryContext::kInvalidILIPDeploymentId) { - const LIPFilterDeployment *lip_filter_deployment = - query_context->getLIPDeployment(lip_deployment_index_); - lip_filter_builder = std::shared_ptr<LIPFilterBuilder>( - lip_filter_deployment->createLIPFilterBuilder()); - } - JoinHashTable *hash_table = query_context->getJoinHashTable(hash_table_index_); if (input_relation_is_stored_) { if (!started_) { @@ -88,8 +79,7 @@ bool BuildHashOperator::getAllWorkOrders( any_join_key_attributes_nullable_, input_block_id, hash_table, - storage_manager, - lip_filter_builder), + storage_manager), op_index_); } started_ = true; @@ -105,8 +95,7 @@ bool BuildHashOperator::getAllWorkOrders( any_join_key_attributes_nullable_, input_relation_block_ids_[num_workorders_generated_], hash_table, - storage_manager, - lip_filter_builder), + storage_manager), op_index_); ++num_workorders_generated_; } @@ -147,23 +136,17 @@ serialization::WorkOrder* BuildHashOperator::createWorkOrderProto(const block_id any_join_key_attributes_nullable_); proto->SetExtension(serialization::BuildHashWorkOrder::join_hash_table_index, hash_table_index_); proto->SetExtension(serialization::BuildHashWorkOrder::block_id, block); - // TODO(jianqiao): update lip_filter related stuff return proto; } + void BuildHashWorkOrder::execute() { BlockReference block( storage_manager_->getBlock(build_block_id_, input_relation_)); TupleReferenceGenerator generator(build_block_id_); std::unique_ptr<ValueAccessor> accessor(block->getTupleStorageSubBlock().createValueAccessor()); - - if (lip_filter_builder_ != nullptr) { - lip_filter_builder_->insertValueAccessor(accessor.get()); - accessor->beginIterationVirtual(); - } - HashTablePutResult result; if (join_key_attributes_.size() == 1) { result = hash_table_->putValueAccessor(accessor.get(), http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/10eff2c1/relational_operators/BuildHashOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/BuildHashOperator.hpp b/relational_operators/BuildHashOperator.hpp index 940298c..4a80a8a 100644 --- a/relational_operators/BuildHashOperator.hpp +++ b/relational_operators/BuildHashOperator.hpp @@ -20,7 +20,6 @@ #ifndef QUICKSTEP_RELATIONAL_OPERATORS_BUILD_HASH_OPERATOR_HPP_ #define QUICKSTEP_RELATIONAL_OPERATORS_BUILD_HASH_OPERATOR_HPP_ -#include <memory> #include <string> #include <utility> #include <vector> @@ -32,7 +31,6 @@ #include "relational_operators/WorkOrder.hpp" #include "storage/StorageBlockInfo.hpp" #include "utility/Macros.hpp" -#include "utility/lip_filter/LIPFilterBuilder.hpp" #include "glog/logging.h" @@ -164,7 +162,6 @@ class BuildHashWorkOrder : public WorkOrder { * @param build_block_id The block id. * @param hash_table The JoinHashTable to use. * @param storage_manager The StorageManager to use. - * @param lip_filter_builder The attached builder for building LIP filters. **/ BuildHashWorkOrder(const std::size_t query_id, const CatalogRelationSchema &input_relation, @@ -172,16 +169,14 @@ class BuildHashWorkOrder : public WorkOrder { const bool any_join_key_attributes_nullable, const block_id build_block_id, JoinHashTable *hash_table, - StorageManager *storage_manager, - LIPFilterBuilderPtr lip_filter_builder = nullptr) + StorageManager *storage_manager) : WorkOrder(query_id), input_relation_(input_relation), join_key_attributes_(join_key_attributes), any_join_key_attributes_nullable_(any_join_key_attributes_nullable), build_block_id_(build_block_id), hash_table_(DCHECK_NOTNULL(hash_table)), - storage_manager_(DCHECK_NOTNULL(storage_manager)), - lip_filter_builder_(lip_filter_builder) {} + storage_manager_(DCHECK_NOTNULL(storage_manager)) {} /** * @brief Constructor for the distributed version. @@ -194,7 +189,6 @@ class BuildHashWorkOrder : public WorkOrder { * @param build_block_id The block id. * @param hash_table The JoinHashTable to use. * @param storage_manager The StorageManager to use. - * @param lip_filter_builder The attached builder for building LIP filters. **/ BuildHashWorkOrder(const std::size_t query_id, const CatalogRelationSchema &input_relation, @@ -202,16 +196,14 @@ class BuildHashWorkOrder : public WorkOrder { const bool any_join_key_attributes_nullable, const block_id build_block_id, JoinHashTable *hash_table, - StorageManager *storage_manager, - LIPFilterBuilderPtr lip_filter_builder = nullptr) + StorageManager *storage_manager) : WorkOrder(query_id), input_relation_(input_relation), join_key_attributes_(std::move(join_key_attributes)), any_join_key_attributes_nullable_(any_join_key_attributes_nullable), build_block_id_(build_block_id), hash_table_(DCHECK_NOTNULL(hash_table)), - storage_manager_(DCHECK_NOTNULL(storage_manager)), - lip_filter_builder_(lip_filter_builder) {} + storage_manager_(DCHECK_NOTNULL(storage_manager)) {} ~BuildHashWorkOrder() override {} @@ -230,8 +222,6 @@ class BuildHashWorkOrder : public WorkOrder { JoinHashTable *hash_table_; StorageManager *storage_manager_; - LIPFilterBuilderPtr lip_filter_builder_; - DISALLOW_COPY_AND_ASSIGN(BuildHashWorkOrder); }; http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/10eff2c1/relational_operators/HashJoinOperator.cpp ---------------------------------------------------------------------- diff --git a/relational_operators/HashJoinOperator.cpp b/relational_operators/HashJoinOperator.cpp index 449a118..779c0fe 100644 --- a/relational_operators/HashJoinOperator.cpp +++ b/relational_operators/HashJoinOperator.cpp @@ -48,7 +48,6 @@ #include "types/TypedValue.hpp" #include "types/containers/ColumnVector.hpp" #include "types/containers/ColumnVectorsValueAccessor.hpp" -#include "utility/lip_filter/LIPFilterAdaptiveProber.hpp" #include "gflags/gflags.h" #include "glog/logging.h" @@ -181,11 +180,6 @@ bool HashJoinOperator::getAllNonOuterJoinWorkOrders( if (blocking_dependencies_met_) { DCHECK(query_context != nullptr); - const LIPFilterDeployment *lip_filter_deployment = nullptr; - if (lip_deployment_index_ != QueryContext::kInvalidILIPDeploymentId) { - lip_filter_deployment = query_context->getLIPDeployment(lip_deployment_index_); - } - const Predicate *residual_predicate = query_context->getPredicate(residual_predicate_index_); const vector<unique_ptr<const Scalar>> &selection = @@ -198,10 +192,6 @@ bool HashJoinOperator::getAllNonOuterJoinWorkOrders( if (probe_relation_is_stored_) { if (!started_) { for (const block_id probe_block_id : probe_relation_block_ids_) { - LIPFilterAdaptiveProber *lip_filter_adaptive_prober = nullptr; - if (lip_filter_deployment != nullptr) { - lip_filter_adaptive_prober = lip_filter_deployment->createLIPFilterAdaptiveProber(); - } container->addNormalWorkOrder( new JoinWorkOrderClass(query_id_, build_relation_, @@ -213,8 +203,7 @@ bool HashJoinOperator::getAllNonOuterJoinWorkOrders( selection, hash_table, output_destination, - storage_manager, - lip_filter_adaptive_prober), + storage_manager), op_index_); } started_ = true; @@ -222,10 +211,6 @@ bool HashJoinOperator::getAllNonOuterJoinWorkOrders( return started_; } else { while (num_workorders_generated_ < probe_relation_block_ids_.size()) { - LIPFilterAdaptiveProber *lip_filter_adaptive_prober = nullptr; - if (lip_filter_deployment != nullptr) { - lip_filter_adaptive_prober = lip_filter_deployment->createLIPFilterAdaptiveProber(); - } container->addNormalWorkOrder( new JoinWorkOrderClass( query_id_, @@ -238,8 +223,7 @@ bool HashJoinOperator::getAllNonOuterJoinWorkOrders( selection, hash_table, output_destination, - storage_manager, - lip_filter_adaptive_prober), + storage_manager), op_index_); ++num_workorders_generated_; } // end while @@ -439,17 +423,6 @@ void HashInnerJoinWorkOrder::execute() { const TupleStorageSubBlock &probe_store = probe_block->getTupleStorageSubBlock(); std::unique_ptr<ValueAccessor> probe_accessor(probe_store.createValueAccessor()); - - std::unique_ptr<TupleIdSequence> lip_filter_existence_map; - std::unique_ptr<ValueAccessor> base_accessor; - if (lip_filter_adaptive_prober_ != nullptr) { - base_accessor.reset(probe_accessor.release()); - lip_filter_existence_map.reset( - lip_filter_adaptive_prober_->filterValueAccessor(base_accessor.get())); - probe_accessor.reset( - base_accessor->createSharedTupleIdSequenceAdapterVirtual(*lip_filter_existence_map)); - } - MapBasedJoinedTupleCollector collector; if (join_key_attributes_.size() == 1) { hash_table_.getAllFromValueAccessor( @@ -556,16 +529,6 @@ void HashSemiJoinWorkOrder::executeWithResidualPredicate() { std::unique_ptr<ValueAccessor> probe_accessor(probe_store.createValueAccessor()); - std::unique_ptr<TupleIdSequence> lip_filter_existence_map; - std::unique_ptr<ValueAccessor> base_accessor; - if (lip_filter_adaptive_prober_ != nullptr) { - base_accessor.reset(probe_accessor.release()); - lip_filter_existence_map.reset( - lip_filter_adaptive_prober_->filterValueAccessor(base_accessor.get())); - probe_accessor.reset( - base_accessor->createSharedTupleIdSequenceAdapterVirtual(*lip_filter_existence_map)); - } - // We collect all the matching probe relation tuples, as there's a residual // preidcate that needs to be applied after collecting these matches. MapBasedJoinedTupleCollector collector; @@ -646,17 +609,6 @@ void HashSemiJoinWorkOrder::executeWithoutResidualPredicate() { const TupleStorageSubBlock &probe_store = probe_block->getTupleStorageSubBlock(); std::unique_ptr<ValueAccessor> probe_accessor(probe_store.createValueAccessor()); - - std::unique_ptr<TupleIdSequence> lip_filter_existence_map; - std::unique_ptr<ValueAccessor> base_accessor; - if (lip_filter_adaptive_prober_ != nullptr) { - base_accessor.reset(probe_accessor.release()); - lip_filter_existence_map.reset( - lip_filter_adaptive_prober_->filterValueAccessor(base_accessor.get())); - probe_accessor.reset( - base_accessor->createSharedTupleIdSequenceAdapterVirtual(*lip_filter_existence_map)); - } - SemiAntiJoinTupleCollector collector(probe_store); // We collect all the probe relation tuples which have at least one matching // tuple in the build relation. As a performance optimization, the hash table http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/10eff2c1/relational_operators/HashJoinOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/HashJoinOperator.hpp b/relational_operators/HashJoinOperator.hpp index 29d6eba..fa393b6 100644 --- a/relational_operators/HashJoinOperator.hpp +++ b/relational_operators/HashJoinOperator.hpp @@ -35,7 +35,6 @@ #include "storage/HashTable.hpp" #include "storage/StorageBlockInfo.hpp" #include "utility/Macros.hpp" -#include "utility/lip_filter/LIPFilterAdaptiveProber.hpp" #include "glog/logging.h" @@ -308,8 +307,7 @@ class HashInnerJoinWorkOrder : public WorkOrder { const std::vector<std::unique_ptr<const Scalar>> &selection, const JoinHashTable &hash_table, InsertDestination *output_destination, - StorageManager *storage_manager, - LIPFilterAdaptiveProber *lip_filter_adaptive_prober = nullptr) + StorageManager *storage_manager) : WorkOrder(query_id), build_relation_(build_relation), probe_relation_(probe_relation), @@ -320,8 +318,7 @@ class HashInnerJoinWorkOrder : public WorkOrder { selection_(selection), hash_table_(hash_table), output_destination_(DCHECK_NOTNULL(output_destination)), - storage_manager_(DCHECK_NOTNULL(storage_manager)), - lip_filter_adaptive_prober_(lip_filter_adaptive_prober) {} + storage_manager_(DCHECK_NOTNULL(storage_manager)) {} /** * @brief Constructor for the distributed version. @@ -357,8 +354,7 @@ class HashInnerJoinWorkOrder : public WorkOrder { const std::vector<std::unique_ptr<const Scalar>> &selection, const JoinHashTable &hash_table, InsertDestination *output_destination, - StorageManager *storage_manager, - LIPFilterAdaptiveProber *lip_filter_adaptive_prober = nullptr) + StorageManager *storage_manager) : WorkOrder(query_id), build_relation_(build_relation), probe_relation_(probe_relation), @@ -369,8 +365,7 @@ class HashInnerJoinWorkOrder : public WorkOrder { selection_(selection), hash_table_(hash_table), output_destination_(DCHECK_NOTNULL(output_destination)), - storage_manager_(DCHECK_NOTNULL(storage_manager)), - lip_filter_adaptive_prober_(lip_filter_adaptive_prober) {} + storage_manager_(DCHECK_NOTNULL(storage_manager)) {} ~HashInnerJoinWorkOrder() override {} @@ -397,8 +392,6 @@ class HashInnerJoinWorkOrder : public WorkOrder { InsertDestination *output_destination_; StorageManager *storage_manager_; - std::unique_ptr<LIPFilterAdaptiveProber> lip_filter_adaptive_prober_; - DISALLOW_COPY_AND_ASSIGN(HashInnerJoinWorkOrder); }; @@ -442,8 +435,7 @@ class HashSemiJoinWorkOrder : public WorkOrder { const std::vector<std::unique_ptr<const Scalar>> &selection, const JoinHashTable &hash_table, InsertDestination *output_destination, - StorageManager *storage_manager, - LIPFilterAdaptiveProber *lip_filter_adaptive_prober = nullptr) + StorageManager *storage_manager) : WorkOrder(query_id), build_relation_(build_relation), probe_relation_(probe_relation), @@ -454,8 +446,7 @@ class HashSemiJoinWorkOrder : public WorkOrder { selection_(selection), hash_table_(hash_table), output_destination_(DCHECK_NOTNULL(output_destination)), - storage_manager_(DCHECK_NOTNULL(storage_manager)), - lip_filter_adaptive_prober_(lip_filter_adaptive_prober) {} + storage_manager_(DCHECK_NOTNULL(storage_manager)) {} /** * @brief Constructor for the distributed version. @@ -491,8 +482,7 @@ class HashSemiJoinWorkOrder : public WorkOrder { const std::vector<std::unique_ptr<const Scalar>> &selection, const JoinHashTable &hash_table, InsertDestination *output_destination, - StorageManager *storage_manager, - LIPFilterAdaptiveProber *lip_filter_adaptive_prober = nullptr) + StorageManager *storage_manager) : WorkOrder(query_id), build_relation_(build_relation), probe_relation_(probe_relation), @@ -503,8 +493,7 @@ class HashSemiJoinWorkOrder : public WorkOrder { selection_(selection), hash_table_(hash_table), output_destination_(DCHECK_NOTNULL(output_destination)), - storage_manager_(DCHECK_NOTNULL(storage_manager)), - lip_filter_adaptive_prober_(lip_filter_adaptive_prober) {} + storage_manager_(DCHECK_NOTNULL(storage_manager)) {} ~HashSemiJoinWorkOrder() override {} @@ -527,8 +516,6 @@ class HashSemiJoinWorkOrder : public WorkOrder { InsertDestination *output_destination_; StorageManager *storage_manager_; - std::unique_ptr<LIPFilterAdaptiveProber> lip_filter_adaptive_prober_; - DISALLOW_COPY_AND_ASSIGN(HashSemiJoinWorkOrder); }; @@ -572,8 +559,7 @@ class HashAntiJoinWorkOrder : public WorkOrder { const std::vector<std::unique_ptr<const Scalar>> &selection, const JoinHashTable &hash_table, InsertDestination *output_destination, - StorageManager *storage_manager, - LIPFilterAdaptiveProber *lip_filter_adaptive_prober = nullptr) + StorageManager *storage_manager) : WorkOrder(query_id), build_relation_(build_relation), probe_relation_(probe_relation), @@ -584,8 +570,7 @@ class HashAntiJoinWorkOrder : public WorkOrder { selection_(selection), hash_table_(hash_table), output_destination_(DCHECK_NOTNULL(output_destination)), - storage_manager_(DCHECK_NOTNULL(storage_manager)), - lip_filter_adaptive_prober_(lip_filter_adaptive_prober) {} + storage_manager_(DCHECK_NOTNULL(storage_manager)) {} /** * @brief Constructor for the distributed version. @@ -621,8 +606,7 @@ class HashAntiJoinWorkOrder : public WorkOrder { const std::vector<std::unique_ptr<const Scalar>> &selection, const JoinHashTable &hash_table, InsertDestination *output_destination, - StorageManager *storage_manager, - LIPFilterAdaptiveProber *lip_filter_adaptive_prober = nullptr) + StorageManager *storage_manager) : WorkOrder(query_id), build_relation_(build_relation), probe_relation_(probe_relation), @@ -633,8 +617,7 @@ class HashAntiJoinWorkOrder : public WorkOrder { selection_(selection), hash_table_(hash_table), output_destination_(DCHECK_NOTNULL(output_destination)), - storage_manager_(DCHECK_NOTNULL(storage_manager)), - lip_filter_adaptive_prober_(lip_filter_adaptive_prober) {} + storage_manager_(DCHECK_NOTNULL(storage_manager)) {} ~HashAntiJoinWorkOrder() override {} @@ -663,8 +646,6 @@ class HashAntiJoinWorkOrder : public WorkOrder { InsertDestination *output_destination_; StorageManager *storage_manager_; - std::unique_ptr<LIPFilterAdaptiveProber> lip_filter_adaptive_prober_; - DISALLOW_COPY_AND_ASSIGN(HashAntiJoinWorkOrder); }; http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/10eff2c1/relational_operators/RelationalOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/RelationalOperator.hpp b/relational_operators/RelationalOperator.hpp index c2c8f11..f0303e5 100644 --- a/relational_operators/RelationalOperator.hpp +++ b/relational_operators/RelationalOperator.hpp @@ -245,13 +245,6 @@ class RelationalOperator { return op_index_; } - /** - * @brief TODO - */ - void deployLIPFilter(const QueryContext::lip_deployment_id lip_deployment_index) { - lip_deployment_index_ = lip_deployment_index; - } - protected: /** * @brief Constructor @@ -264,8 +257,7 @@ class RelationalOperator { const bool blocking_dependencies_met = false) : query_id_(query_id), blocking_dependencies_met_(blocking_dependencies_met), - done_feeding_input_relation_(false), - lip_deployment_index_(QueryContext::kInvalidILIPDeploymentId) {} + done_feeding_input_relation_(false) {} const std::size_t query_id_; @@ -273,8 +265,6 @@ class RelationalOperator { bool done_feeding_input_relation_; std::size_t op_index_; - QueryContext::lip_deployment_id lip_deployment_index_; - private: DISALLOW_COPY_AND_ASSIGN(RelationalOperator); }; http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/10eff2c1/relational_operators/SelectOperator.cpp ---------------------------------------------------------------------- diff --git a/relational_operators/SelectOperator.cpp b/relational_operators/SelectOperator.cpp index d7855cf..d56326e 100644 --- a/relational_operators/SelectOperator.cpp +++ b/relational_operators/SelectOperator.cpp @@ -30,7 +30,6 @@ #include "storage/StorageBlock.hpp" #include "storage/StorageBlockInfo.hpp" #include "storage/StorageManager.hpp" -#include "utility/lip_filter/LIPFilterDeployment.hpp" #include "glog/logging.h" @@ -44,14 +43,9 @@ void SelectOperator::addWorkOrders(WorkOrdersContainer *container, StorageManager *storage_manager, const Predicate *predicate, const std::vector<std::unique_ptr<const Scalar>> *selection, - InsertDestination *output_destination, - const LIPFilterDeployment *lip_filter_deployment) { + InsertDestination *output_destination) { if (input_relation_is_stored_) { for (const block_id input_block_id : input_relation_block_ids_) { - LIPFilterAdaptiveProber *lip_filter_adaptive_prober = nullptr; - if (lip_filter_deployment != nullptr) { - lip_filter_adaptive_prober = lip_filter_deployment->createLIPFilterAdaptiveProber(); - } container->addNormalWorkOrder(new SelectWorkOrder(query_id_, input_relation_, input_block_id, @@ -60,16 +54,11 @@ void SelectOperator::addWorkOrders(WorkOrdersContainer *container, simple_selection_, selection, output_destination, - storage_manager, - lip_filter_adaptive_prober), + storage_manager), op_index_); } } else { while (num_workorders_generated_ < input_relation_block_ids_.size()) { - LIPFilterAdaptiveProber *lip_filter_adaptive_prober = nullptr; - if (lip_filter_deployment != nullptr) { - lip_filter_adaptive_prober = lip_filter_deployment->createLIPFilterAdaptiveProber(); - } container->addNormalWorkOrder( new SelectWorkOrder( query_id_, @@ -92,18 +81,13 @@ void SelectOperator::addPartitionAwareWorkOrders(WorkOrdersContainer *container, StorageManager *storage_manager, const Predicate *predicate, const std::vector<std::unique_ptr<const Scalar>> *selection, - InsertDestination *output_destination, - const LIPFilterDeployment *lip_filter_deployment) { + InsertDestination *output_destination) { DCHECK(placement_scheme_ != nullptr); const std::size_t num_partitions = input_relation_.getPartitionScheme().getPartitionSchemeHeader().getNumPartitions(); if (input_relation_is_stored_) { for (std::size_t part_id = 0; part_id < num_partitions; ++part_id) { for (const block_id input_block_id : input_relation_block_ids_in_partition_[part_id]) { - LIPFilterAdaptiveProber *lip_filter_adaptive_prober = nullptr; - if (lip_filter_deployment != nullptr) { - lip_filter_adaptive_prober = lip_filter_deployment->createLIPFilterAdaptiveProber(); - } container->addNormalWorkOrder( new SelectWorkOrder( query_id_, @@ -115,7 +99,6 @@ void SelectOperator::addPartitionAwareWorkOrders(WorkOrdersContainer *container, selection, output_destination, storage_manager, - lip_filter_adaptive_prober, placement_scheme_->getNUMANodeForBlock(input_block_id)), op_index_); } @@ -126,10 +109,6 @@ void SelectOperator::addPartitionAwareWorkOrders(WorkOrdersContainer *container, input_relation_block_ids_in_partition_[part_id].size()) { block_id block_in_partition = input_relation_block_ids_in_partition_[part_id][num_workorders_generated_in_partition_[part_id]]; - LIPFilterAdaptiveProber *lip_filter_adaptive_prober = nullptr; - if (lip_filter_deployment != nullptr) { - lip_filter_adaptive_prober = lip_filter_deployment->createLIPFilterAdaptiveProber(); - } container->addNormalWorkOrder( new SelectWorkOrder( query_id_, @@ -141,7 +120,6 @@ void SelectOperator::addPartitionAwareWorkOrders(WorkOrdersContainer *container, selection, output_destination, storage_manager, - lip_filter_adaptive_prober, placement_scheme_->getNUMANodeForBlock(block_in_partition)), op_index_); ++num_workorders_generated_in_partition_[part_id]; @@ -168,31 +146,16 @@ bool SelectOperator::getAllWorkOrders( InsertDestination *output_destination = query_context->getInsertDestination(output_destination_index_); - const LIPFilterDeployment *lip_filter_deployment = nullptr; - if (lip_deployment_index_ != QueryContext::kInvalidILIPDeploymentId) { - lip_filter_deployment = query_context->getLIPDeployment(lip_deployment_index_); - } - if (input_relation_is_stored_) { if (!started_) { if (input_relation_.hasPartitionScheme()) { #ifdef QUICKSTEP_HAVE_LIBNUMA if (input_relation_.hasNUMAPlacementScheme()) { - addPartitionAwareWorkOrders(container, - storage_manager, - predicate, - selection, - output_destination, - lip_filter_deployment); + addPartitionAwareWorkOrders(container, storage_manager, predicate, selection, output_destination); } #endif } else { - addWorkOrders(container, - storage_manager, - predicate, - selection, - output_destination, - lip_filter_deployment); + addWorkOrders(container, storage_manager, predicate, selection, output_destination); } started_ = true; } @@ -201,21 +164,11 @@ bool SelectOperator::getAllWorkOrders( if (input_relation_.hasPartitionScheme()) { #ifdef QUICKSTEP_HAVE_LIBNUMA if (input_relation_.hasNUMAPlacementScheme()) { - addPartitionAwareWorkOrders(container, - storage_manager, - predicate, - selection, - output_destination, - lip_filter_deployment); + addPartitionAwareWorkOrders(container, storage_manager, predicate, selection, output_destination); } #endif } else { - addWorkOrders(container, - storage_manager, - predicate, - selection, - output_destination, - lip_filter_deployment); + addWorkOrders(container, storage_manager, predicate, selection, output_destination); } return done_feeding_input_relation_; } @@ -269,13 +222,11 @@ void SelectWorkOrder::execute() { if (simple_projection_) { block->selectSimple(simple_selection_, predicate_, - output_destination_, - lip_filter_adaptive_prober_.get()); + output_destination_); } else { block->select(*DCHECK_NOTNULL(selection_), predicate_, - output_destination_, - lip_filter_adaptive_prober_.get()); + output_destination_); } } http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/10eff2c1/relational_operators/SelectOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/SelectOperator.hpp b/relational_operators/SelectOperator.hpp index 0d2ae16..0f5c712 100644 --- a/relational_operators/SelectOperator.hpp +++ b/relational_operators/SelectOperator.hpp @@ -38,7 +38,6 @@ #include "relational_operators/WorkOrder.hpp" #include "storage/StorageBlockInfo.hpp" #include "utility/Macros.hpp" -#include "utility/lip_filter/LIPFilterAdaptiveProber.hpp" #include "glog/logging.h" @@ -50,7 +49,6 @@ namespace quickstep { class CatalogRelationSchema; class InsertDestination; -class LIPFilterDeployment; class Predicate; class Scalar; class StorageManager; @@ -252,15 +250,13 @@ class SelectOperator : public RelationalOperator { StorageManager *storage_manager, const Predicate *predicate, const std::vector<std::unique_ptr<const Scalar>> *selection, - InsertDestination *output_destination, - const LIPFilterDeployment *lip_filter_deployment); + InsertDestination *output_destination); void addPartitionAwareWorkOrders(WorkOrdersContainer *container, StorageManager *storage_manager, const Predicate *predicate, const std::vector<std::unique_ptr<const Scalar>> *selection, - InsertDestination *output_destination, - const LIPFilterDeployment *lip_filter_deployment); + InsertDestination *output_destination); private: /** @@ -332,7 +328,6 @@ class SelectWorkOrder : public WorkOrder { const std::vector<std::unique_ptr<const Scalar>> *selection, InsertDestination *output_destination, StorageManager *storage_manager, - LIPFilterAdaptiveProber *lip_filter_adaptive_prober = nullptr, const numa_node_id numa_node = 0) : WorkOrder(query_id), input_relation_(input_relation), @@ -342,8 +337,7 @@ class SelectWorkOrder : public WorkOrder { simple_selection_(simple_selection), selection_(selection), output_destination_(DCHECK_NOTNULL(output_destination)), - storage_manager_(DCHECK_NOTNULL(storage_manager)), - lip_filter_adaptive_prober_(lip_filter_adaptive_prober) { + storage_manager_(DCHECK_NOTNULL(storage_manager)) { preferred_numa_nodes_.push_back(numa_node); } @@ -376,7 +370,6 @@ class SelectWorkOrder : public WorkOrder { const std::vector<std::unique_ptr<const Scalar>> *selection, InsertDestination *output_destination, StorageManager *storage_manager, - LIPFilterAdaptiveProber *lip_filter_adaptive_prober = nullptr, const numa_node_id numa_node = 0) : WorkOrder(query_id), input_relation_(input_relation), @@ -386,8 +379,7 @@ class SelectWorkOrder : public WorkOrder { simple_selection_(std::move(simple_selection)), selection_(selection), output_destination_(DCHECK_NOTNULL(output_destination)), - storage_manager_(DCHECK_NOTNULL(storage_manager)), - lip_filter_adaptive_prober_(lip_filter_adaptive_prober) { + storage_manager_(DCHECK_NOTNULL(storage_manager)) { preferred_numa_nodes_.push_back(numa_node); } @@ -415,8 +407,6 @@ class SelectWorkOrder : public WorkOrder { InsertDestination *output_destination_; StorageManager *storage_manager_; - std::unique_ptr<LIPFilterAdaptiveProber> lip_filter_adaptive_prober_; - DISALLOW_COPY_AND_ASSIGN(SelectWorkOrder); };
