Repository: incubator-quickstep Updated Branches: refs/heads/adaptive-bloom-filters 547cb39d9 -> 5ae557ae5 (forced update)
profiling Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/5ae557ae Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/5ae557ae Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/5ae557ae Branch: refs/heads/adaptive-bloom-filters Commit: 5ae557ae5f73f8206ac2a06288181f7fbd09a08b Parents: ed9cd43 Author: Jianqiao Zhu <jianq...@cs.wisc.edu> Authored: Wed Jun 15 23:16:41 2016 -0500 Committer: Jianqiao Zhu <jianq...@cs.wisc.edu> Committed: Fri Jun 24 23:16:10 2016 -0500 ---------------------------------------------------------------------- CMakeLists.txt | 1 + catalog/CMakeLists.txt | 9 + catalog/Catalog.proto | 5 + catalog/CatalogRelation.cpp | 16 +- catalog/CatalogRelationConstraints.cpp | 55 ++++++ catalog/CatalogRelationConstraints.hpp | 97 +++++++++++ catalog/CatalogRelationSchema.cpp | 15 ++ catalog/CatalogRelationSchema.hpp | 16 +- cli/CommandExecutor.cpp | 25 ++- cli/QuickstepCli.cpp | 42 ++++- query_execution/CMakeLists.txt | 1 + query_execution/Worker.cpp | 5 + query_execution/tests/QueryManager_unittest.cpp | 4 + query_optimizer/PhysicalGenerator.cpp | 2 +- .../StarSchemaHashJoinOrderOptimization.cpp | 1 + relational_operators/AggregationOperator.hpp | 4 + relational_operators/BuildHashOperator.hpp | 4 + relational_operators/CreateIndexOperator.hpp | 4 + relational_operators/CreateTableOperator.hpp | 4 + relational_operators/DeleteOperator.hpp | 4 + relational_operators/DestroyHashOperator.hpp | 4 + relational_operators/DropTableOperator.hpp | 4 + .../FinalizeAggregationOperator.hpp | 4 + relational_operators/HashJoinOperator.cpp | 46 ++++- relational_operators/HashJoinOperator.hpp | 35 +++- relational_operators/InsertOperator.hpp | 4 + .../NestedLoopsJoinOperator.hpp | 4 + relational_operators/RelationalOperator.hpp | 2 +- relational_operators/SampleOperator.hpp | 4 + relational_operators/SaveBlocksOperator.hpp | 4 + relational_operators/SelectOperator.hpp | 4 + relational_operators/SortMergeRunOperator.hpp | 4 + .../SortRunGenerationOperator.hpp | 4 + relational_operators/TableGeneratorOperator.hpp | 4 + relational_operators/TextScanOperator.hpp | 4 + relational_operators/UpdateOperator.hpp | 4 + relational_operators/WorkOrder.hpp | 11 +- utility/CMakeLists.txt | 7 + utility/DAGVisualizer.cpp | 167 +++++++++++++++++++ utility/DAGVisualizer.hpp | 85 ++++++++++ utility/EventProfiler.cpp | 3 +- utility/EventProfiler.hpp | 62 +++++-- 42 files changed, 731 insertions(+), 53 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae557ae/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/CMakeLists.txt b/CMakeLists.txt index b4728a1..ae85b75 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -760,6 +760,7 @@ target_link_libraries(quickstep_cli_shell quickstep_queryoptimizer_QueryProcessor quickstep_storage_PreloaderThread quickstep_threading_ThreadIDBasedMap + quickstep_utility_DAGVisualizer quickstep_utility_EventProfiler quickstep_utility_Macros quickstep_utility_PtrVector http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae557ae/catalog/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/catalog/CMakeLists.txt b/catalog/CMakeLists.txt index 64b4f16..0f50706 100644 --- a/catalog/CMakeLists.txt +++ b/catalog/CMakeLists.txt @@ -35,6 +35,9 @@ add_library(quickstep_catalog_CatalogDatabaseCache CatalogDatabaseCache.cpp Cata add_library(quickstep_catalog_CatalogDatabaseLite ../empty_src.cpp CatalogDatabaseLite.hpp) add_library(quickstep_catalog_CatalogErrors ../empty_src.cpp CatalogErrors.hpp) add_library(quickstep_catalog_CatalogRelation CatalogRelation.cpp CatalogRelation.hpp) +add_library(quickstep_catalog_CatalogRelationConstraints + CatalogRelationConstraints.cpp + CatalogRelationConstraints.hpp) add_library(quickstep_catalog_CatalogRelationSchema CatalogRelationSchema.cpp CatalogRelationSchema.hpp) @@ -117,6 +120,10 @@ target_link_libraries(quickstep_catalog_CatalogRelation quickstep_threading_SpinSharedMutex quickstep_utility_Macros quickstep_utility_PtrVector) +target_link_libraries(quickstep_catalog_CatalogRelationConstraints + quickstep_catalog_CatalogTypedefs + quickstep_catalog_Catalog_proto + quickstep_utility_Macros) target_link_libraries(quickstep_catalog_CatalogRelationStatistics quickstep_catalog_CatalogTypedefs quickstep_catalog_Catalog_proto @@ -136,6 +143,7 @@ target_link_libraries(quickstep_catalog_CatalogRelationSchema glog quickstep_catalog_CatalogAttribute quickstep_catalog_CatalogErrors + quickstep_catalog_CatalogRelationConstraints quickstep_catalog_CatalogTypedefs quickstep_catalog_Catalog_proto quickstep_types_Type @@ -182,6 +190,7 @@ target_link_libraries(quickstep_catalog quickstep_catalog_CatalogDatabaseLite quickstep_catalog_CatalogErrors quickstep_catalog_CatalogRelation + quickstep_catalog_CatalogRelationConstraints quickstep_catalog_CatalogRelationSchema quickstep_catalog_CatalogRelationStatistics quickstep_catalog_CatalogTypedefs http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae557ae/catalog/Catalog.proto ---------------------------------------------------------------------- diff --git a/catalog/Catalog.proto b/catalog/Catalog.proto index ce4bc2e..a51172f 100644 --- a/catalog/Catalog.proto +++ b/catalog/Catalog.proto @@ -80,6 +80,10 @@ message IndexScheme { repeated IndexEntry index_entries = 1; } +message CatalogRelationConstraints { + repeated int32 primary_key = 1; +} + message CatalogRelationStatistics { optional fixed64 num_tuples = 1; @@ -96,6 +100,7 @@ message CatalogRelationSchema { required bool temporary = 3; repeated CatalogAttribute attributes = 4; + optional CatalogRelationConstraints constraints = 5; extensions 16 to max; } http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae557ae/catalog/CatalogRelation.cpp ---------------------------------------------------------------------- diff --git a/catalog/CatalogRelation.cpp b/catalog/CatalogRelation.cpp index 01aebb5..682b6be 100644 --- a/catalog/CatalogRelation.cpp +++ b/catalog/CatalogRelation.cpp @@ -143,21 +143,7 @@ CatalogRelation::CatalogRelation(const serialization::CatalogRelationSchema &pro } serialization::CatalogRelationSchema CatalogRelation::getProto() const { - serialization::CatalogRelationSchema proto; - - proto.set_relation_id(id_); - proto.set_name(name_); - proto.set_temporary(temporary_); - - for (PtrVector<CatalogAttribute, true>::const_iterator it = attr_vec_.begin(); - it != attr_vec_.end(); - ++it) { - if (it.isNull()) { - proto.add_attributes(); - } else { - proto.add_attributes()->MergeFrom(it->getProto()); - } - } + serialization::CatalogRelationSchema proto = CatalogRelationSchema::getProto(); proto.MutableExtension(serialization::CatalogRelation::default_layout) ->MergeFrom(getDefaultStorageBlockLayout().getDescription()); http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae557ae/catalog/CatalogRelationConstraints.cpp ---------------------------------------------------------------------- diff --git a/catalog/CatalogRelationConstraints.cpp b/catalog/CatalogRelationConstraints.cpp new file mode 100644 index 0000000..4525a98 --- /dev/null +++ b/catalog/CatalogRelationConstraints.cpp @@ -0,0 +1,55 @@ +/** + * Copyright 2016, Quickstep Research Group, Computer Sciences Department, + * University of WisconsinâMadison. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + **/ + +#include "catalog/CatalogRelationConstraints.hpp" + +#include "catalog/Catalog.pb.h" + +namespace quickstep { + +CatalogRelationConstraints::CatalogRelationConstraints( + const serialization::CatalogRelationConstraints &proto) { + if (proto.primary_key_size() > 0) { + primary_key_.reset(new std::set<attribute_id>()); + for (std::size_t i = 0; i < proto.primary_key_size(); ++i) { + primary_key_->emplace(proto.primary_key(i)); + } + } +} + +serialization::CatalogRelationConstraints CatalogRelationConstraints::getProto() const { + serialization::CatalogRelationConstraints proto; + if (primary_key_ != nullptr) { + for (const auto attr_id : *primary_key_) { + proto.add_primary_key(attr_id); + } + } + return proto; +} + +bool CatalogRelationConstraints::ProtoIsValid( + const serialization::CatalogRelationConstraints &proto, + const std::size_t num_attributes) { + for (std::size_t i = 0; i < proto.primary_key_size(); ++i) { + if (proto.primary_key(i) >= num_attributes) { + return false; + } + } + return true; +} + +} // namespace quickstep http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae557ae/catalog/CatalogRelationConstraints.hpp ---------------------------------------------------------------------- diff --git a/catalog/CatalogRelationConstraints.hpp b/catalog/CatalogRelationConstraints.hpp new file mode 100644 index 0000000..135ccb9 --- /dev/null +++ b/catalog/CatalogRelationConstraints.hpp @@ -0,0 +1,97 @@ +/** + * Copyright 2016, Quickstep Research Group, Computer Sciences Department, + * University of WisconsinâMadison. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + **/ + +#ifndef QUICKSTEP_CATALOG_CATALOG_RELATION_CONSTRAINTS_HPP_ +#define QUICKSTEP_CATALOG_CATALOG_RELATION_CONSTRAINTS_HPP_ + +#include <cstddef> +#include <memory> +#include <set> +#include <utility> + +#include "catalog/Catalog.pb.h" +#include "catalog/CatalogTypedefs.hpp" +#include "utility/Macros.hpp" + +#include "glog/logging.h" + +namespace quickstep { + +/** \addtogroup Catalog + * @{ + */ + +/** + * @brief Constraints on a catalog relation. + **/ +class CatalogRelationConstraints { + public: + /** + * @brief Constructor. + **/ + CatalogRelationConstraints() {} + + /** + * @brief Reconstruct a CatalogRelationConstraints object from its serialized + * Protocol Buffer form. + * + * @param proto The Protocol Buffer serialization of a CatalogRelationConstraints + * object, previously produced by getProto(). + **/ + explicit CatalogRelationConstraints(const serialization::CatalogRelationConstraints &proto); + + /** + * @brief Serialize the CatalogRelationConstraints object as Protocol Buffer. + * + * @return The Protocol Buffer representation of the CatalogRelationConstraints + * object. + **/ + serialization::CatalogRelationConstraints getProto() const; + + static bool ProtoIsValid(const serialization::CatalogRelationConstraints &proto, + const std::size_t num_attributes); + + bool hasPrimaryKey() const { + return (primary_key_ != nullptr); + } + + const std::set<attribute_id>* getPrimaryKey() const { + return primary_key_.get(); + } + + template <typename IterableT> + void setPrimaryKey(IterableT &&primary_key) { + CHECK(!primary_key.empty()); + primary_key_.reset( + new std::set<attribute_id>(primary_key.begin(), primary_key.end())); + } + + void removePrimaryKey() { + primary_key_.reset(); + } + + private: + std::unique_ptr<std::set<attribute_id>> primary_key_; + + DISALLOW_COPY_AND_ASSIGN(CatalogRelationConstraints); +}; + +/** @} */ + +} // namespace quickstep + +#endif // QUICKSTEP_CATALOG_CATALOG_RELATION_CONSTRAINTS_HPP_ http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae557ae/catalog/CatalogRelationSchema.cpp ---------------------------------------------------------------------- diff --git a/catalog/CatalogRelationSchema.cpp b/catalog/CatalogRelationSchema.cpp index 97c834f..bf8217d 100644 --- a/catalog/CatalogRelationSchema.cpp +++ b/catalog/CatalogRelationSchema.cpp @@ -27,6 +27,7 @@ #include "catalog/Catalog.pb.h" #include "catalog/CatalogAttribute.hpp" #include "catalog/CatalogErrors.hpp" +#include "catalog/CatalogRelationConstraints.hpp" #include "catalog/CatalogTypedefs.hpp" #include "types/Type.hpp" #include "utility/PtrVector.hpp" @@ -70,6 +71,12 @@ CatalogRelationSchema::CatalogRelationSchema(const serialization::CatalogRelatio attr_vec_.push_back(nullptr); } } + + if (proto.has_constraints()) { + constraints_.reset(new CatalogRelationConstraints(proto.constraints())); + } else { + constraints_.reset(new CatalogRelationConstraints()); + } } bool CatalogRelationSchema::ProtoIsValid(const serialization::CatalogRelationSchema &proto) { @@ -84,6 +91,12 @@ bool CatalogRelationSchema::ProtoIsValid(const serialization::CatalogRelationSch } } + if (proto.has_constraints() + && !CatalogRelationConstraints::ProtoIsValid(proto.constraints(), + proto.attributes_size())) { + return false; + } + return true; } @@ -104,6 +117,8 @@ serialization::CatalogRelationSchema CatalogRelationSchema::getProto() const { } } + proto.mutable_constraints()->CopyFrom(constraints_->getProto()); + return proto; } http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae557ae/catalog/CatalogRelationSchema.hpp ---------------------------------------------------------------------- diff --git a/catalog/CatalogRelationSchema.hpp b/catalog/CatalogRelationSchema.hpp index d773bc7..0c6c207 100644 --- a/catalog/CatalogRelationSchema.hpp +++ b/catalog/CatalogRelationSchema.hpp @@ -21,12 +21,14 @@ #define QUICKSTEP_CATALOG_CATALOG_RELATION_SCHEMA_HPP_ #include <cstddef> +#include <memory> #include <string> #include <unordered_map> #include <vector> #include "catalog/Catalog.pb.h" #include "catalog/CatalogAttribute.hpp" +#include "catalog/CatalogRelationConstraints.hpp" #include "catalog/CatalogTypedefs.hpp" #include "utility/Macros.hpp" #include "utility/PtrVector.hpp" @@ -427,6 +429,14 @@ class CatalogRelationSchema { return max_byte_lengths_; } + const CatalogRelationConstraints& getConstraints() const { + return *constraints_; + } + + CatalogRelationConstraints* getConstraintsMutable() { + return constraints_.get(); + } + protected: /** * @brief Create a new relation. @@ -456,7 +466,8 @@ class CatalogRelationSchema { min_variable_byte_length_excluding_nullable_(0), estimated_variable_byte_length_(0), current_nullable_attribute_index_(-1), - current_variable_length_attribute_index_(-1) { + current_variable_length_attribute_index_(-1), + constraints_(new CatalogRelationConstraints()) { } /** @@ -532,6 +543,9 @@ class CatalogRelationSchema { std::vector<int> variable_length_attribute_indices_; int current_variable_length_attribute_index_; + // Primary key, foreign keys, etc. + std::unique_ptr<CatalogRelationConstraints> constraints_; + private: friend class CatalogDatabase; http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae557ae/cli/CommandExecutor.cpp ---------------------------------------------------------------------- diff --git a/cli/CommandExecutor.cpp b/cli/CommandExecutor.cpp index 7083ef5..bff8c11 100644 --- a/cli/CommandExecutor.cpp +++ b/cli/CommandExecutor.cpp @@ -252,7 +252,8 @@ inline TypedValue executeQueryForSingleResult( return value; } -void executeAnalyze(const tmb::client_id main_thread_client_id, +void executeAnalyze(const PtrVector<ParseString> *arguments, + const tmb::client_id main_thread_client_id, const tmb::client_id foreman_client_id, MessageBus *bus, QueryProcessor *query_processor, @@ -261,8 +262,19 @@ void executeAnalyze(const tmb::client_id main_thread_client_id, StorageManager *storage_manager = query_processor->getStorageManager(); std::unique_ptr<SqlParserWrapper> parser_wrapper(new SqlParserWrapper()); - std::vector<std::reference_wrapper<const CatalogRelation>> relations( - database.begin(), database.end()); + std::vector<std::reference_wrapper<const CatalogRelation>> relations; + if (arguments->size() == 0) { + relations.insert(relations.begin(), database.begin(), database.end()); + } else { + for (const auto &rel_name : *arguments) { + const CatalogRelation *rel = database.getRelationByName(rel_name.value()); + if (rel == nullptr) { + THROW_SQL_ERROR_AT(&rel_name) << "Table does not exist"; + } else { + relations.emplace_back(*rel); + } + } + } // Analyze each relation in the database. for (const CatalogRelation &relation : relations) { @@ -342,8 +354,11 @@ void executeCommand(const ParseStatement &statement, executeDescribeTable(arguments, catalog_database, out); } } else if (command_str == C::kAnalyzeCommand) { - executeAnalyze( - main_thread_client_id, foreman_client_id, bus, query_processor, out); + executeAnalyze(arguments, + main_thread_client_id, + foreman_client_id, + bus, + query_processor, out); } else { THROW_SQL_ERROR_AT(command.command()) << "Invalid Command"; } http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae557ae/cli/QuickstepCli.cpp ---------------------------------------------------------------------- diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp index 21965aa..937abf6 100644 --- a/cli/QuickstepCli.cpp +++ b/cli/QuickstepCli.cpp @@ -52,6 +52,9 @@ typedef quickstep::LineReaderDumb LineReaderImpl; #include <gperftools/profiler.h> #endif +#include "catalog/CatalogDatabase.hpp" +#include "catalog/CatalogRelation.hpp" +#include "catalog/CatalogRelationConstraints.hpp" #include "cli/DefaultsConfigurator.hpp" #include "cli/InputParserUtil.hpp" #include "cli/PrintToScreen.hpp" @@ -75,6 +78,7 @@ typedef quickstep::LineReaderDumb LineReaderImpl; #include "storage/PreloaderThread.hpp" #include "threading/ThreadIDBasedMap.hpp" +#include "utility/DAGVisualizer.hpp" #include "utility/EventProfiler.hpp" #include "utility/Macros.hpp" #include "utility/PtrVector.hpp" @@ -90,6 +94,8 @@ typedef quickstep::LineReaderDumb LineReaderImpl; #include "tmb/message_bus.h" #include "tmb/message_style.h" +#include "google/protobuf/text_format.h" + namespace quickstep { class CatalogRelation; } @@ -188,9 +194,31 @@ DEFINE_string(profile_file_name, "", // run at least a hundred times to make the impact of the first run small (< 5 %). DEFINE_string(profile_output, "", "Output file name for writing the profiled events."); +DEFINE_bool(visualize_dag, false, + "If true, visualize the execution plan DAG into a graph in DOT format."); } // namespace quickstep +void addPrimaryKeyInfoForTPCHTables(quickstep::CatalogDatabase *database) { + const std::vector<std::pair<std::string, std::vector<std::string>>> rel_pkeys = { + { "region", { "r_regionkey" } }, + { "nation", { "n_nationkey" } }, + { "supplier", { "s_suppkey" } }, + { "customer", { "c_custkey" } }, + { "part", { "p_partkey" } }, + { "partsupp", { "ps_partkey", "ps_suppkey" } }, + { "orders", { "o_orderkey" } } + }; + for (const auto &rel_pair : rel_pkeys) { + CatalogRelation *rel = database->getRelationByNameMutable(rel_pair.first); + std::vector<quickstep::attribute_id> attrs; + for (const auto &pkey : rel_pair.second) { + attrs.emplace_back(rel->getAttributeByName(pkey)->getID()); + } + rel->getConstraintsMutable()->setPrimaryKey(attrs); + } +} + int main(int argc, char* argv[]) { google::InitGoogleLogging(argv[0]); gflags::ParseCommandLineFlags(&argc, &argv, true); @@ -299,6 +327,12 @@ int main(int argc, char* argv[]) { LOG(FATAL) << "NON-STANDARD EXCEPTION DURING STARTUP"; } +// addPrimaryKeyInfoForTPCHTables(query_processor->getDefaultDatabase()); +// std::string proto_str; +// google::protobuf::TextFormat::PrintToString( +// query_processor->getDefaultDatabase()->getProto(), &proto_str); +// std::cerr << proto_str << "\n"; + // Parse the CPU affinities for workers and the preloader thread, if enabled // to warm up the buffer pool. const vector<int> worker_cpu_affinities = @@ -437,6 +471,8 @@ int main(int argc, char* argv[]) { } DCHECK(query_handle->getQueryPlanMutable() != nullptr); + quickstep::simple_profiler.clear(); + quickstep::relop_profiler.clear(); start = std::chrono::steady_clock::now(); QueryExecutionUtil::ConstructAndSendAdmitRequestMessage( main_thread_client_id, @@ -449,6 +485,11 @@ int main(int argc, char* argv[]) { main_thread_client_id, &bus); end = std::chrono::steady_clock::now(); + if (quickstep::FLAGS_visualize_dag) { + quickstep::DAGVisualizer visualizer(*query_handle->getQueryPlanMutable()); + std::cerr << "\n" << visualizer.toDOT() << "\n"; + } + const CatalogRelation *query_result_relation = query_handle->getQueryResultRelation(); if (query_result_relation) { PrintToScreen::PrintRelation(*query_result_relation, @@ -477,7 +518,6 @@ int main(int argc, char* argv[]) { if (!quickstep::FLAGS_profile_output.empty()) { std::ofstream ofs(quickstep::FLAGS_profile_output, std::ios::out); quickstep::simple_profiler.writeToStream(ofs); - quickstep::simple_profiler.clear(); ofs.close(); } } catch (const std::exception &e) { http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae557ae/query_execution/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/query_execution/CMakeLists.txt b/query_execution/CMakeLists.txt index b031a44..5facbb0 100644 --- a/query_execution/CMakeLists.txt +++ b/query_execution/CMakeLists.txt @@ -186,6 +186,7 @@ target_link_libraries(quickstep_queryexecution_Worker quickstep_threading_Thread quickstep_threading_ThreadIDBasedMap quickstep_threading_ThreadUtil + quickstep_utility_EventProfiler quickstep_utility_Macros tmb) target_link_libraries(quickstep_queryexecution_WorkerDirectory http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae557ae/query_execution/Worker.cpp ---------------------------------------------------------------------- diff --git a/query_execution/Worker.cpp b/query_execution/Worker.cpp index ae889c7..1a92e34 100644 --- a/query_execution/Worker.cpp +++ b/query_execution/Worker.cpp @@ -29,6 +29,7 @@ #include "relational_operators/WorkOrder.hpp" #include "threading/ThreadIDBasedMap.hpp" #include "threading/ThreadUtil.hpp" +#include "utility/EventProfiler.hpp" #include "glog/logging.h" @@ -116,8 +117,12 @@ void Worker::executeWorkOrderHelper(const TaggedMessage &tagged_message, const size_t query_id_for_workorder = worker_message.getWorkOrder()->getQueryID(); // Start measuring the execution time. + auto *container = relop_profiler.getContainer(); + auto *line = container->getEventLine(worker_message.getRelationalOpIndex()); start = std::chrono::steady_clock::now(); + line->emplace_back(); worker_message.getWorkOrder()->execute(); + line->back().endEvent(); end = std::chrono::steady_clock::now(); delete worker_message.getWorkOrder(); const uint64_t execution_time_microseconds = http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae557ae/query_execution/tests/QueryManager_unittest.cpp ---------------------------------------------------------------------- diff --git a/query_execution/tests/QueryManager_unittest.cpp b/query_execution/tests/QueryManager_unittest.cpp index 37e2cdd..dd3196b 100644 --- a/query_execution/tests/QueryManager_unittest.cpp +++ b/query_execution/tests/QueryManager_unittest.cpp @@ -105,6 +105,10 @@ class MockOperator: public RelationalOperator { num_calls_donefeedingblocks_(0) { } + std::string getName() const override { + return "MockOperator"; + } + #define MOCK_OP_LOG(x) VLOG(x) << "Op[" << op_index_ << "]: " << __func__ << ": " // The methods below are used to check whether QueryManager calls the Relational http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae557ae/query_optimizer/PhysicalGenerator.cpp ---------------------------------------------------------------------- diff --git a/query_optimizer/PhysicalGenerator.cpp b/query_optimizer/PhysicalGenerator.cpp index 75a7bc9..ee133b5 100644 --- a/query_optimizer/PhysicalGenerator.cpp +++ b/query_optimizer/PhysicalGenerator.cpp @@ -108,7 +108,7 @@ P::PhysicalPtr PhysicalGenerator::optimizePlan() { DVLOG(4) << "Optimized physical plan:\n" << physical_plan_->toString(); if (FLAGS_visualize_plan) { - quickstep::PlanVisualizer plan_visualizer; + quickstep::PlanVisualizer plan_visualizer; std::cerr << "\n" << plan_visualizer.visualize(physical_plan_) << "\n"; } http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae557ae/query_optimizer/rules/StarSchemaHashJoinOrderOptimization.cpp ---------------------------------------------------------------------- diff --git a/query_optimizer/rules/StarSchemaHashJoinOrderOptimization.cpp b/query_optimizer/rules/StarSchemaHashJoinOrderOptimization.cpp index 9770606..9357590 100644 --- a/query_optimizer/rules/StarSchemaHashJoinOrderOptimization.cpp +++ b/query_optimizer/rules/StarSchemaHashJoinOrderOptimization.cpp @@ -281,6 +281,7 @@ physical::PhysicalPtr StarSchemaHashJoinOrderOptimization::generatePlan( // TODO(jianqiao): Cache the estimated cardinality for each plan in cost // model to avoid duplicated estimation. second_table_info->estimated_cardinality = cost_model_->estimateCardinality(output); +// second_table_info->estimated_selectivity = cost_model_->estimateSelectivity(output); second_table_info->join_attribute_pairs.insert(first_table_info->join_attribute_pairs.begin(), first_table_info->join_attribute_pairs.end()); http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae557ae/relational_operators/AggregationOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/AggregationOperator.hpp b/relational_operators/AggregationOperator.hpp index 4bcbcf6..c46ba2c 100644 --- a/relational_operators/AggregationOperator.hpp +++ b/relational_operators/AggregationOperator.hpp @@ -77,6 +77,10 @@ class AggregationOperator : public RelationalOperator { ~AggregationOperator() override {} + std::string getName() const override { + return "AggregationOperator"; + } + bool getAllWorkOrders(WorkOrdersContainer *container, QueryContext *query_context, StorageManager *storage_manager, http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae557ae/relational_operators/BuildHashOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/BuildHashOperator.hpp b/relational_operators/BuildHashOperator.hpp index 464bbf8..952c7ac 100644 --- a/relational_operators/BuildHashOperator.hpp +++ b/relational_operators/BuildHashOperator.hpp @@ -93,6 +93,10 @@ class BuildHashOperator : public RelationalOperator { ~BuildHashOperator() override {} + std::string getName() const override { + return "BuildHashOperator"; + } + bool getAllWorkOrders(WorkOrdersContainer *container, QueryContext *query_context, StorageManager *storage_manager, http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae557ae/relational_operators/CreateIndexOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/CreateIndexOperator.hpp b/relational_operators/CreateIndexOperator.hpp index 18ca656..4e05448 100644 --- a/relational_operators/CreateIndexOperator.hpp +++ b/relational_operators/CreateIndexOperator.hpp @@ -69,6 +69,10 @@ class CreateIndexOperator : public RelationalOperator { ~CreateIndexOperator() override {} + std::string getName() const override { + return "CreateIndexOperator"; + } + /** * @note No WorkOrder generated for this operator. **/ http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae557ae/relational_operators/CreateTableOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/CreateTableOperator.hpp b/relational_operators/CreateTableOperator.hpp index 6d91142..b7b707b 100644 --- a/relational_operators/CreateTableOperator.hpp +++ b/relational_operators/CreateTableOperator.hpp @@ -66,6 +66,10 @@ class CreateTableOperator : public RelationalOperator { ~CreateTableOperator() override {} + std::string getName() const override { + return "CreateTableOperator"; + } + /** * @note No WorkOrder generated for this operator. **/ http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae557ae/relational_operators/DeleteOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/DeleteOperator.hpp b/relational_operators/DeleteOperator.hpp index 74da8c1..abfe4a9 100644 --- a/relational_operators/DeleteOperator.hpp +++ b/relational_operators/DeleteOperator.hpp @@ -81,6 +81,10 @@ class DeleteOperator : public RelationalOperator { ~DeleteOperator() override {} + std::string getName() const override { + return "DeleteOperator"; + } + bool getAllWorkOrders(WorkOrdersContainer *container, QueryContext *query_context, StorageManager *storage_manager, http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae557ae/relational_operators/DestroyHashOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/DestroyHashOperator.hpp b/relational_operators/DestroyHashOperator.hpp index 181386f..ae65de5 100644 --- a/relational_operators/DestroyHashOperator.hpp +++ b/relational_operators/DestroyHashOperator.hpp @@ -58,6 +58,10 @@ class DestroyHashOperator : public RelationalOperator { ~DestroyHashOperator() override {} + std::string getName() const override { + return "DestroyHashOperator"; + } + bool getAllWorkOrders(WorkOrdersContainer *container, QueryContext *query_context, StorageManager *storage_manager, http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae557ae/relational_operators/DropTableOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/DropTableOperator.hpp b/relational_operators/DropTableOperator.hpp index 6c7fca3..f854b4f 100644 --- a/relational_operators/DropTableOperator.hpp +++ b/relational_operators/DropTableOperator.hpp @@ -74,6 +74,10 @@ class DropTableOperator : public RelationalOperator { ~DropTableOperator() override {} + std::string getName() const override { + return "DropTableOperator"; + } + bool getAllWorkOrders(WorkOrdersContainer *container, QueryContext *query_context, StorageManager *storage_manager, http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae557ae/relational_operators/FinalizeAggregationOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/FinalizeAggregationOperator.hpp b/relational_operators/FinalizeAggregationOperator.hpp index 158a637..0dcfc9e 100644 --- a/relational_operators/FinalizeAggregationOperator.hpp +++ b/relational_operators/FinalizeAggregationOperator.hpp @@ -74,6 +74,10 @@ class FinalizeAggregationOperator : public RelationalOperator { ~FinalizeAggregationOperator() override {} + std::string getName() const override { + return "FinalizeAggregationOperator"; + } + bool getAllWorkOrders(WorkOrdersContainer *container, QueryContext *query_context, StorageManager *storage_manager, http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae557ae/relational_operators/HashJoinOperator.cpp ---------------------------------------------------------------------- diff --git a/relational_operators/HashJoinOperator.cpp b/relational_operators/HashJoinOperator.cpp index 5a47b50..2add678 100644 --- a/relational_operators/HashJoinOperator.cpp +++ b/relational_operators/HashJoinOperator.cpp @@ -48,6 +48,7 @@ #include "types/TypedValue.hpp" #include "types/containers/ColumnVector.hpp" #include "types/containers/ColumnVectorsValueAccessor.hpp" +#include "utility/EventProfiler.hpp" #include "gflags/gflags.h" #include "glog/logging.h" @@ -302,7 +303,8 @@ bool HashJoinOperator::getAllNonOuterJoinWorkOrders( selection, hash_table, output_destination, - storage_manager), + storage_manager, + static_cast<int>(op_index_)), op_index_); } started_ = true; @@ -322,7 +324,8 @@ bool HashJoinOperator::getAllNonOuterJoinWorkOrders( selection, hash_table, output_destination, - storage_manager), + storage_manager, + static_cast<int>(op_index_)), op_index_); ++num_workorders_generated_; } // end while @@ -525,10 +528,20 @@ void HashInnerJoinWorkOrder::execute() { template <typename CollectorT> void HashInnerJoinWorkOrder::executeWithCollectorType() { + auto *container = simple_profiler.getContainer(); +// auto *hash_join_line = container->getEventLine("hashJoin"); +// hash_join_line->emplace_back(); + +// auto *get_probe_block_line = container->getEventLine("getProbeBlock"); +// get_probe_block_line->emplace_back(); BlockReference probe_block( storage_manager_->getBlock(block_id_, probe_relation_)); const TupleStorageSubBlock &probe_store = probe_block->getTupleStorageSubBlock(); +// get_probe_block_line->back().endEvent(); +// get_probe_block_line->back().setPayload(getOperatorIndex()); +// auto *collect_matches_line = container->getEventLine("collectMatches"); +// collect_matches_line->emplace_back(); std::unique_ptr<ValueAccessor> probe_accessor(probe_store.createValueAccessor()); CollectorT collector; if (join_key_attributes_.size() == 1) { @@ -545,16 +558,30 @@ void HashInnerJoinWorkOrder::executeWithCollectorType() { &collector); } collector.consolidate(); +// collect_matches_line->back().endEvent(); +// collect_matches_line->back().setPayload(getOperatorIndex()); const relation_id build_relation_id = build_relation_.getID(); const relation_id probe_relation_id = probe_relation_.getID(); + auto *materialize_line = container->getEventLine("materialize"); + materialize_line->emplace_back(); + auto *get_build_block_line = container->getEventLine("getBuildBlock"); + auto *create_temp_line = container->getEventLine("createTempResults"); + auto *bulk_insert_line = container->getEventLine("bulkInsertTuples"); + auto *pair_iteration_line = container->getEventLine("pairIteration"); + pair_iteration_line->emplace_back(); for (std::pair<const block_id, std::vector<std::pair<tuple_id, tuple_id>>> &build_block_entry : *collector.getJoinedTuples()) { + pair_iteration_line->back().setPayload(getOperatorIndex()); + pair_iteration_line->back().endEvent(); + get_build_block_line->emplace_back(); BlockReference build_block = storage_manager_->getBlock(build_block_entry.first, build_relation_); const TupleStorageSubBlock &build_store = build_block->getTupleStorageSubBlock(); std::unique_ptr<ValueAccessor> build_accessor(build_store.createValueAccessor()); + get_build_block_line->back().setPayload(getOperatorIndex()); + get_build_block_line->back().endEvent(); // Evaluate '*residual_predicate_', if any. // @@ -600,6 +627,7 @@ void HashInnerJoinWorkOrder::executeWithCollectorType() { // benefit (probably only a real performance win when there are very few // matching tuples in each individual inner block but very many inner // blocks with at least one match). + create_temp_line->emplace_back(); ColumnVectorsValueAccessor temp_result; for (vector<unique_ptr<const Scalar>>::const_iterator selection_cit = selection_.begin(); selection_cit != selection_.end(); @@ -610,13 +638,25 @@ void HashInnerJoinWorkOrder::executeWithCollectorType() { probe_accessor.get(), build_block_entry.second)); } + create_temp_line->back().endEvent(); + create_temp_line->back().setPayload(getOperatorIndex()); // NOTE(chasseur): calling the bulk-insert method of InsertDestination once // for each pair of joined blocks incurs some extra overhead that could be // avoided by keeping checked-out MutableBlockReferences across iterations // of this loop, but that would get messy when combined with partitioning. + bulk_insert_line->emplace_back(); output_destination_->bulkInsertTuples(&temp_result); - } + bulk_insert_line->back().endEvent(); + bulk_insert_line->back().setPayload(getOperatorIndex()); + pair_iteration_line->emplace_back(); + } + pair_iteration_line->back().setPayload(getOperatorIndex()); + pair_iteration_line->back().endEvent(); + materialize_line->back().endEvent(); + materialize_line->back().setPayload(getOperatorIndex()); +// hash_join_line->back().endEvent(); +// hash_join_line->back().setPayload(getOperatorIndex()); } void HashSemiJoinWorkOrder::execute() { http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae557ae/relational_operators/HashJoinOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/HashJoinOperator.hpp b/relational_operators/HashJoinOperator.hpp index 9762f04..4f6c3d6 100644 --- a/relational_operators/HashJoinOperator.hpp +++ b/relational_operators/HashJoinOperator.hpp @@ -157,6 +157,21 @@ class HashJoinOperator : public RelationalOperator { ~HashJoinOperator() override {} + std::string getName() const override { + switch (join_type_) { + case JoinType::kInnerJoin: + return "HashJoinOperator"; + case JoinType::kLeftSemiJoin: + return "HashJoinOperator(LeftSemi)"; + case JoinType::kLeftAntiJoin: + return "HashJoinOperator(LeftAnti)"; + case JoinType::kLeftOuterJoin: + return "HashJoinOperator(LeftOuter)"; + default: break; + } + LOG(FATAL) << "Unknown join type in HashJoinOperator::getName()"; + } + bool getAllWorkOrders(WorkOrdersContainer *container, QueryContext *query_context, StorageManager *storage_manager, @@ -283,8 +298,9 @@ class HashInnerJoinWorkOrder : public WorkOrder { const std::vector<std::unique_ptr<const Scalar>> &selection, const JoinHashTable &hash_table, InsertDestination *output_destination, - StorageManager *storage_manager) - : WorkOrder(query_id), + StorageManager *storage_manager, + const int op_index = -1) + : WorkOrder(query_id, op_index), build_relation_(build_relation), probe_relation_(probe_relation), join_key_attributes_(join_key_attributes), @@ -414,8 +430,9 @@ class HashSemiJoinWorkOrder : public WorkOrder { const std::vector<std::unique_ptr<const Scalar>> &selection, const JoinHashTable &hash_table, InsertDestination *output_destination, - StorageManager *storage_manager) - : WorkOrder(query_id), + StorageManager *storage_manager, + const int op_index = -1) + : WorkOrder(query_id, op_index), build_relation_(build_relation), probe_relation_(probe_relation), join_key_attributes_(join_key_attributes), @@ -461,8 +478,9 @@ class HashSemiJoinWorkOrder : public WorkOrder { const std::vector<std::unique_ptr<const Scalar>> &selection, const JoinHashTable &hash_table, InsertDestination *output_destination, - StorageManager *storage_manager) - : WorkOrder(query_id), + StorageManager *storage_manager, + const int op_index = -1) + : WorkOrder(query_id, op_index), build_relation_(build_relation), probe_relation_(probe_relation), join_key_attributes_(std::move(join_key_attributes)), @@ -538,8 +556,9 @@ class HashAntiJoinWorkOrder : public WorkOrder { const std::vector<std::unique_ptr<const Scalar>> &selection, const JoinHashTable &hash_table, InsertDestination *output_destination, - StorageManager *storage_manager) - : WorkOrder(query_id), + StorageManager *storage_manager, + const int op_index = -1) + : WorkOrder(query_id, op_index), build_relation_(build_relation), probe_relation_(probe_relation), join_key_attributes_(join_key_attributes), http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae557ae/relational_operators/InsertOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/InsertOperator.hpp b/relational_operators/InsertOperator.hpp index 78f5199..2c6aca7 100644 --- a/relational_operators/InsertOperator.hpp +++ b/relational_operators/InsertOperator.hpp @@ -73,6 +73,10 @@ class InsertOperator : public RelationalOperator { ~InsertOperator() override {} + std::string getName() const override { + return "InsertOperator"; + } + bool getAllWorkOrders(WorkOrdersContainer *container, QueryContext *query_context, StorageManager *storage_manager, http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae557ae/relational_operators/NestedLoopsJoinOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/NestedLoopsJoinOperator.hpp b/relational_operators/NestedLoopsJoinOperator.hpp index 992e76d..cf190fe 100644 --- a/relational_operators/NestedLoopsJoinOperator.hpp +++ b/relational_operators/NestedLoopsJoinOperator.hpp @@ -116,6 +116,10 @@ class NestedLoopsJoinOperator : public RelationalOperator { ~NestedLoopsJoinOperator() override {} + std::string getName() const override { + return "NestedLoopsJoinOperator"; + } + bool getAllWorkOrders(WorkOrdersContainer *container, QueryContext *query_context, StorageManager *storage_manager, http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae557ae/relational_operators/RelationalOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/RelationalOperator.hpp b/relational_operators/RelationalOperator.hpp index 852006b..65cd213 100644 --- a/relational_operators/RelationalOperator.hpp +++ b/relational_operators/RelationalOperator.hpp @@ -59,7 +59,7 @@ class RelationalOperator { * * @return The name of this relational operator. */ - virtual std::string getName() = 0; + virtual std::string getName() const = 0; /** * @brief Generate all the next WorkOrders for this RelationalOperator. http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae557ae/relational_operators/SampleOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/SampleOperator.hpp b/relational_operators/SampleOperator.hpp index f8fe5f6..08f08c8 100644 --- a/relational_operators/SampleOperator.hpp +++ b/relational_operators/SampleOperator.hpp @@ -93,6 +93,10 @@ class SampleOperator : public RelationalOperator { ~SampleOperator() override {} + std::string getName() const override { + return "SampleOperator"; + } + bool getAllWorkOrders(WorkOrdersContainer *container, QueryContext *query_context, StorageManager *storage_manager, http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae557ae/relational_operators/SaveBlocksOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/SaveBlocksOperator.hpp b/relational_operators/SaveBlocksOperator.hpp index 50032b6..ebc5ffc 100644 --- a/relational_operators/SaveBlocksOperator.hpp +++ b/relational_operators/SaveBlocksOperator.hpp @@ -64,6 +64,10 @@ class SaveBlocksOperator : public RelationalOperator { ~SaveBlocksOperator() override {} + std::string getName() const override { + return "SaveBlocksOperator"; + } + bool getAllWorkOrders(WorkOrdersContainer *container, QueryContext *query_context, StorageManager *storage_manager, http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae557ae/relational_operators/SelectOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/SelectOperator.hpp b/relational_operators/SelectOperator.hpp index 0c10686..ee25886 100644 --- a/relational_operators/SelectOperator.hpp +++ b/relational_operators/SelectOperator.hpp @@ -189,6 +189,10 @@ class SelectOperator : public RelationalOperator { ~SelectOperator() override {} + std::string getName() const override { + return "SelectOperator"; + } + bool getAllWorkOrders(WorkOrdersContainer *container, QueryContext *query_context, StorageManager *storage_manager, http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae557ae/relational_operators/SortMergeRunOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/SortMergeRunOperator.hpp b/relational_operators/SortMergeRunOperator.hpp index 177836f..9b07ad6 100644 --- a/relational_operators/SortMergeRunOperator.hpp +++ b/relational_operators/SortMergeRunOperator.hpp @@ -129,6 +129,10 @@ class SortMergeRunOperator : public RelationalOperator { **/ ~SortMergeRunOperator() {} + std::string getName() const override { + return "SortMergeRunOperator"; + } + bool getAllWorkOrders(WorkOrdersContainer *container, QueryContext *query_context, StorageManager *storage_manager, http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae557ae/relational_operators/SortRunGenerationOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/SortRunGenerationOperator.hpp b/relational_operators/SortRunGenerationOperator.hpp index 96a3ce1..54c7feb 100644 --- a/relational_operators/SortRunGenerationOperator.hpp +++ b/relational_operators/SortRunGenerationOperator.hpp @@ -109,6 +109,10 @@ class SortRunGenerationOperator : public RelationalOperator { ~SortRunGenerationOperator() {} + std::string getName() const override { + return "SortRunGenerationOperator"; + } + bool getAllWorkOrders(WorkOrdersContainer *container, QueryContext *query_context, StorageManager *storage_manager, http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae557ae/relational_operators/TableGeneratorOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/TableGeneratorOperator.hpp b/relational_operators/TableGeneratorOperator.hpp index 1b791a6..15e7052 100644 --- a/relational_operators/TableGeneratorOperator.hpp +++ b/relational_operators/TableGeneratorOperator.hpp @@ -76,6 +76,10 @@ class TableGeneratorOperator : public RelationalOperator { ~TableGeneratorOperator() override {} + std::string getName() const override { + return "TableGeneratorOperator"; + } + bool getAllWorkOrders(WorkOrdersContainer *container, QueryContext *query_context, StorageManager *storage_manager, http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae557ae/relational_operators/TextScanOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/TextScanOperator.hpp b/relational_operators/TextScanOperator.hpp index 1a62ded..6890d7d 100644 --- a/relational_operators/TextScanOperator.hpp +++ b/relational_operators/TextScanOperator.hpp @@ -134,6 +134,10 @@ class TextScanOperator : public RelationalOperator { ~TextScanOperator() override {} + std::string getName() const override { + return "TextScanOperator"; + } + bool getAllWorkOrders(WorkOrdersContainer *container, QueryContext *query_context, StorageManager *storage_manager, http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae557ae/relational_operators/UpdateOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/UpdateOperator.hpp b/relational_operators/UpdateOperator.hpp index 4471a17..d021844 100644 --- a/relational_operators/UpdateOperator.hpp +++ b/relational_operators/UpdateOperator.hpp @@ -94,6 +94,10 @@ class UpdateOperator : public RelationalOperator { ~UpdateOperator() override {} + std::string getName() const override { + return "UpdateOperator"; + } + bool getAllWorkOrders(WorkOrdersContainer *container, QueryContext *query_context, StorageManager *storage_manager, http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae557ae/relational_operators/WorkOrder.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/WorkOrder.hpp b/relational_operators/WorkOrder.hpp index df195cc..4eb6b3a 100644 --- a/relational_operators/WorkOrder.hpp +++ b/relational_operators/WorkOrder.hpp @@ -299,16 +299,23 @@ class WorkOrder { return query_id_; } + inline const int getOperatorIndex() const { + return op_index_; + } + protected: /** * @brief Constructor. * * @param query_id The ID of the query to which this WorkOrder belongs. **/ - explicit WorkOrder(const std::size_t query_id) - : query_id_(query_id) {} + explicit WorkOrder(const std::size_t query_id, + const int op_index = -1) + : query_id_(query_id), + op_index_(op_index) {} const std::size_t query_id_; + const int op_index_; // A vector of preferred NUMA node IDs where this workorder should be executed. // These node IDs typically indicate the NUMA node IDs of the input(s) of the // workorder. Derived classes should ensure that there are no duplicate entries http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae557ae/utility/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/utility/CMakeLists.txt b/utility/CMakeLists.txt index de0e737..133e2f3 100644 --- a/utility/CMakeLists.txt +++ b/utility/CMakeLists.txt @@ -167,6 +167,7 @@ add_library(quickstep_utility_CalculateInstalledMemory CalculateInstalledMemory. add_library(quickstep_utility_Cast ../empty_src.cpp Cast.hpp) add_library(quickstep_utility_CheckSnprintf ../empty_src.cpp CheckSnprintf.hpp) add_library(quickstep_utility_DAG ../empty_src.cpp DAG.hpp) +add_library(quickstep_utility_DAGVisualizer DAGVisualizer.cpp DAGVisualizer.hpp) add_library(quickstep_utility_EventProfiler EventProfiler.cpp EventProfiler.hpp) add_library(quickstep_utility_EqualsAnyConstant ../empty_src.cpp EqualsAnyConstant.hpp) add_library(quickstep_utility_Glob Glob.cpp Glob.hpp) @@ -251,6 +252,11 @@ target_link_libraries(quickstep_utility_PlanVisualizer quickstep_queryoptimizer_physical_TopLevelPlan quickstep_utility_Macros quickstep_utility_StringUtil) +target_link_libraries(quickstep_utility_DAGVisualizer + quickstep_queryoptimizer_QueryPlan + quickstep_utility_EventProfiler + quickstep_utility_Macros + quickstep_utility_StringUtil) target_link_libraries(quickstep_utility_PtrList quickstep_utility_Macros) target_link_libraries(quickstep_utility_PtrMap @@ -311,6 +317,7 @@ target_link_libraries(quickstep_utility quickstep_utility_Cast quickstep_utility_CheckSnprintf quickstep_utility_DAG + quickstep_utility_DAGVisualizer quickstep_utility_EventProfiler quickstep_utility_EqualsAnyConstant quickstep_utility_Glob http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae557ae/utility/DAGVisualizer.cpp ---------------------------------------------------------------------- diff --git a/utility/DAGVisualizer.cpp b/utility/DAGVisualizer.cpp new file mode 100644 index 0000000..e62f948 --- /dev/null +++ b/utility/DAGVisualizer.cpp @@ -0,0 +1,167 @@ +/** + * Copyright 2016, Quickstep Research Group, Computer Sciences Department, + * University of WisconsinâMadison. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + **/ + +#include "utility/DAGVisualizer.hpp" +#include "utility/EventProfiler.hpp" + +#include <cmath> +#include <cstddef> +#include <iomanip> +#include <memory> +#include <sstream> +#include <string> +#include <unordered_map> +#include <vector> + +#include "query_optimizer/QueryPlan.hpp" +#include "utility/EventProfiler.hpp" +#include "utility/StringUtil.hpp" + +#include "glog/logging.h" + +namespace quickstep { + +std::string DAGVisualizer::toDOT() { + std::set<std::string> no_display_op_names = + { "DestroyHashOperator", "DropTableOperator" }; + + const auto &dag = plan_.getQueryPlanDAG(); + const std::size_t num_nodes = dag.size(); + + std::vector<double> time_elapsed(num_nodes, 0); + std::vector<double> time_percentage(num_nodes, 0); + std::vector<double> time_start(num_nodes, std::numeric_limits<double>::max()); + std::vector<double> time_end(num_nodes, 0); + const auto &zero_time = relop_profiler.zero_time(); + for (const auto &container : relop_profiler.containers()) { + for (const auto &line : container.second.events) { + const std::size_t relop_index = line.first; + for (const auto &event : line.second) { + time_elapsed[relop_index] += + std::chrono::duration<double>(event.end_time - event.start_time).count(); + time_start[relop_index] = + std::min(time_start[relop_index], + std::chrono::duration<double>(event.start_time - zero_time).count()); + time_end[relop_index] = + std::max(time_end[relop_index], + std::chrono::duration<double>(event.end_time - zero_time).count()); + } + } + } + const std::size_t num_threads = relop_profiler.containers().size(); + double total_time_elapsed = 0; + double max_percentage = 0; + for (std::size_t i = 0; i < time_elapsed.size(); ++i) { + time_elapsed[i] /= num_threads; + total_time_elapsed += time_elapsed[i]; + } + for (std::size_t i = 0; i < time_elapsed.size(); ++i) { + time_percentage[i] = time_elapsed[i] / total_time_elapsed; + max_percentage = std::max(max_percentage, time_percentage[i]); + } + + std::vector<bool> display_ops(num_nodes, false); + for (std::size_t node_index = 0; node_index < num_nodes; ++node_index) { + const auto &node = dag.getNodePayload(node_index); + const std::string relop_name = node.getName(); + if (no_display_op_names.find(relop_name) == no_display_op_names.end()) { + display_ops[node_index] = true; + + nodes_.emplace_back(); + NodeInfo &node_info = nodes_.back(); + node_info.id = node_index; + + std::string hue = + std::to_string(std::sqrt(time_percentage[node_index] / max_percentage)); + node_info.color = hue + " " + hue + " 1.0"; + + node_info.labels.emplace_back( + "[" + std::to_string(node.getOperatorIndex()) + "] " + relop_name); + node_info.labels.emplace_back( + std::to_string(std::lround(time_elapsed[node_index] * 1000)) + + "ms (" + PercentageToString(time_percentage[node_index] * 100) + "%)"); + node_info.labels.emplace_back( + "span: [" + + std::to_string(std::lround(time_start[node_index] * 1000)) + "ms, " + + std::to_string(std::lround(time_end[node_index] * 1000)) + "ms]"); + } + } + for (std::size_t node_index = 0; node_index < num_nodes; ++node_index) { + if (display_ops[node_index]) { + for (const auto &link : dag.getDependents(node_index)) { + if (display_ops[link.first]) { + edges_.emplace_back(); + EdgeInfo &edge_info = edges_.back(); + edge_info.src_node_id = node_index; + edge_info.dst_node_id = link.first; + edge_info.is_pipeline_breaker = link.second; + } + } + } + } + + // Format output graph + std::ostringstream graph_oss; + graph_oss << "digraph g {\n"; + graph_oss << " rankdir=BT\n"; + graph_oss << " node [penwidth=2]\n"; + graph_oss << " edge [fontsize=16 fontcolor=gray penwidth=2]\n\n"; + + // Format nodes + for (const NodeInfo &node_info : nodes_) { + graph_oss << " " << node_info.id << " [ "; + if (!node_info.labels.empty()) { + graph_oss << "label=\"" + << EscapeSpecialChars(JoinToString(node_info.labels, " ")) + << "\" "; + } + if (!node_info.color.empty()) { + graph_oss << "style=filled fillcolor=\"" << node_info.color << "\" "; + } + graph_oss << "]\n"; + } + graph_oss << "\n"; + + // Format edges + for (const EdgeInfo &edge_info : edges_) { + graph_oss << " " << edge_info.src_node_id << " -> " + << edge_info.dst_node_id << " [ "; + if (edge_info.is_pipeline_breaker) { + graph_oss << "style=dashed "; + } + if (!edge_info.labels.empty()) { + graph_oss << "label=\"" + << EscapeSpecialChars(JoinToString(edge_info.labels, " ")) + << "\" "; + } + graph_oss << "]\n"; + } + + graph_oss << "}\n"; + + return graph_oss.str(); +} + +std::string DAGVisualizer::PercentageToString(double percentage) { + std::ostringstream oss; + oss << static_cast<std::uint32_t>(percentage) << "."; + int digits = std::lround(percentage * 10000) % 100; + oss << digits / 10 << digits % 10; + return oss.str(); +} + +} // namespace quickstep http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae557ae/utility/DAGVisualizer.hpp ---------------------------------------------------------------------- diff --git a/utility/DAGVisualizer.hpp b/utility/DAGVisualizer.hpp new file mode 100644 index 0000000..5c81d22 --- /dev/null +++ b/utility/DAGVisualizer.hpp @@ -0,0 +1,85 @@ +/** + * Copyright 2016, Quickstep Research Group, Computer Sciences Department, + * University of WisconsinâMadison. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + **/ + +#ifndef QUICKSTEP_UTILITY_DAG_VISUALIZER_HPP_ +#define QUICKSTEP_UTILITY_DAG_VISUALIZER_HPP_ + +#include <memory> +#include <string> +#include <unordered_map> +#include <vector> + +#include "utility/Macros.hpp" + +namespace quickstep { + +class QueryPlan; + +/** \addtogroup Utility + * @{ + */ + +/** + * @brief A visualizer that converts an execution plan DAG into a graph in + * DOT format. Note that DOT is a plain text graph description language. + * + * @note This utility tool can be further extended to be more generic. + */ +class DAGVisualizer { + public: + DAGVisualizer(const QueryPlan &plan) + : plan_(plan) {} + + ~DAGVisualizer() {} + + std::string toDOT(); + + private: + static std::string PercentageToString(double percentage); + + /** + * @brief Information of a graph node. + */ + struct NodeInfo { + std::size_t id; + std::vector<std::string> labels; + std::string color; + }; + + /** + * @brief Information of a graph edge. + */ + struct EdgeInfo { + std::size_t src_node_id; + std::size_t dst_node_id; + std::vector<std::string> labels; + bool is_pipeline_breaker; + }; + + const QueryPlan &plan_; + + std::vector<NodeInfo> nodes_; + std::vector<EdgeInfo> edges_; + + DISALLOW_COPY_AND_ASSIGN(DAGVisualizer); +}; + +/** @} */ + +} // namespace quickstep + +#endif /* QUICKSTEP_UTILITY_DAG_VISUALIZER_HPP_ */ http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae557ae/utility/EventProfiler.cpp ---------------------------------------------------------------------- diff --git a/utility/EventProfiler.cpp b/utility/EventProfiler.cpp index 794c67d..cac6737 100644 --- a/utility/EventProfiler.cpp +++ b/utility/EventProfiler.cpp @@ -23,6 +23,7 @@ namespace quickstep { -EventProfiler simple_profiler; +EventProfiler<std::string, int> simple_profiler; +EventProfiler<std::size_t> relop_profiler; } // namespace quickstep http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/5ae557ae/utility/EventProfiler.hpp ---------------------------------------------------------------------- diff --git a/utility/EventProfiler.hpp b/utility/EventProfiler.hpp index 9c842a7..ca91593 100644 --- a/utility/EventProfiler.hpp +++ b/utility/EventProfiler.hpp @@ -42,18 +42,19 @@ namespace quickstep { using clock = std::chrono::steady_clock; +template <typename TagT, typename ...PayloadT> class EventProfiler { public: EventProfiler() - : zero_time(clock::now()) { + : zero_time_(clock::now()) { } struct EventInfo { clock::time_point start_time; clock::time_point end_time; bool is_finished; - std::size_t payload; + std::tuple<PayloadT...> payload; explicit EventInfo(const clock::time_point &start_time_in) : start_time(start_time_in), @@ -65,8 +66,8 @@ class EventProfiler { is_finished(false) { } - inline void setPayload(const std::size_t &in_payload) { - payload = in_payload; + inline void setPayload(PayloadT &&...in_payload) { + payload = std::make_tuple(in_payload...); } inline void endEvent() { @@ -76,21 +77,21 @@ class EventProfiler { }; struct EventContainer { - inline void startEvent(const std::string &tag) { + inline void startEvent(const TagT &tag) { events[tag].emplace_back(clock::now()); } - inline void endEvent(const std::string &tag) { + inline void endEvent(const TagT &tag) { auto &event_info = events.at(tag).back(); event_info.is_finished = true; event_info.end_time = clock::now(); } - inline std::vector<EventInfo> *getEventLine(const std::string &tag) { + inline std::vector<EventInfo> *getEventLine(const TagT &tag) { return &events[tag]; } - std::map<std::string, std::vector<EventInfo>> events; + std::map<TagT, std::vector<EventInfo>> events; }; EventContainer *getContainer() { @@ -112,11 +113,13 @@ class EventProfiler { os << std::setprecision(12) << event_id << "," - << thread_id << "," << event_group.first << "," - << event_info.payload << "," - << std::chrono::duration<double>(event_info.start_time - zero_time).count() + << thread_id << "," << event_group.first << ","; + + PrintTuple(os, event_info.payload, ","); + + os << std::chrono::duration<double>(event_info.start_time - zero_time_).count() << "," - << std::chrono::duration<double>(event_info.end_time - zero_time).count() + << std::chrono::duration<double>(event_info.end_time - zero_time_).count() << "\n"; } } @@ -125,17 +128,46 @@ class EventProfiler { } void clear() { - zero_time = clock::now(); + zero_time_ = clock::now(); thread_map_.clear(); } + const std::map<std::thread::id, EventContainer> &containers() { + return thread_map_; + } + + const clock::time_point &zero_time() { + return zero_time_; + } + private: - clock::time_point zero_time; + template<class Tuple, std::size_t N> + struct TuplePrinter { + static void Print(std::ostream &os, const Tuple &t, const std::string &sep) { + TuplePrinter<Tuple, N-1>::Print(os, t, sep); + os << std::get<N-1>(t) << sep; + } + }; + + template<class Tuple> + struct TuplePrinter<Tuple, 1> { + static void Print(std::ostream &os, const Tuple &t, const std::string &sep) { + os << std::get<0>(t) << sep; + } + }; + + template<class... Args> + static void PrintTuple(std::ostream &os, const std::tuple<Args...>& t, const std::string &sep) { + TuplePrinter<decltype(t), sizeof...(Args)>::Print(os, t, sep); + } + + clock::time_point zero_time_; std::map<std::thread::id, EventContainer> thread_map_; Mutex mutex_; }; -extern EventProfiler simple_profiler; +extern EventProfiler<std::string, int> simple_profiler; +extern EventProfiler<std::size_t> relop_profiler; /** @} */