profiling
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/871ccac9 Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/871ccac9 Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/871ccac9 Branch: refs/heads/adaptive-bloom-filters Commit: 871ccac9e934afc66052c121ce683951cf0db923 Parents: b3f380f Author: Jianqiao Zhu <jianq...@cs.wisc.edu> Authored: Wed Jun 15 23:16:41 2016 -0500 Committer: Jianqiao Zhu <jianq...@cs.wisc.edu> Committed: Wed Jun 15 23:16:41 2016 -0500 ---------------------------------------------------------------------- CMakeLists.txt | 1 + cli/QuickstepCli.cpp | 9 ++ query_execution/CMakeLists.txt | 1 + query_execution/Worker.cpp | 6 + query_execution/tests/QueryManager_unittest.cpp | 4 + relational_operators/AggregationOperator.hpp | 4 + relational_operators/BuildHashOperator.hpp | 4 + relational_operators/CreateIndexOperator.hpp | 4 + relational_operators/CreateTableOperator.hpp | 4 + relational_operators/DeleteOperator.hpp | 4 + relational_operators/DestroyHashOperator.hpp | 4 + relational_operators/DropTableOperator.hpp | 4 + .../FinalizeAggregationOperator.hpp | 4 + relational_operators/HashJoinOperator.hpp | 15 ++ relational_operators/InsertOperator.hpp | 4 + .../NestedLoopsJoinOperator.hpp | 4 + relational_operators/RelationalOperator.hpp | 2 +- relational_operators/SampleOperator.hpp | 4 + relational_operators/SaveBlocksOperator.hpp | 4 + relational_operators/SelectOperator.hpp | 4 + relational_operators/SortMergeRunOperator.hpp | 4 + .../SortRunGenerationOperator.hpp | 4 + relational_operators/TableGeneratorOperator.hpp | 4 + relational_operators/TextScanOperator.hpp | 4 + relational_operators/UpdateOperator.hpp | 4 + utility/CMakeLists.txt | 7 + utility/DAGVisualizer.cpp | 138 +++++++++++++++++++ utility/DAGVisualizer.hpp | 85 ++++++++++++ utility/EventProfiler.cpp | 3 +- utility/EventProfiler.hpp | 50 +++++-- 30 files changed, 380 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/871ccac9/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/CMakeLists.txt b/CMakeLists.txt index b4728a1..ae85b75 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -760,6 +760,7 @@ target_link_libraries(quickstep_cli_shell quickstep_queryoptimizer_QueryProcessor quickstep_storage_PreloaderThread quickstep_threading_ThreadIDBasedMap + quickstep_utility_DAGVisualizer quickstep_utility_EventProfiler quickstep_utility_Macros quickstep_utility_PtrVector http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/871ccac9/cli/QuickstepCli.cpp ---------------------------------------------------------------------- diff --git a/cli/QuickstepCli.cpp b/cli/QuickstepCli.cpp index e81f2fc..cb39c52 100644 --- a/cli/QuickstepCli.cpp +++ b/cli/QuickstepCli.cpp @@ -75,6 +75,7 @@ typedef quickstep::LineReaderDumb LineReaderImpl; #include "storage/PreloaderThread.hpp" #include "threading/ThreadIDBasedMap.hpp" +#include "utility/DAGVisualizer.hpp" #include "utility/EventProfiler.hpp" #include "utility/Macros.hpp" #include "utility/PtrVector.hpp" @@ -185,6 +186,8 @@ DEFINE_string(profile_file_name, "", // run at least a hundred times to make the impact of the first run small (< 5 %). DEFINE_string(profile_output, "", "Output file name for writing the profiled events."); +DEFINE_bool(visualize_dag, false, + "If true, visualize the execution plan DAG into a graph in DOT format."); } // namespace quickstep @@ -432,6 +435,7 @@ int main(int argc, char* argv[]) { } DCHECK(query_handle->getQueryPlanMutable() != nullptr); + quickstep::relop_profiler.clear(); start = std::chrono::steady_clock::now(); QueryExecutionUtil::ConstructAndSendAdmitRequestMessage( main_thread_client_id, @@ -444,6 +448,11 @@ int main(int argc, char* argv[]) { main_thread_client_id, &bus); end = std::chrono::steady_clock::now(); + if (quickstep::FLAGS_visualize_dag) { + quickstep::DAGVisualizer visualizer(*query_handle->getQueryPlanMutable()); + std::cerr << "\n" << visualizer.toDOT() << "\n"; + } + const CatalogRelation *query_result_relation = query_handle->getQueryResultRelation(); if (query_result_relation) { PrintToScreen::PrintRelation(*query_result_relation, http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/871ccac9/query_execution/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/query_execution/CMakeLists.txt b/query_execution/CMakeLists.txt index 501166e..ab3d936 100644 --- a/query_execution/CMakeLists.txt +++ b/query_execution/CMakeLists.txt @@ -184,6 +184,7 @@ target_link_libraries(quickstep_queryexecution_Worker quickstep_threading_Thread quickstep_threading_ThreadIDBasedMap quickstep_threading_ThreadUtil + quickstep_utility_EventProfiler quickstep_utility_Macros tmb) target_link_libraries(quickstep_queryexecution_WorkerDirectory http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/871ccac9/query_execution/Worker.cpp ---------------------------------------------------------------------- diff --git a/query_execution/Worker.cpp b/query_execution/Worker.cpp index ef596e1..1372e1d 100644 --- a/query_execution/Worker.cpp +++ b/query_execution/Worker.cpp @@ -27,6 +27,7 @@ #include "relational_operators/WorkOrder.hpp" #include "threading/ThreadIDBasedMap.hpp" #include "threading/ThreadUtil.hpp" +#include "utility/EventProfiler.hpp" #include "glog/logging.h" @@ -57,7 +58,12 @@ void Worker::run() { case kRebuildWorkOrderMessage: { WorkerMessage message(*static_cast<const WorkerMessage*>(tagged_message.message())); DCHECK(message.getWorkOrder() != nullptr); + auto *container = relop_profiler.getContainer(); + auto *line = container->getEventLine(message.getRelationalOpIndex()); + line->emplace_back(); message.getWorkOrder()->execute(); + line->back().endEvent(); + container->startEvent(message.getRelationalOpIndex()); const std::size_t query_id_for_workorder = message.getWorkOrder()->getQueryID(); delete message.getWorkOrder(); http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/871ccac9/query_execution/tests/QueryManager_unittest.cpp ---------------------------------------------------------------------- diff --git a/query_execution/tests/QueryManager_unittest.cpp b/query_execution/tests/QueryManager_unittest.cpp index 4f98748..ad4e1a5 100644 --- a/query_execution/tests/QueryManager_unittest.cpp +++ b/query_execution/tests/QueryManager_unittest.cpp @@ -105,6 +105,10 @@ class MockOperator: public RelationalOperator { num_calls_donefeedingblocks_(0) { } + std::string getName() const override { + return "MockOperator"; + } + #define MOCK_OP_LOG(x) VLOG(x) << "Op[" << op_index_ << "]: " << __func__ << ": " // The methods below are used to check whether QueryManager calls the Relational http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/871ccac9/relational_operators/AggregationOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/AggregationOperator.hpp b/relational_operators/AggregationOperator.hpp index 4bcbcf6..c46ba2c 100644 --- a/relational_operators/AggregationOperator.hpp +++ b/relational_operators/AggregationOperator.hpp @@ -77,6 +77,10 @@ class AggregationOperator : public RelationalOperator { ~AggregationOperator() override {} + std::string getName() const override { + return "AggregationOperator"; + } + bool getAllWorkOrders(WorkOrdersContainer *container, QueryContext *query_context, StorageManager *storage_manager, http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/871ccac9/relational_operators/BuildHashOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/BuildHashOperator.hpp b/relational_operators/BuildHashOperator.hpp index 464bbf8..952c7ac 100644 --- a/relational_operators/BuildHashOperator.hpp +++ b/relational_operators/BuildHashOperator.hpp @@ -93,6 +93,10 @@ class BuildHashOperator : public RelationalOperator { ~BuildHashOperator() override {} + std::string getName() const override { + return "BuildHashOperator"; + } + bool getAllWorkOrders(WorkOrdersContainer *container, QueryContext *query_context, StorageManager *storage_manager, http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/871ccac9/relational_operators/CreateIndexOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/CreateIndexOperator.hpp b/relational_operators/CreateIndexOperator.hpp index 18ca656..4e05448 100644 --- a/relational_operators/CreateIndexOperator.hpp +++ b/relational_operators/CreateIndexOperator.hpp @@ -69,6 +69,10 @@ class CreateIndexOperator : public RelationalOperator { ~CreateIndexOperator() override {} + std::string getName() const override { + return "CreateIndexOperator"; + } + /** * @note No WorkOrder generated for this operator. **/ http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/871ccac9/relational_operators/CreateTableOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/CreateTableOperator.hpp b/relational_operators/CreateTableOperator.hpp index 6d91142..b7b707b 100644 --- a/relational_operators/CreateTableOperator.hpp +++ b/relational_operators/CreateTableOperator.hpp @@ -66,6 +66,10 @@ class CreateTableOperator : public RelationalOperator { ~CreateTableOperator() override {} + std::string getName() const override { + return "CreateTableOperator"; + } + /** * @note No WorkOrder generated for this operator. **/ http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/871ccac9/relational_operators/DeleteOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/DeleteOperator.hpp b/relational_operators/DeleteOperator.hpp index 74da8c1..abfe4a9 100644 --- a/relational_operators/DeleteOperator.hpp +++ b/relational_operators/DeleteOperator.hpp @@ -81,6 +81,10 @@ class DeleteOperator : public RelationalOperator { ~DeleteOperator() override {} + std::string getName() const override { + return "DeleteOperator"; + } + bool getAllWorkOrders(WorkOrdersContainer *container, QueryContext *query_context, StorageManager *storage_manager, http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/871ccac9/relational_operators/DestroyHashOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/DestroyHashOperator.hpp b/relational_operators/DestroyHashOperator.hpp index 181386f..ae65de5 100644 --- a/relational_operators/DestroyHashOperator.hpp +++ b/relational_operators/DestroyHashOperator.hpp @@ -58,6 +58,10 @@ class DestroyHashOperator : public RelationalOperator { ~DestroyHashOperator() override {} + std::string getName() const override { + return "DestroyHashOperator"; + } + bool getAllWorkOrders(WorkOrdersContainer *container, QueryContext *query_context, StorageManager *storage_manager, http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/871ccac9/relational_operators/DropTableOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/DropTableOperator.hpp b/relational_operators/DropTableOperator.hpp index 6c7fca3..f854b4f 100644 --- a/relational_operators/DropTableOperator.hpp +++ b/relational_operators/DropTableOperator.hpp @@ -74,6 +74,10 @@ class DropTableOperator : public RelationalOperator { ~DropTableOperator() override {} + std::string getName() const override { + return "DropTableOperator"; + } + bool getAllWorkOrders(WorkOrdersContainer *container, QueryContext *query_context, StorageManager *storage_manager, http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/871ccac9/relational_operators/FinalizeAggregationOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/FinalizeAggregationOperator.hpp b/relational_operators/FinalizeAggregationOperator.hpp index 158a637..0dcfc9e 100644 --- a/relational_operators/FinalizeAggregationOperator.hpp +++ b/relational_operators/FinalizeAggregationOperator.hpp @@ -74,6 +74,10 @@ class FinalizeAggregationOperator : public RelationalOperator { ~FinalizeAggregationOperator() override {} + std::string getName() const override { + return "FinalizeAggregationOperator"; + } + bool getAllWorkOrders(WorkOrdersContainer *container, QueryContext *query_context, StorageManager *storage_manager, http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/871ccac9/relational_operators/HashJoinOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/HashJoinOperator.hpp b/relational_operators/HashJoinOperator.hpp index 9762f04..d3e30f6 100644 --- a/relational_operators/HashJoinOperator.hpp +++ b/relational_operators/HashJoinOperator.hpp @@ -157,6 +157,21 @@ class HashJoinOperator : public RelationalOperator { ~HashJoinOperator() override {} + std::string getName() const override { + switch (join_type_) { + case JoinType::kInnerJoin: + return "HashJoinOperator"; + case JoinType::kLeftSemiJoin: + return "HashJoinOperator(LeftSemi)"; + case JoinType::kLeftAntiJoin: + return "HashJoinOperator(LeftAnti)"; + case JoinType::kLeftOuterJoin: + return "HashJoinOperator(LeftOuter)"; + default: break; + } + LOG(FATAL) << "Unknown join type in HashJoinOperator::getName()"; + } + bool getAllWorkOrders(WorkOrdersContainer *container, QueryContext *query_context, StorageManager *storage_manager, http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/871ccac9/relational_operators/InsertOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/InsertOperator.hpp b/relational_operators/InsertOperator.hpp index 78f5199..2c6aca7 100644 --- a/relational_operators/InsertOperator.hpp +++ b/relational_operators/InsertOperator.hpp @@ -73,6 +73,10 @@ class InsertOperator : public RelationalOperator { ~InsertOperator() override {} + std::string getName() const override { + return "InsertOperator"; + } + bool getAllWorkOrders(WorkOrdersContainer *container, QueryContext *query_context, StorageManager *storage_manager, http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/871ccac9/relational_operators/NestedLoopsJoinOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/NestedLoopsJoinOperator.hpp b/relational_operators/NestedLoopsJoinOperator.hpp index 992e76d..cf190fe 100644 --- a/relational_operators/NestedLoopsJoinOperator.hpp +++ b/relational_operators/NestedLoopsJoinOperator.hpp @@ -116,6 +116,10 @@ class NestedLoopsJoinOperator : public RelationalOperator { ~NestedLoopsJoinOperator() override {} + std::string getName() const override { + return "NestedLoopsJoinOperator"; + } + bool getAllWorkOrders(WorkOrdersContainer *container, QueryContext *query_context, StorageManager *storage_manager, http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/871ccac9/relational_operators/RelationalOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/RelationalOperator.hpp b/relational_operators/RelationalOperator.hpp index 852006b..65cd213 100644 --- a/relational_operators/RelationalOperator.hpp +++ b/relational_operators/RelationalOperator.hpp @@ -59,7 +59,7 @@ class RelationalOperator { * * @return The name of this relational operator. */ - virtual std::string getName() = 0; + virtual std::string getName() const = 0; /** * @brief Generate all the next WorkOrders for this RelationalOperator. http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/871ccac9/relational_operators/SampleOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/SampleOperator.hpp b/relational_operators/SampleOperator.hpp index f8fe5f6..08f08c8 100644 --- a/relational_operators/SampleOperator.hpp +++ b/relational_operators/SampleOperator.hpp @@ -93,6 +93,10 @@ class SampleOperator : public RelationalOperator { ~SampleOperator() override {} + std::string getName() const override { + return "SampleOperator"; + } + bool getAllWorkOrders(WorkOrdersContainer *container, QueryContext *query_context, StorageManager *storage_manager, http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/871ccac9/relational_operators/SaveBlocksOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/SaveBlocksOperator.hpp b/relational_operators/SaveBlocksOperator.hpp index 50032b6..ebc5ffc 100644 --- a/relational_operators/SaveBlocksOperator.hpp +++ b/relational_operators/SaveBlocksOperator.hpp @@ -64,6 +64,10 @@ class SaveBlocksOperator : public RelationalOperator { ~SaveBlocksOperator() override {} + std::string getName() const override { + return "SaveBlocksOperator"; + } + bool getAllWorkOrders(WorkOrdersContainer *container, QueryContext *query_context, StorageManager *storage_manager, http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/871ccac9/relational_operators/SelectOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/SelectOperator.hpp b/relational_operators/SelectOperator.hpp index 0c10686..ee25886 100644 --- a/relational_operators/SelectOperator.hpp +++ b/relational_operators/SelectOperator.hpp @@ -189,6 +189,10 @@ class SelectOperator : public RelationalOperator { ~SelectOperator() override {} + std::string getName() const override { + return "SelectOperator"; + } + bool getAllWorkOrders(WorkOrdersContainer *container, QueryContext *query_context, StorageManager *storage_manager, http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/871ccac9/relational_operators/SortMergeRunOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/SortMergeRunOperator.hpp b/relational_operators/SortMergeRunOperator.hpp index 177836f..9b07ad6 100644 --- a/relational_operators/SortMergeRunOperator.hpp +++ b/relational_operators/SortMergeRunOperator.hpp @@ -129,6 +129,10 @@ class SortMergeRunOperator : public RelationalOperator { **/ ~SortMergeRunOperator() {} + std::string getName() const override { + return "SortMergeRunOperator"; + } + bool getAllWorkOrders(WorkOrdersContainer *container, QueryContext *query_context, StorageManager *storage_manager, http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/871ccac9/relational_operators/SortRunGenerationOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/SortRunGenerationOperator.hpp b/relational_operators/SortRunGenerationOperator.hpp index 96a3ce1..54c7feb 100644 --- a/relational_operators/SortRunGenerationOperator.hpp +++ b/relational_operators/SortRunGenerationOperator.hpp @@ -109,6 +109,10 @@ class SortRunGenerationOperator : public RelationalOperator { ~SortRunGenerationOperator() {} + std::string getName() const override { + return "SortRunGenerationOperator"; + } + bool getAllWorkOrders(WorkOrdersContainer *container, QueryContext *query_context, StorageManager *storage_manager, http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/871ccac9/relational_operators/TableGeneratorOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/TableGeneratorOperator.hpp b/relational_operators/TableGeneratorOperator.hpp index 1b791a6..15e7052 100644 --- a/relational_operators/TableGeneratorOperator.hpp +++ b/relational_operators/TableGeneratorOperator.hpp @@ -76,6 +76,10 @@ class TableGeneratorOperator : public RelationalOperator { ~TableGeneratorOperator() override {} + std::string getName() const override { + return "TableGeneratorOperator"; + } + bool getAllWorkOrders(WorkOrdersContainer *container, QueryContext *query_context, StorageManager *storage_manager, http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/871ccac9/relational_operators/TextScanOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/TextScanOperator.hpp b/relational_operators/TextScanOperator.hpp index 1a62ded..6890d7d 100644 --- a/relational_operators/TextScanOperator.hpp +++ b/relational_operators/TextScanOperator.hpp @@ -134,6 +134,10 @@ class TextScanOperator : public RelationalOperator { ~TextScanOperator() override {} + std::string getName() const override { + return "TextScanOperator"; + } + bool getAllWorkOrders(WorkOrdersContainer *container, QueryContext *query_context, StorageManager *storage_manager, http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/871ccac9/relational_operators/UpdateOperator.hpp ---------------------------------------------------------------------- diff --git a/relational_operators/UpdateOperator.hpp b/relational_operators/UpdateOperator.hpp index 4471a17..d021844 100644 --- a/relational_operators/UpdateOperator.hpp +++ b/relational_operators/UpdateOperator.hpp @@ -94,6 +94,10 @@ class UpdateOperator : public RelationalOperator { ~UpdateOperator() override {} + std::string getName() const override { + return "UpdateOperator"; + } + bool getAllWorkOrders(WorkOrdersContainer *container, QueryContext *query_context, StorageManager *storage_manager, http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/871ccac9/utility/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/utility/CMakeLists.txt b/utility/CMakeLists.txt index de0e737..133e2f3 100644 --- a/utility/CMakeLists.txt +++ b/utility/CMakeLists.txt @@ -167,6 +167,7 @@ add_library(quickstep_utility_CalculateInstalledMemory CalculateInstalledMemory. add_library(quickstep_utility_Cast ../empty_src.cpp Cast.hpp) add_library(quickstep_utility_CheckSnprintf ../empty_src.cpp CheckSnprintf.hpp) add_library(quickstep_utility_DAG ../empty_src.cpp DAG.hpp) +add_library(quickstep_utility_DAGVisualizer DAGVisualizer.cpp DAGVisualizer.hpp) add_library(quickstep_utility_EventProfiler EventProfiler.cpp EventProfiler.hpp) add_library(quickstep_utility_EqualsAnyConstant ../empty_src.cpp EqualsAnyConstant.hpp) add_library(quickstep_utility_Glob Glob.cpp Glob.hpp) @@ -251,6 +252,11 @@ target_link_libraries(quickstep_utility_PlanVisualizer quickstep_queryoptimizer_physical_TopLevelPlan quickstep_utility_Macros quickstep_utility_StringUtil) +target_link_libraries(quickstep_utility_DAGVisualizer + quickstep_queryoptimizer_QueryPlan + quickstep_utility_EventProfiler + quickstep_utility_Macros + quickstep_utility_StringUtil) target_link_libraries(quickstep_utility_PtrList quickstep_utility_Macros) target_link_libraries(quickstep_utility_PtrMap @@ -311,6 +317,7 @@ target_link_libraries(quickstep_utility quickstep_utility_Cast quickstep_utility_CheckSnprintf quickstep_utility_DAG + quickstep_utility_DAGVisualizer quickstep_utility_EventProfiler quickstep_utility_EqualsAnyConstant quickstep_utility_Glob http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/871ccac9/utility/DAGVisualizer.cpp ---------------------------------------------------------------------- diff --git a/utility/DAGVisualizer.cpp b/utility/DAGVisualizer.cpp new file mode 100644 index 0000000..64c235d --- /dev/null +++ b/utility/DAGVisualizer.cpp @@ -0,0 +1,138 @@ +/** + * Copyright 2016, Quickstep Research Group, Computer Sciences Department, + * University of WisconsinâMadison. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + **/ + +#include "utility/DAGVisualizer.hpp" +#include "utility/EventProfiler.hpp" + +#include <cmath> +#include <cstddef> +#include <iomanip> +#include <memory> +#include <sstream> +#include <string> +#include <unordered_map> +#include <vector> + +#include "query_optimizer/QueryPlan.hpp" +#include "utility/StringUtil.hpp" + +#include "glog/logging.h" + +namespace quickstep { + +std::string DAGVisualizer::toDOT() { +// color_map_["TableReference"] = "skyblue"; +// color_map_["Selection"] = "#90EE90"; +// color_map_["HashJoin"] = "red"; +// color_map_["HashLeftOuterJoin"] = "orange"; + + std::set<std::string> no_display_op_names = + { "DestroyHashOperator", "DropTableOperator" }; + + const auto &dag = plan_.getQueryPlanDAG(); + const std::size_t num_nodes = dag.size(); + + std::vector<double> time_elapsed(num_nodes, 0); + std::vector<double> time_percentage(num_nodes, 0); + for (const auto &container : relop_profiler.containers()) { + for (const auto &line : container.second.events) { + for (const auto &event : line.second) { + time_elapsed[line.first] += + std::chrono::duration<double>(event.start_time - event.end_time).count(); + } + } + } + const std::size_t num_threads = relop_profiler.containers().size(); + double total_time_elapsed = 0; + for (std::size_t i = 0; i < time_elapsed.size(); ++i) { + time_elapsed[i] /= num_threads; + total_time_elapsed += time_elapsed[i]; + } + for (std::size_t i = 0; i < time_elapsed.size(); ++i) { + time_percentage[i] = time_elapsed[i] / total_time_elapsed; + } + + std::vector<bool> display_ops(num_nodes, false); + for (std::size_t node_index = 0; node_index < num_nodes; ++node_index) { + const std::string relop_name = dag.getNodePayload(node_index).getName(); + if (no_display_op_names.find(relop_name) == no_display_op_names.end()) { + display_ops[node_index] = true; + nodes_.emplace_back(); + NodeInfo &node_info = nodes_.back(); + node_info.id = node_index; + node_info.labels.emplace_back(relop_name); + node_info.labels.emplace_back( + std::to_string(std::lround(time_elapsed[node_index] / 100))); + } + } + for (std::size_t node_index = 0; node_index < num_nodes; ++node_index) { + if (display_ops[node_index]) { + for (const auto &link : dag.getDependents(node_index)) { + if (display_ops[link.first]) { + edges_.emplace_back(); + EdgeInfo &edge_info = edges_.back(); + edge_info.src_node_id = node_index; + edge_info.dst_node_id = link.first; + edge_info.is_pipeline_breaker = link.second; + } + } + } + } + + // Format output graph + std::ostringstream graph_oss; + graph_oss << "digraph g {\n"; + graph_oss << " rankdir=BT\n"; + graph_oss << " node [penwidth=2]\n"; + graph_oss << " edge [fontsize=16 fontcolor=gray penwidth=2]\n\n"; + + // Format nodes + for (const NodeInfo &node_info : nodes_) { + graph_oss << " " << node_info.id << " [ "; + if (!node_info.labels.empty()) { + graph_oss << "label=\"" + << EscapeSpecialChars(JoinToString(node_info.labels, " ")) + << "\" "; + } + if (!node_info.color.empty()) { + graph_oss << "style=filled fillcolor=\"" << node_info.color << "\" "; + } + graph_oss << "]\n"; + } + graph_oss << "\n"; + + // Format edges + for (const EdgeInfo &edge_info : edges_) { + graph_oss << " " << edge_info.src_node_id << " -> " + << edge_info.dst_node_id << " [ "; + if (edge_info.is_pipeline_breaker) { + graph_oss << "style=dashed "; + } + if (!edge_info.labels.empty()) { + graph_oss << "label=\"" + << EscapeSpecialChars(JoinToString(edge_info.labels, " ")) + << "\" "; + } + graph_oss << "]\n"; + } + + graph_oss << "}\n"; + + return graph_oss.str(); +} + +} // namespace quickstep http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/871ccac9/utility/DAGVisualizer.hpp ---------------------------------------------------------------------- diff --git a/utility/DAGVisualizer.hpp b/utility/DAGVisualizer.hpp new file mode 100644 index 0000000..5736f4b --- /dev/null +++ b/utility/DAGVisualizer.hpp @@ -0,0 +1,85 @@ +/** + * Copyright 2016, Quickstep Research Group, Computer Sciences Department, + * University of WisconsinâMadison. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + **/ + +#ifndef QUICKSTEP_UTILITY_DAG_VISUALIZER_HPP_ +#define QUICKSTEP_UTILITY_DAG_VISUALIZER_HPP_ + +#include <memory> +#include <string> +#include <unordered_map> +#include <vector> + +#include "utility/Macros.hpp" + +namespace quickstep { + +class QueryPlan; + +/** \addtogroup Utility + * @{ + */ + +/** + * @brief A visualizer that converts an execution plan DAG into a graph in + * DOT format. Note that DOT is a plain text graph description language. + * + * @note This utility tool can be further extended to be more generic. + */ +class DAGVisualizer { + public: + DAGVisualizer(const QueryPlan &plan) + : plan_(plan) {} + + ~DAGVisualizer() {} + + std::string toDOT(); + + private: + /** + * @brief Information of a graph node. + */ + struct NodeInfo { + std::size_t id; + std::vector<std::string> labels; + std::string color; + }; + + /** + * @brief Information of a graph edge. + */ + struct EdgeInfo { + std::size_t src_node_id; + std::size_t dst_node_id; + std::vector<std::string> labels; + bool is_pipeline_breaker; + }; + + const QueryPlan &plan_; + + std::unordered_map<std::string, std::string> color_map_; + + std::vector<NodeInfo> nodes_; + std::vector<EdgeInfo> edges_; + + DISALLOW_COPY_AND_ASSIGN(DAGVisualizer); +}; + +/** @} */ + +} // namespace quickstep + +#endif /* QUICKSTEP_UTILITY_DAG_VISUALIZER_HPP_ */ http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/871ccac9/utility/EventProfiler.cpp ---------------------------------------------------------------------- diff --git a/utility/EventProfiler.cpp b/utility/EventProfiler.cpp index 794c67d..dbd3e45 100644 --- a/utility/EventProfiler.cpp +++ b/utility/EventProfiler.cpp @@ -23,6 +23,7 @@ namespace quickstep { -EventProfiler simple_profiler; +EventProfiler<std::string, std::size_t> simple_profiler; +EventProfiler<std::size_t> relop_profiler; } // namespace quickstep http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/871ccac9/utility/EventProfiler.hpp ---------------------------------------------------------------------- diff --git a/utility/EventProfiler.hpp b/utility/EventProfiler.hpp index 9c842a7..cd5b1e2 100644 --- a/utility/EventProfiler.hpp +++ b/utility/EventProfiler.hpp @@ -42,6 +42,7 @@ namespace quickstep { using clock = std::chrono::steady_clock; +template <typename TagT, typename ...PayloadT> class EventProfiler { public: @@ -53,7 +54,7 @@ class EventProfiler { clock::time_point start_time; clock::time_point end_time; bool is_finished; - std::size_t payload; + std::tuple<PayloadT...> payload; explicit EventInfo(const clock::time_point &start_time_in) : start_time(start_time_in), @@ -65,8 +66,8 @@ class EventProfiler { is_finished(false) { } - inline void setPayload(const std::size_t &in_payload) { - payload = in_payload; + inline void setPayload(PayloadT &&...in_payload) { + payload = std::make_tuple(in_payload...); } inline void endEvent() { @@ -76,21 +77,21 @@ class EventProfiler { }; struct EventContainer { - inline void startEvent(const std::string &tag) { + inline void startEvent(const TagT &tag) { events[tag].emplace_back(clock::now()); } - inline void endEvent(const std::string &tag) { + inline void endEvent(const TagT &tag) { auto &event_info = events.at(tag).back(); event_info.is_finished = true; event_info.end_time = clock::now(); } - inline std::vector<EventInfo> *getEventLine(const std::string &tag) { + inline std::vector<EventInfo> *getEventLine(const TagT &tag) { return &events[tag]; } - std::map<std::string, std::vector<EventInfo>> events; + std::map<TagT, std::vector<EventInfo>> events; }; EventContainer *getContainer() { @@ -112,9 +113,11 @@ class EventProfiler { os << std::setprecision(12) << event_id << "," - << thread_id << "," << event_group.first << "," - << event_info.payload << "," - << std::chrono::duration<double>(event_info.start_time - zero_time).count() + << thread_id << "," << event_group.first << ","; + + PrintTuple(event_info.payload, ","); + + os << std::chrono::duration<double>(event_info.start_time - zero_time).count() << "," << std::chrono::duration<double>(event_info.end_time - zero_time).count() << "\n"; @@ -129,13 +132,38 @@ class EventProfiler { thread_map_.clear(); } + const std::map<std::thread::id, EventContainer> &containers() { + return thread_map_; + } + private: + template<class Tuple, std::size_t N> + struct TuplePrinter { + static void Print(const Tuple &t, const std::string &sep) { + TuplePrinter<Tuple, N-1>::print(t); + std::cout << std::get<N-1>(t) << sep; + } + }; + + template<class Tuple> + struct TuplePrinter<Tuple, 1> { + static void Print(const Tuple &t, const std::string &sep) { + std::cout << std::get<0>(t) << sep; + } + }; + + template<class... Args> + static void PrintTuple(const std::tuple<Args...>& t, const std::string &sep) { + TuplePrinter<decltype(t), sizeof...(Args)>::Print(t, sep); + } + clock::time_point zero_time; std::map<std::thread::id, EventContainer> thread_map_; Mutex mutex_; }; -extern EventProfiler simple_profiler; +extern EventProfiler<std::string, std::size_t> simple_profiler; +extern EventProfiler<std::size_t> relop_profiler; /** @} */