Visualized Execution DAG in the distributed version.
Project: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/commit/f5c063a1 Tree: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/tree/f5c063a1 Diff: http://git-wip-us.apache.org/repos/asf/incubator-quickstep/diff/f5c063a1 Branch: refs/heads/dist-patch Commit: f5c063a19d0b9ff4327041f707a1dc38c343f727 Parents: 2b2d7ba Author: Zuyu Zhang <zu...@apache.org> Authored: Fri Feb 10 22:01:48 2017 -0800 Committer: Zuyu Zhang <zu...@apache.org> Committed: Thu Feb 23 00:39:05 2017 -0800 ---------------------------------------------------------------------- query_execution/CMakeLists.txt | 5 ++++- query_execution/PolicyEnforcerBase.cpp | 7 ++---- query_execution/PolicyEnforcerBase.hpp | 16 +++++++++++-- query_execution/PolicyEnforcerDistributed.cpp | 26 +++++++++++++++++++++- query_execution/QueryManagerBase.cpp | 11 +++++++++ query_execution/QueryManagerBase.hpp | 12 ++++++++++ query_optimizer/QueryHandle.hpp | 7 ++++++ 7 files changed, 75 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5c063a1/query_execution/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/query_execution/CMakeLists.txt b/query_execution/CMakeLists.txt index 5ad6999..50bf694 100644 --- a/query_execution/CMakeLists.txt +++ b/query_execution/CMakeLists.txt @@ -166,6 +166,7 @@ if (ENABLE_DISTRIBUTED) quickstep_queryexecution_ShiftbossDirectory quickstep_queryoptimizer_QueryHandle quickstep_storage_StorageBlockInfo + quickstep_utility_ExecutionDAGVisualizer quickstep_utility_Macros tmb ${GFLAGS_LIB_NAME}) @@ -246,7 +247,9 @@ target_link_libraries(quickstep_queryexecution_QueryManagerBase quickstep_relationaloperators_WorkOrder quickstep_storage_StorageBlockInfo quickstep_utility_DAG - quickstep_utility_Macros) + quickstep_utility_ExecutionDAGVisualizer + quickstep_utility_Macros + ${GFLAGS_LIB_NAME}) if (ENABLE_DISTRIBUTED) target_link_libraries(quickstep_queryexecution_QueryManagerDistributed quickstep_catalog_CatalogTypedefs http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5c063a1/query_execution/PolicyEnforcerBase.cpp ---------------------------------------------------------------------- diff --git a/query_execution/PolicyEnforcerBase.cpp b/query_execution/PolicyEnforcerBase.cpp index 082f6e9..1ffde4d 100644 --- a/query_execution/PolicyEnforcerBase.cpp +++ b/query_execution/PolicyEnforcerBase.cpp @@ -40,15 +40,12 @@ namespace quickstep { +DECLARE_bool(visualize_execution_dag); + DEFINE_bool(profile_and_report_workorder_perf, false, "If true, Quickstep will record the exceution time of all the individual " "normal work orders and report it at the end of query execution."); -DEFINE_bool(visualize_execution_dag, false, - "If true, visualize the execution plan DAG into a graph in DOT " - "format (DOT is a plain text graph description language) which is " - "then printed via stderr."); - PolicyEnforcerBase::PolicyEnforcerBase(CatalogDatabaseLite *catalog_database) : catalog_database_(catalog_database), profile_individual_workorders_(FLAGS_profile_and_report_workorder_perf || FLAGS_visualize_execution_dag) { http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5c063a1/query_execution/PolicyEnforcerBase.hpp ---------------------------------------------------------------------- diff --git a/query_execution/PolicyEnforcerBase.hpp b/query_execution/PolicyEnforcerBase.hpp index 4107817..f66134b 100644 --- a/query_execution/PolicyEnforcerBase.hpp +++ b/query_execution/PolicyEnforcerBase.hpp @@ -113,6 +113,19 @@ class PolicyEnforcerBase { } /** + * @brief Check if the given query has profiling results. + * + * @note Even enabled profiling, not every query has profiling results. + * For example, CreateTable and CreateIndex do not produce work orders, + * so they do not have profiling results. + * + * @return True if it has profiling results, otherwise false. + **/ + bool hasProfilingResults(const std::size_t query_id) const { + return workorder_time_recorder_.find(query_id) != workorder_time_recorder_.end(); + } + + /** * @brief Get the profiling results for individual work order execution for a * given query. * @@ -127,8 +140,7 @@ class PolicyEnforcerBase { inline const std::vector<WorkOrderTimeEntry>& getProfilingResults( const std::size_t query_id) const { DCHECK(profile_individual_workorders_); - DCHECK(workorder_time_recorder_.find(query_id) != - workorder_time_recorder_.end()); + DCHECK(hasProfilingResults(query_id)); return workorder_time_recorder_.at(query_id); } http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5c063a1/query_execution/PolicyEnforcerDistributed.cpp ---------------------------------------------------------------------- diff --git a/query_execution/PolicyEnforcerDistributed.cpp b/query_execution/PolicyEnforcerDistributed.cpp index 8f0332d..6ee58a8 100644 --- a/query_execution/PolicyEnforcerDistributed.cpp +++ b/query_execution/PolicyEnforcerDistributed.cpp @@ -15,11 +15,14 @@ #include "query_execution/PolicyEnforcerDistributed.hpp" #include <cstddef> +#include <cstdio> #include <cstdlib> #include <memory> #include <queue> -#include <utility> +#include <sstream> +#include <string> #include <unordered_map> +#include <utility> #include <vector> #include "catalog/Catalog.pb.h" @@ -33,6 +36,7 @@ #include "query_execution/QueryManagerDistributed.hpp" #include "query_optimizer/QueryHandle.hpp" #include "storage/StorageBlockInfo.hpp" +#include "utility/ExecutionDAGVisualizer.hpp" #include "gflags/gflags.h" #include "glog/logging.h" @@ -53,6 +57,8 @@ using tmb::TaggedMessage; namespace quickstep { +DECLARE_bool(visualize_execution_dag); + namespace S = serialization; DEFINE_uint64(max_msgs_per_dispatch_round, 20, "Maximum number of messages that" @@ -228,6 +234,24 @@ void PolicyEnforcerDistributed::onQueryCompletion(QueryManagerBase *query_manage const tmb::client_id cli_id = query_handle->getClientId(); const std::size_t query_id = query_handle->query_id(); + if (FLAGS_visualize_execution_dag && hasProfilingResults(query_id)) { + ExecutionDAGVisualizer* dag_visualizer = query_manager->dag_visualizer(); + dag_visualizer->bindProfilingStats(getProfilingResults(query_id)); + + std::ostringstream dot_filename; + dot_filename << query_id << ".dot"; + + FILE *fp = std::fopen(dot_filename.str().c_str(), "w"); + CHECK_NOTNULL(fp); + const std::string dot_file_content(dag_visualizer->toDOT()); + const std::size_t dot_file_length = dot_file_content.length(); + + CHECK_EQ(dot_file_length, + std::fwrite(dot_file_content.c_str(), sizeof(char), dot_file_length, fp)); + + std::fclose(fp); + } + // TODO(quickstep-team): Dynamically scale-up/down Shiftbosses. tmb::Address shiftboss_addresses; for (std::size_t i = 0; i < shiftboss_directory_->size(); ++i) { http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5c063a1/query_execution/QueryManagerBase.cpp ---------------------------------------------------------------------- diff --git a/query_execution/QueryManagerBase.cpp b/query_execution/QueryManagerBase.cpp index 5f8c6a3..f84ad4e 100644 --- a/query_execution/QueryManagerBase.cpp +++ b/query_execution/QueryManagerBase.cpp @@ -30,12 +30,18 @@ #include "relational_operators/WorkOrder.hpp" #include "storage/StorageBlockInfo.hpp" +#include "gflags/gflags.h" #include "glog/logging.h" using std::pair; namespace quickstep { +DEFINE_bool(visualize_execution_dag, false, + "If true, visualize the execution plan DAG into a graph in DOT " + "format (DOT is a plain text graph description language) which is " + "then printed via stderr."); + QueryManagerBase::QueryManagerBase(QueryHandle *query_handle) : query_handle_(DCHECK_NOTNULL(query_handle)), query_id_(query_handle->query_id()), @@ -45,6 +51,11 @@ QueryManagerBase::QueryManagerBase(QueryHandle *query_handle) output_consumers_(num_operators_in_dag_), blocking_dependencies_(num_operators_in_dag_), query_exec_state_(new QueryExecutionState(num_operators_in_dag_)) { + if (FLAGS_visualize_execution_dag) { + dag_visualizer_ = + std::make_unique<quickstep::ExecutionDAGVisualizer>(query_handle_->getQueryPlan()); + } + for (dag_node_index node_index = 0; node_index < num_operators_in_dag_; ++node_index) { http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5c063a1/query_execution/QueryManagerBase.hpp ---------------------------------------------------------------------- diff --git a/query_execution/QueryManagerBase.hpp b/query_execution/QueryManagerBase.hpp index d0bb0ea..27fa6dc 100644 --- a/query_execution/QueryManagerBase.hpp +++ b/query_execution/QueryManagerBase.hpp @@ -31,6 +31,7 @@ #include "relational_operators/WorkOrder.hpp" #include "storage/StorageBlockInfo.hpp" #include "utility/DAG.hpp" +#include "utility/ExecutionDAGVisualizer.hpp" #include "utility/Macros.hpp" namespace quickstep { @@ -149,6 +150,15 @@ class QueryManagerBase { **/ QueryStatusCode queryStatus(const dag_node_index op_index); + /** + * @brief Get the execution DAG visualizer. + * + * @return the execution DAG visualizer. + **/ + ExecutionDAGVisualizer* dag_visualizer() { + return dag_visualizer_.get(); + } + protected: /** * @brief Process a current relational operator: Get its workorders and store @@ -276,6 +286,8 @@ class QueryManagerBase { std::unique_ptr<QueryExecutionState> query_exec_state_; + std::unique_ptr<ExecutionDAGVisualizer> dag_visualizer_; + private: /** * @brief Check if the given operator's normal execution is over. http://git-wip-us.apache.org/repos/asf/incubator-quickstep/blob/f5c063a1/query_optimizer/QueryHandle.hpp ---------------------------------------------------------------------- diff --git a/query_optimizer/QueryHandle.hpp b/query_optimizer/QueryHandle.hpp index cbd1cd9..7cb4f68 100644 --- a/query_optimizer/QueryHandle.hpp +++ b/query_optimizer/QueryHandle.hpp @@ -87,6 +87,13 @@ class QueryHandle { } /** + * @return The const query plan. + */ + const QueryPlan& getQueryPlan() const { + return *query_plan_; + } + + /** * @return The mutable query plan. */ QueryPlan* getQueryPlanMutable() {