HAWQ-551. Add datalocility time to explain analyse result
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/dfc67313 Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/dfc67313 Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/dfc67313 Branch: refs/heads/HAWQ-459 Commit: dfc673135f9f1c92487423436428be82ad1db1e4 Parents: 94b7a82 Author: ivan <[email protected]> Authored: Thu Mar 17 15:07:27 2016 +0800 Committer: ivan <[email protected]> Committed: Fri Mar 18 09:32:31 2016 +0800 ---------------------------------------------------------------------- src/backend/cdb/cdbdatalocality.c | 28 +++++++++++++++++++++++++--- src/backend/commands/explain.c | 2 +- src/backend/optimizer/plan/planner.c | 1 + src/include/cdb/cdbdatalocality.h | 1 + src/include/nodes/plannodes.h | 1 + 5 files changed, 29 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/dfc67313/src/backend/cdb/cdbdatalocality.c ---------------------------------------------------------------------- diff --git a/src/backend/cdb/cdbdatalocality.c b/src/backend/cdb/cdbdatalocality.c index c1f829b..eec87b4 100644 --- a/src/backend/cdb/cdbdatalocality.c +++ b/src/backend/cdb/cdbdatalocality.c @@ -309,6 +309,10 @@ typedef struct split_to_segment_mapping_context { int64 total_file_count; int64 total_metadata_logic_len; + + int metadata_cache_time_us; + int alloc_resource_time_us; + int cal_datalocality_time_us; } split_to_segment_mapping_context; typedef struct vseg_list{ @@ -569,6 +573,10 @@ static void init_datalocality_context(split_to_segment_mapping_context *context) context->total_split_count = 0; context->total_file_count = 0; context->total_metadata_logic_len = 0; + + context->metadata_cache_time_us = 0; + context->alloc_resource_time_us = 0; + context->cal_datalocality_time_us = 0; return; } @@ -928,6 +936,8 @@ int64 get_block_locations_and_claculte_table_size(split_to_segment_mapping_conte } context->total_file_count = totalFileCount; context->total_size = total_size; + + context->metadata_cache_time_us = eclaspeTime; if(debug_datalocality_time){ elog(LOG, "metadata overall execution time: %d us. \n", eclaspeTime); @@ -3395,13 +3405,13 @@ static void print_datalocality_overall_log_information(SplitAllocResult *result, appendStringInfo(result->datalocalityInfo, "data locality ratio: %.3f; virtual segment number: %d; " "different host number: %d; virtual segment number per host(avg/min/max): (%d/%d/%d); " "segment size(avg/min/max): (%.3f B/"INT64_FORMAT" B/"INT64_FORMAT" B); " - "segment size with penalty(avg/min/max): (%.3f B/"INT64_FORMAT" B/"INT64_FORMAT" B); continuity(avg/min/max): (%.3f/%.3f/%.3f)." + "segment size with penalty(avg/min/max): (%.3f B/"INT64_FORMAT" B/"INT64_FORMAT" B); continuity(avg/min/max): (%.3f/%.3f/%.3f); " ,log_context->datalocalityRatio,assignment_context->virtual_segment_num,log_context->numofDifferentHost, log_context->avgSegmentNumofHost,log_context->minSegmentNumofHost,log_context->maxSegmentNumofHost, log_context->avgSizeOverall,log_context->minSizeSegmentOverall,log_context->maxSizeSegmentOverall, log_context->avgSizeOverallPenalty,log_context->minSizeSegmentOverallPenalty,log_context->maxSizeSegmentOverallPenalty, - log_context->avgContinuityOverall,log_context->minContinuityOverall,log_context->maxContinuityOverall - ); + log_context->avgContinuityOverall,log_context->minContinuityOverall,log_context->maxContinuityOverall); + if (debug_fake_datalocality) { fprintf(fp, "datalocality ratio: %.3f; virtual segments number: %d, " "different host number: %d, segment number per host(avg/min/max): (%d/%d/%d); " @@ -3919,9 +3929,17 @@ run_allocation_algorithm(SplitAllocResult *result, List *virtual_segments, Query uint64_t run_datalocality = 0; run_datalocality = gettime_microsec(); int dl_overall_time = run_datalocality - before_run_allocation; + + context->cal_datalocality_time_us = dl_overall_time; + if(debug_datalocality_time){ elog(LOG, "datalocality overall execution time: %d us. \n", dl_overall_time); } + + result->datalocalityTime = (double)(context->metadata_cache_time_us + context->alloc_resource_time_us + context->cal_datalocality_time_us)/ 1000; + appendStringInfo(result->datalocalityInfo, "DFS metadatacache: %.3f ms; resource allocation: %.3f ms; datalocality calculation: %.3f ms.", + (double)context->metadata_cache_time_us/1000, (double)context->alloc_resource_time_us/1000, (double)context->cal_datalocality_time_us/1000); + return alloc_result; } @@ -3980,6 +3998,7 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife, result->relsType = NIL; result->planner_segments = -1; result->datalocalityInfo = makeStringInfo(); + result->datalocalityTime = 0; /* fake data locality */ if (debug_fake_datalocality) { @@ -4230,6 +4249,9 @@ calculate_planner_segment_num(Query *query, QueryResourceLife resourceLife, } uint64_t after_rm_allocate_resource = gettime_microsec(); int eclaspeTime = after_rm_allocate_resource - before_rm_allocate_resource; + + context.alloc_resource_time_us = eclaspeTime; + if(debug_datalocality_time){ elog(LOG, "rm allocate resource overall execution time: %d us. \n", eclaspeTime); } http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/dfc67313/src/backend/commands/explain.c ---------------------------------------------------------------------- diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c index 51afde2..0e7bff5 100644 --- a/src/backend/commands/explain.c +++ b/src/backend/commands/explain.c @@ -470,7 +470,7 @@ ExplainOnePlan_internal(PlannedStmt *plannedstmt, MemoryContext oldcxt = CurrentMemoryContext; QueryDesc *queryDesc; instr_time starttime; - double totaltime = 0; + double totaltime = plannedstmt->datalocalityTime / 1000; StringInfo buf = &es->outbuf; EState *estate = NULL; int eflags; http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/dfc67313/src/backend/optimizer/plan/planner.c ---------------------------------------------------------------------- diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c index be057e0..6b2e13e 100644 --- a/src/backend/optimizer/plan/planner.c +++ b/src/backend/optimizer/plan/planner.c @@ -459,6 +459,7 @@ planner(Query *parse, int cursorOptions, result->scantable_splits = ppResult->saResult.alloc_results; result->planner_segments = ppResult->saResult.planner_segments; result->datalocalityInfo = ppResult->saResult.datalocalityInfo; + result->datalocalityTime = ppResult->saResult.datalocalityTime; } plannerLevel--; if ((ppResult != NULL)) http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/dfc67313/src/include/cdb/cdbdatalocality.h ---------------------------------------------------------------------- diff --git a/src/include/cdb/cdbdatalocality.h b/src/include/cdb/cdbdatalocality.h index 95f2bd5..d28fc3e 100644 --- a/src/include/cdb/cdbdatalocality.h +++ b/src/include/cdb/cdbdatalocality.h @@ -46,6 +46,7 @@ typedef struct SplitAllocResult int planner_segments; List *relsType;// relation type after datalocality changing StringInfo datalocalityInfo; + double datalocalityTime; } SplitAllocResult; /* http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/dfc67313/src/include/nodes/plannodes.h ---------------------------------------------------------------------- diff --git a/src/include/nodes/plannodes.h b/src/include/nodes/plannodes.h index ebccb9d..7c10cd5 100644 --- a/src/include/nodes/plannodes.h +++ b/src/include/nodes/plannodes.h @@ -204,6 +204,7 @@ typedef struct PlannedStmt MemoryAccount *memoryAccount; StringInfo datalocalityInfo; + double datalocalityTime; } PlannedStmt;
