This is an automated email from the ASF dual-hosted git repository. wzhou pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit 69da2ff86ec0914419e3b2d2ce755d1bb73c46aa Author: Riza Suminto <[email protected]> AuthorDate: Fri Apr 28 09:21:21 2023 -0700 IMPALA-12106: Fix overparallelization of Union fragment by 1 IMPALA-10973 has a bug where a union fragment without a scan node can be over-parallelized by the backend scheduler by 1. It is reproducible by running TPC-DS Q11 with MT_DOP=1. This patch additionally checks that such a fragment does not have an input fragment before randomizing the host assignment. Testing: Add TPC-DS Q11 to test_mt_dop.py::TestMtDopScheduling::test_scheduling and verify the number of fragment instances scheduled in the ExecSummary. Change-Id: Ic69e7c8c0cadb4b07ee398aff362fbc6513eb08d Reviewed-on: http://gerrit.cloudera.org:8080/19816 Reviewed-by: Impala Public Jenkins <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> --- be/src/scheduling/scheduler.cc | 5 +- .../QueryTest/mt-dop-parquet-scheduling.test | 176 +++++++++++++++++++++ 2 files changed, 179 insertions(+), 2 deletions(-) diff --git a/be/src/scheduling/scheduler.cc b/be/src/scheduling/scheduler.cc index 09204ff4a..9f4a96592 100644 --- a/be/src/scheduling/scheduler.cc +++ b/be/src/scheduling/scheduler.cc @@ -438,8 +438,9 @@ void Scheduler::CreateCollocatedAndScanInstances(const ExecutorConfig& executor_ << "for plans with no union and multiple scans per fragment"; vector<NetworkAddressPB> scan_hosts; GetScanHosts(scan_node_ids, *fragment_state, &scan_hosts); - if (scan_hosts.empty()) { - // None of the scan nodes have any scan ranges; run it on a random executor. + if (scan_hosts.empty() && instances_per_host.empty()) { + // None of the scan nodes have any scan ranges and there is no input fragment feeding + // into this fragment; run it on a random executor. // TODO TODO: the TODO below seems partially stale // TODO: we'll need to revisit this strategy once we can partition joins // (in which case this fragment might be executing a right outer join diff --git a/testdata/workloads/functional-query/queries/QueryTest/mt-dop-parquet-scheduling.test b/testdata/workloads/functional-query/queries/QueryTest/mt-dop-parquet-scheduling.test index 3b382015f..169385d3d 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/mt-dop-parquet-scheduling.test +++ b/testdata/workloads/functional-query/queries/QueryTest/mt-dop-parquet-scheduling.test @@ -131,3 +131,179 @@ row_regex:.*03:AGGREGATE * 3 * 6 .* row_regex:.*04:SCAN (HDFS|OZONE) * 3 * 6 .* row_regex:.*01:SCAN (HDFS|OZONE) * 3 * 6 .* ==== +---- QUERY: TPCDS-Q11 +with year_total as ( + select c_customer_id customer_id + ,c_first_name customer_first_name + ,c_last_name customer_last_name + ,c_preferred_cust_flag customer_preferred_cust_flag + ,c_birth_country customer_birth_country + ,c_login customer_login + ,c_email_address customer_email_address + ,d_year dyear + ,sum(ss_ext_list_price-ss_ext_discount_amt) year_total + ,'s' sale_type + from tpcds_parquet.customer + ,tpcds_parquet.store_sales + ,tpcds_parquet.date_dim + where c_customer_sk = ss_customer_sk + and ss_sold_date_sk = d_date_sk + group by c_customer_id + ,c_first_name + ,c_last_name + ,c_preferred_cust_flag + ,c_birth_country + ,c_login + ,c_email_address + ,d_year + union all + select c_customer_id customer_id + ,c_first_name customer_first_name + ,c_last_name customer_last_name + ,c_preferred_cust_flag customer_preferred_cust_flag + ,c_birth_country customer_birth_country + ,c_login customer_login + ,c_email_address customer_email_address + ,d_year dyear + ,sum(ws_ext_list_price-ws_ext_discount_amt) year_total + ,'w' sale_type + from tpcds_parquet.customer + ,tpcds_parquet.web_sales + ,tpcds_parquet.date_dim + where c_customer_sk = ws_bill_customer_sk + and ws_sold_date_sk = d_date_sk + group by c_customer_id + ,c_first_name + ,c_last_name + ,c_preferred_cust_flag + ,c_birth_country + ,c_login + ,c_email_address + ,d_year + ) + select + t_s_secyear.customer_id + ,t_s_secyear.customer_first_name + ,t_s_secyear.customer_last_name + ,t_s_secyear.customer_preferred_cust_flag + from year_total t_s_firstyear + ,year_total t_s_secyear + ,year_total t_w_firstyear + ,year_total t_w_secyear + where t_s_secyear.customer_id = t_s_firstyear.customer_id + and t_s_firstyear.customer_id = t_w_secyear.customer_id + and t_s_firstyear.customer_id = t_w_firstyear.customer_id + and t_s_firstyear.sale_type = 's' + and t_w_firstyear.sale_type = 'w' + and t_s_secyear.sale_type = 's' + and t_w_secyear.sale_type = 'w' + and t_s_firstyear.dyear = 2001 + and t_s_secyear.dyear = 2001+1 + and t_w_firstyear.dyear = 2001 + and t_w_secyear.dyear = 2001+1 + and t_s_firstyear.year_total > 0 + and t_w_firstyear.year_total > 0 + and case when t_w_firstyear.year_total > 0 then t_w_secyear.year_total / t_w_firstyear.year_total else 0.0 end + > case when t_s_firstyear.year_total > 0 then t_s_secyear.year_total / t_s_firstyear.year_total else 0.0 end + order by t_s_secyear.customer_id + ,t_s_secyear.customer_first_name + ,t_s_secyear.customer_last_name + ,t_s_secyear.customer_preferred_cust_flag +limit 100; +---- RESULTS +'AAAAAAAAAFGBBAAA','Howard','Major','Y' +'AAAAAAAAAMGDAAAA','Kenneth','Harlan','Y' +'AAAAAAAAAOPFBAAA','Jerry','Fields','Y' +'AAAAAAAABLEIBAAA','Paula','Wakefield','Y' +'AAAAAAAABNBBAAAA','Irma','Smith','Y' +'AAAAAAAACADPAAAA','Cristobal','Thomas','Y' +'AAAAAAAACFENAAAA','Christopher','Dawson','NULL' +'AAAAAAAACIJMAAAA','Elizabeth','Thomas','Y' +'AAAAAAAACJDIAAAA','James','Kerr','N' +'AAAAAAAACNAGBAAA','Virginia','May','N' +'AAAAAAAADBEFBAAA','Bennie','Bowers','N' +'AAAAAAAADCKOAAAA','Robert','Gonzalez','N' +'AAAAAAAADFKABAAA','Latoya','Craft','N' +'AAAAAAAADIIOAAAA','David','Carroll','Y' +'AAAAAAAADIJGBAAA','Ruth','Sanders','N' +'AAAAAAAADLHBBAAA','Henry','Bertrand','N' +'AAAAAAAAEADJAAAA','Ruth','Carroll','N' +'AAAAAAAAEJDLAAAA','Alice','Wright','N' +'AAAAAAAAEKFPAAAA','Annika','Chin','N' +'AAAAAAAAEKJLAAAA','Aisha','Carlson','Y' +'AAAAAAAAEPOGAAAA','Felisha','Mendes','Y' +'AAAAAAAAFACEAAAA','Priscilla','Miller','N' +'AAAAAAAAFBAHAAAA','Michael','Williams','N' +'AAAAAAAAFGIGAAAA','Eduardo','Miller','Y' +'AAAAAAAAFGPGAAAA','Albert','Wadsworth','Y' +'AAAAAAAAFMHIAAAA','Emilio','Darling','Y' +'AAAAAAAAFOGIAAAA','Michelle','Greene','N' +'AAAAAAAAFOJAAAAA','Don','Castillo','Y' +'AAAAAAAAGEHIAAAA','Tyler','Miller','N' +'AAAAAAAAGHPBBAAA','Nick','Mendez','Y' +'AAAAAAAAGNDAAAAA','Terry','Mcdowell','N' +'AAAAAAAAHGOABAAA','Sonia','White','N' +'AAAAAAAAHHCABAAA','William','Stewart','Y' +'AAAAAAAAHJLAAAAA','Audrey','Beltran','Y' +'AAAAAAAAHMJNAAAA','Ryan','Baptiste','Y' +'AAAAAAAAHMOIAAAA','Grace','Henderson','N' +'AAAAAAAAIADEBAAA','Diane','Aldridge','N' +'AAAAAAAAIBAEBAAA','Sandra','Wilson','N' +'AAAAAAAAIBFCBAAA','Ruth','Grantham','N' +'AAAAAAAAIBHHAAAA','Jennifer','Ballard','Y' +'AAAAAAAAICHFAAAA','Linda','Mccoy','N' +'AAAAAAAAIDKFAAAA','Michael','Mack','N' +'AAAAAAAAIJEMAAAA','Charlie','Cummings','Y' +'AAAAAAAAIMHBAAAA','Kathy','Knowles','N' +'AAAAAAAAIMHHBAAA','Lillian','Davidson','Y' +'AAAAAAAAJDBLAAAA','Melvin','Taylor','Y' +'AAAAAAAAJEKFBAAA','Norma','Burkholder','N' +'AAAAAAAAJGMMAAAA','Richard','Larson','Y' +'AAAAAAAAJIALAAAA','Santos','Gutierrez','N' +'AAAAAAAAJKBNAAAA','Julie','Kern','N' +'AAAAAAAAJMHLAAAA','Wanda','Ryan','Y' +'AAAAAAAAJONHBAAA','Warren','Orozco','N' +'AAAAAAAAJPINAAAA','Rose','Waite','Y' +'AAAAAAAAKAECAAAA','Milton','Mackey','N' +'AAAAAAAAKAPPAAAA','Karen','Parker','Y' +'AAAAAAAAKJBKAAAA','Georgia','Scott','N' +'AAAAAAAAKJBLAAAA','Kerry','Davis','Y' +'AAAAAAAAKKGEAAAA','Katie','Dunbar','N' +'AAAAAAAAKLHHBAAA','Manuel','Castaneda','N' +'AAAAAAAAKNAKAAAA','Gladys','Banks','N' +'AAAAAAAALFKKAAAA','Ignacio','Miller','Y' +'AAAAAAAALHMCAAAA','Brooke','Nelson','Y' +'AAAAAAAALIOPAAAA','Derek','Allen','Y' +'AAAAAAAALJNCBAAA','George','Gamez','Y' +'AAAAAAAAMDCAAAAA','Louann','Hamel','Y' +'AAAAAAAAMFFLAAAA','Margret','Gray','Y' +'AAAAAAAAMMOBBAAA','Margaret','Smith','N' +'AAAAAAAANFBDBAAA','Vernice','Fernandez','Y' +'AAAAAAAANGDBBAAA','Carlos','Jewell','N' +'AAAAAAAANIPLAAAA','Eric','Lawrence','Y' +'AAAAAAAANJAGAAAA','Allen','Hood','Y' +'AAAAAAAANJHCBAAA','Christopher','Schreiber','N' +'AAAAAAAANJOLAAAA','Debra','Underwood','Y' +'AAAAAAAAOBADBAAA','Elizabeth','Burnham','N' +'AAAAAAAAOCAJAAAA','Jenna','Staton','N' +'AAAAAAAAOCLBBAAA','NULL','NULL','NULL' +'AAAAAAAAODMMAAAA','Gayla','Cline','N' +'AAAAAAAAOFLCAAAA','James','Taylor','N' +'AAAAAAAAOPDLAAAA','Ann','Pence','N' +'AAAAAAAAPDFBAAAA','Terrance','Banks','Y' +'AAAAAAAAPEHEBAAA','Edith','Molina','Y' +'AAAAAAAAPFCLAAAA','Felicia','Neville','N' +'AAAAAAAAPICEAAAA','Jennifer','Cortez','Y' +'AAAAAAAAPJENAAAA','Ashley','Norton','Y' +'AAAAAAAAPKBCBAAA','Andrea','White','N' +'AAAAAAAAPKIKAAAA','Wendy','Horvath','Y' +'AAAAAAAAPMMBBAAA','Paul','Jordan','N' +'AAAAAAAAPPIBBAAA','Candice','Lee','Y' +---- TYPES +STRING, STRING, STRING, STRING +---- RUNTIME_PROFILE +row_regex:.*21:UNION * 2 * 2 .* +row_regex:.*00:UNION * 3 * 12 .* +row_regex:.*14:UNION * 2 * 2 .* +row_regex:.*07:UNION * 3 * 12 .* +====
