Repository: incubator-impala Updated Branches: refs/heads/master d0d0af278 -> f9b222e92
IMPALA-5641: mem-estimate should never be less than mem-reservation This is implemented in the ResourceProfileBuilder to avoid duplicating the login in every plan node. Testing: Updated planner tests. Change-Id: I1e2853300371e31b13d81a763dbafb21709b16c4 Reviewed-on: http://gerrit.cloudera.org:8080/7703 Reviewed-by: Tim Armstrong <[email protected]> Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/5ec40464 Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/5ec40464 Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/5ec40464 Branch: refs/heads/master Commit: 5ec404640a4aea64d086b9828a317a424f8b5f65 Parents: d0d0af2 Author: Tim Armstrong <[email protected]> Authored: Mon Aug 14 11:28:06 2017 -0700 Committer: Impala Public Jenkins <[email protected]> Committed: Tue Aug 22 07:28:10 2017 +0000 ---------------------------------------------------------------------- .../apache/impala/planner/ResourceProfile.java | 6 +- .../queries/PlannerTest/constant-folding.test | 14 +- .../queries/PlannerTest/disable-codegen.test | 2 +- .../PlannerTest/fk-pk-join-detection.test | 40 ++-- .../queries/PlannerTest/mt-dop-validation.test | 20 +- .../PlannerTest/resource-requirements.test | 224 +++++++++---------- .../PlannerTest/sort-expr-materialization.test | 30 +-- .../PlannerTest/spillable-buffer-sizing.test | 44 ++-- .../queries/PlannerTest/tablesample.test | 4 +- 9 files changed, 193 insertions(+), 191 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/5ec40464/fe/src/main/java/org/apache/impala/planner/ResourceProfile.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/planner/ResourceProfile.java b/fe/src/main/java/org/apache/impala/planner/ResourceProfile.java index 3c13812..d2d0f70 100644 --- a/fe/src/main/java/org/apache/impala/planner/ResourceProfile.java +++ b/fe/src/main/java/org/apache/impala/planner/ResourceProfile.java @@ -29,7 +29,8 @@ public class ResourceProfile { // If the computed values are valid. private final boolean isValid_; - // Estimated memory consumption in bytes. + // Estimated memory consumption in bytes. Guaranteed to be >= minReservationBytes_ if + // both are set (the constructor ensures this). // TODO: IMPALA-5013: currently we are inconsistent about how these estimates are // derived or what they mean. Re-evaluate what they mean and either deprecate or // fix them. @@ -51,7 +52,8 @@ public class ResourceProfile { private ResourceProfile(boolean isValid, long memEstimateBytes, long minReservationBytes, long maxReservationBytes, long spillableBufferBytes) { isValid_ = isValid; - memEstimateBytes_ = memEstimateBytes; + memEstimateBytes_ = (minReservationBytes != -1) ? + Math.max(memEstimateBytes, minReservationBytes) : memEstimateBytes; minReservationBytes_ = minReservationBytes; maxReservationBytes_ = maxReservationBytes; spillableBufferBytes_ = spillableBufferBytes; http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/5ec40464/testdata/workloads/functional-planner/queries/PlannerTest/constant-folding.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/constant-folding.test b/testdata/workloads/functional-planner/queries/PlannerTest/constant-folding.test index f4ae6c3..888c7c6 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/constant-folding.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/constant-folding.test @@ -129,7 +129,7 @@ left outer join functional.alltypes b where round(1.11 + 2.22 + 3.33 + 4.44, 1) < cast(b.double_col as decimal(3, 2)) ---- PLAN F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=256.02MB mem-reservation=1.06MB +| Per-Host Resources: mem-estimate=257.06MB mem-reservation=1.06MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | @@ -138,7 +138,7 @@ PLAN-ROOT SINK | fk/pk conjuncts: assumed fk/pk | other join predicates: a.int_col <= b.bigint_col + 97, a.int_col >= 0 + b.bigint_col | other predicates: CAST(b.double_col AS DECIMAL(3,2)) > 11.1 -| mem-estimate=15.68KB mem-reservation=1.06MB spill-buffer=64.00KB +| mem-estimate=1.06MB mem-reservation=1.06MB spill-buffer=64.00KB | tuple-ids=0,1N row-size=28B cardinality=7300 | |--01:SCAN HDFS [functional.alltypes b] @@ -265,7 +265,7 @@ select first_value(1 + 1 + int_col - (1 - 1)) over from functional.alltypes ---- PLAN F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=130.00MB mem-reservation=16.00MB +| Per-Host Resources: mem-estimate=144.00MB mem-reservation=16.00MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | @@ -274,13 +274,13 @@ PLAN-ROOT SINK | partition by: concat('ab', string_col) | order by: greatest(20, bigint_col) ASC | window: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW -| mem-estimate=0B mem-reservation=4.00MB spill-buffer=2.00MB +| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB | tuple-ids=3,2 row-size=61B cardinality=7300 | 01:SORT | order by: concat('ab', string_col) ASC NULLS FIRST, greatest(20, bigint_col) ASC | materialized: concat('ab', string_col), greatest(20, bigint_col) -| mem-estimate=2.00MB mem-reservation=12.00MB spill-buffer=2.00MB +| mem-estimate=12.00MB mem-reservation=12.00MB spill-buffer=2.00MB | tuple-ids=3 row-size=53B cardinality=7300 | 00:SCAN HDFS [functional.alltypes] @@ -296,13 +296,13 @@ select int_col from functional.alltypes order by id * abs((factorial(5) / power(2, 4))) ---- PLAN F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=130.00MB mem-reservation=6.00MB +| Per-Host Resources: mem-estimate=134.00MB mem-reservation=6.00MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | 01:SORT | order by: id * 7.5 ASC -| mem-estimate=2.00MB mem-reservation=6.00MB spill-buffer=2.00MB +| mem-estimate=6.00MB mem-reservation=6.00MB spill-buffer=2.00MB | tuple-ids=1 row-size=8B cardinality=7300 | 00:SCAN HDFS [functional.alltypes] http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/5ec40464/testdata/workloads/functional-planner/queries/PlannerTest/disable-codegen.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/disable-codegen.test b/testdata/workloads/functional-planner/queries/PlannerTest/disable-codegen.test index b68ab20..04241c7 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/disable-codegen.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/disable-codegen.test @@ -79,7 +79,7 @@ from functional.alltypes t1 join functional.alltypestiny t2 on t1.id = t2.id ---- DISTRIBUTEDPLAN Max Per-Host Resource Reservation: Memory=1.06MB -Per-Host Resource Estimates: Memory=180.00MB +Per-Host Resource Estimates: Memory=181.06MB Codegen disabled by planner PLAN-ROOT SINK http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/5ec40464/testdata/workloads/functional-planner/queries/PlannerTest/fk-pk-join-detection.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/fk-pk-join-detection.test b/testdata/workloads/functional-planner/queries/PlannerTest/fk-pk-join-detection.test index 54ce9b0..632881a 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/fk-pk-join-detection.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/fk-pk-join-detection.test @@ -5,7 +5,7 @@ on ss_customer_sk = c_customer_sk where c_salutation = 'Mrs.' ---- PLAN F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=180.46MB mem-reservation=8.50MB +| Per-Host Resources: mem-estimate=184.50MB mem-reservation=8.50MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | @@ -13,7 +13,7 @@ PLAN-ROOT SINK | hash predicates: ss_customer_sk = c_customer_sk | fk/pk conjuncts: ss_customer_sk = c_customer_sk | runtime filters: RF000 <- c_customer_sk -| mem-estimate=4.46MB mem-reservation=8.50MB spill-buffer=512.00KB +| mem-estimate=8.50MB mem-reservation=8.50MB spill-buffer=512.00KB | tuple-ids=0,1 row-size=355B cardinality=529700 | |--01:SCAN HDFS [tpcds.customer] @@ -43,7 +43,7 @@ on ss_customer_sk = c_customer_sk where c_salutation = 'Mrs.' ---- PLAN F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=180.46MB mem-reservation=8.50MB +| Per-Host Resources: mem-estimate=184.50MB mem-reservation=8.50MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | @@ -51,7 +51,7 @@ PLAN-ROOT SINK | hash predicates: ss_customer_sk = c_customer_sk | fk/pk conjuncts: ss_customer_sk = c_customer_sk | other predicates: c_salutation = 'Mrs.' -| mem-estimate=4.46MB mem-reservation=8.50MB spill-buffer=512.00KB +| mem-estimate=8.50MB mem-reservation=8.50MB spill-buffer=512.00KB | tuple-ids=0,1N row-size=355B cardinality=2880404 | |--01:SCAN HDFS [tpcds.customer] @@ -80,7 +80,7 @@ on ss_customer_sk = c_customer_sk where c_salutation = 'Mrs.' ---- PLAN F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=180.46MB mem-reservation=8.50MB +| Per-Host Resources: mem-estimate=184.50MB mem-reservation=8.50MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | @@ -88,7 +88,7 @@ PLAN-ROOT SINK | hash predicates: ss_customer_sk = c_customer_sk | fk/pk conjuncts: ss_customer_sk = c_customer_sk | runtime filters: RF000 <- c_customer_sk -| mem-estimate=4.46MB mem-reservation=8.50MB spill-buffer=512.00KB +| mem-estimate=8.50MB mem-reservation=8.50MB spill-buffer=512.00KB | tuple-ids=0N,1 row-size=355B cardinality=529700 | |--01:SCAN HDFS [tpcds.customer] @@ -117,7 +117,7 @@ on ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number where sr_return_quantity < 10 ---- PLAN F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=210.65MB mem-reservation=4.25MB +| Per-Host Resources: mem-estimate=212.25MB mem-reservation=4.25MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | @@ -125,7 +125,7 @@ PLAN-ROOT SINK | hash predicates: ss_item_sk = sr_item_sk, ss_ticket_number = sr_ticket_number | fk/pk conjuncts: ss_item_sk = sr_item_sk, ss_ticket_number = sr_ticket_number | runtime filters: RF000 <- sr_item_sk, RF001 <- sr_ticket_number -| mem-estimate=2.65MB mem-reservation=4.25MB spill-buffer=256.00KB +| mem-estimate=4.25MB mem-reservation=4.25MB spill-buffer=256.00KB | tuple-ids=0,1 row-size=188B cardinality=211838 | |--01:SCAN HDFS [tpcds.store_returns] @@ -188,7 +188,7 @@ on a.d_date_sk = b.d_date_sk where a.d_holiday = "Y" ---- PLAN F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=107.62MB mem-reservation=17.00MB +| Per-Host Resources: mem-estimate=113.00MB mem-reservation=17.00MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | @@ -196,7 +196,7 @@ PLAN-ROOT SINK | hash predicates: b.d_date_sk = a.d_date_sk | fk/pk conjuncts: b.d_date_sk = a.d_date_sk | runtime filters: RF000 <- a.d_date_sk -| mem-estimate=11.62MB mem-reservation=17.00MB spill-buffer=1.00MB +| mem-estimate=17.00MB mem-reservation=17.00MB spill-buffer=1.00MB | tuple-ids=1,0 row-size=606B cardinality=36525 | |--00:SCAN HDFS [tpcds.date_dim a] @@ -229,7 +229,7 @@ where ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number and d1.d_fy_week_seq = 1000 ---- PLAN F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=352.73MB mem-reservation=4.25MB +| Per-Host Resources: mem-estimate=355.19MB mem-reservation=4.25MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | @@ -237,7 +237,7 @@ PLAN-ROOT SINK | hash predicates: ss_addr_sk = c_current_addr_sk | fk/pk conjuncts: none | runtime filters: RF000 <- c_current_addr_sk -| mem-estimate=429.69KB mem-reservation=1.06MB spill-buffer=64.00KB +| mem-estimate=1.06MB mem-reservation=1.06MB spill-buffer=64.00KB | tuple-ids=1,0,3,4,2 row-size=60B cardinality=19358 | |--02:SCAN HDFS [tpcds.customer] @@ -252,7 +252,7 @@ PLAN-ROOT SINK | hash predicates: sr_returned_date_sk = d2.d_date_sk | fk/pk conjuncts: sr_returned_date_sk = d2.d_date_sk | runtime filters: RF001 <- d2.d_date_sk -| mem-estimate=313.88KB mem-reservation=1.06MB spill-buffer=64.00KB +| mem-estimate=1.06MB mem-reservation=1.06MB spill-buffer=64.00KB | tuple-ids=1,0,3,4 row-size=56B cardinality=8131 | |--04:SCAN HDFS [tpcds.date_dim d2] @@ -267,14 +267,14 @@ PLAN-ROOT SINK | hash predicates: sr_item_sk = ss_item_sk, sr_ticket_number = ss_ticket_number | fk/pk conjuncts: sr_item_sk = ss_item_sk, sr_ticket_number = ss_ticket_number | runtime filters: RF002 <- ss_item_sk, RF003 <- ss_ticket_number -| mem-estimate=380.02KB mem-reservation=1.06MB spill-buffer=64.00KB +| mem-estimate=1.06MB mem-reservation=1.06MB spill-buffer=64.00KB | tuple-ids=1,0,3 row-size=52B cardinality=8131 | |--05:HASH JOIN [INNER JOIN] | | hash predicates: ss_sold_date_sk = d1.d_date_sk | | fk/pk conjuncts: ss_sold_date_sk = d1.d_date_sk | | runtime filters: RF004 <- d1.d_date_sk -| | mem-estimate=62B mem-reservation=1.06MB spill-buffer=64.00KB +| | mem-estimate=1.06MB mem-reservation=1.06MB spill-buffer=64.00KB | | tuple-ids=0,3 row-size=32B cardinality=11055 | | | |--03:SCAN HDFS [tpcds.date_dim d1] @@ -311,7 +311,7 @@ tpcds.store_sales inner join tpcds.customer on ss_customer_sk % 10 = c_customer_sk / 100 ---- PLAN F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=202.79MB mem-reservation=34.00MB +| Per-Host Resources: mem-estimate=210.00MB mem-reservation=34.00MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | @@ -319,7 +319,7 @@ PLAN-ROOT SINK | hash predicates: ss_customer_sk % 10 = c_customer_sk / 100 | fk/pk conjuncts: assumed fk/pk | runtime filters: RF000 <- c_customer_sk / 100 -| mem-estimate=26.79MB mem-reservation=34.00MB spill-buffer=2.00MB +| mem-estimate=34.00MB mem-reservation=34.00MB spill-buffer=2.00MB | tuple-ids=0,1 row-size=355B cardinality=2880404 | |--01:SCAN HDFS [tpcds.customer] @@ -380,7 +380,7 @@ tpcds_seq_snap.store_sales inner join tpcds.customer on ss_customer_sk = c_customer_sk ---- PLAN F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=176.42MB mem-reservation=1.06MB +| Per-Host Resources: mem-estimate=177.06MB mem-reservation=1.06MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | @@ -388,7 +388,7 @@ PLAN-ROOT SINK | hash predicates: ss_customer_sk = c_customer_sk | fk/pk conjuncts: assumed fk/pk | runtime filters: RF000 <- c_customer_sk -| mem-estimate=429.69KB mem-reservation=1.06MB spill-buffer=64.00KB +| mem-estimate=1.06MB mem-reservation=1.06MB spill-buffer=64.00KB | tuple-ids=0,1 row-size=8B cardinality=unavailable | |--01:SCAN HDFS [tpcds.customer] @@ -424,7 +424,7 @@ PLAN-ROOT SINK | hash predicates: ss_sold_time_sk = ws_sold_time_sk | fk/pk conjuncts: none | runtime filters: RF000 <- ws_sold_time_sk -| mem-estimate=170.89KB mem-reservation=1.06MB spill-buffer=64.00KB +| mem-estimate=1.06MB mem-reservation=1.06MB spill-buffer=64.00KB | tuple-ids=0,2 row-size=104B cardinality=2440073 | |--02:AGGREGATE [FINALIZE] http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/5ec40464/testdata/workloads/functional-planner/queries/PlannerTest/mt-dop-validation.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/mt-dop-validation.test b/testdata/workloads/functional-planner/queries/PlannerTest/mt-dop-validation.test index 8bd09be..131c038 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/mt-dop-validation.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/mt-dop-validation.test @@ -119,7 +119,7 @@ from functional_parquet.alltypes where id < 10 ---- PLAN F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=16.00MB mem-reservation=10.00MB +| Per-Host Resources: mem-estimate=26.00MB mem-reservation=10.00MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | @@ -128,12 +128,12 @@ PLAN-ROOT SINK | partition by: int_col | order by: id ASC | window: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW -| mem-estimate=0B mem-reservation=4.00MB spill-buffer=2.00MB +| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB | tuple-ids=4,3 row-size=16B cardinality=unavailable | 01:SORT | order by: int_col ASC NULLS FIRST, id ASC -| mem-estimate=0B mem-reservation=6.00MB spill-buffer=2.00MB +| mem-estimate=6.00MB mem-reservation=6.00MB spill-buffer=2.00MB | tuple-ids=4 row-size=8B cardinality=unavailable | 00:SCAN HDFS [functional_parquet.alltypes] @@ -157,18 +157,18 @@ PLAN-ROOT SINK | tuple-ids=4,3 row-size=16B cardinality=unavailable | F01:PLAN FRAGMENT [HASH(int_col)] hosts=3 instances=9 -Per-Host Resources: mem-estimate=0B mem-reservation=30.00MB +Per-Host Resources: mem-estimate=30.00MB mem-reservation=30.00MB 02:ANALYTIC | functions: row_number() | partition by: int_col | order by: id ASC | window: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW -| mem-estimate=0B mem-reservation=4.00MB spill-buffer=2.00MB +| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB | tuple-ids=4,3 row-size=16B cardinality=unavailable | 01:SORT | order by: int_col ASC NULLS FIRST, id ASC -| mem-estimate=0B mem-reservation=6.00MB spill-buffer=2.00MB +| mem-estimate=6.00MB mem-reservation=6.00MB spill-buffer=2.00MB | tuple-ids=4 row-size=8B cardinality=unavailable | 03:EXCHANGE [HASH(int_col)] @@ -313,7 +313,7 @@ from tpch_nested_parquet.customer c, c.c_orders o1, c.c_orders o2 where o1.o_orderkey = o2.o_orderkey + 2 and o1.o_orderkey < 5 ---- PLAN F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=88.00MB mem-reservation=1.06MB +| Per-Host Resources: mem-estimate=89.06MB mem-reservation=1.06MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | @@ -324,7 +324,7 @@ PLAN-ROOT SINK |--06:HASH JOIN [INNER JOIN] | | hash predicates: o1.o_orderkey = o2.o_orderkey + 2 | | fk/pk conjuncts: assumed fk/pk -| | mem-estimate=0B mem-reservation=1.06MB spill-buffer=64.00KB +| | mem-estimate=1.06MB mem-reservation=1.06MB spill-buffer=64.00KB | | tuple-ids=1,0,2 row-size=286B cardinality=10 | | | |--04:UNNEST [c.c_orders o2] @@ -366,7 +366,7 @@ PLAN-ROOT SINK | tuple-ids=1,0,2 row-size=286B cardinality=1500000 | F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=9 -Per-Host Resources: mem-estimate=264.00MB mem-reservation=3.19MB +Per-Host Resources: mem-estimate=267.19MB mem-reservation=3.19MB 01:SUBPLAN | mem-estimate=0B mem-reservation=0B | tuple-ids=1,0,2 row-size=286B cardinality=1500000 @@ -374,7 +374,7 @@ Per-Host Resources: mem-estimate=264.00MB mem-reservation=3.19MB |--06:HASH JOIN [INNER JOIN] | | hash predicates: o1.o_orderkey = o2.o_orderkey + 2 | | fk/pk conjuncts: assumed fk/pk -| | mem-estimate=0B mem-reservation=1.06MB spill-buffer=64.00KB +| | mem-estimate=1.06MB mem-reservation=1.06MB spill-buffer=64.00KB | | tuple-ids=1,0,2 row-size=286B cardinality=10 | | | |--04:UNNEST [c.c_orders o2] http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/5ec40464/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test b/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test index 320b2cd..73129ec 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test @@ -354,17 +354,17 @@ from tpch_parquet.lineitem group by l_orderkey ---- PLAN Max Per-Host Resource Reservation: Memory=34.00MB -Per-Host Resource Estimates: Memory=106.24MB +Per-Host Resource Estimates: Memory=114.00MB F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=106.24MB mem-reservation=34.00MB +| Per-Host Resources: mem-estimate=114.00MB mem-reservation=34.00MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | 01:AGGREGATE [FINALIZE] | output: count(*) | group by: l_orderkey -| mem-estimate=26.24MB mem-reservation=34.00MB spill-buffer=2.00MB +| mem-estimate=34.00MB mem-reservation=34.00MB spill-buffer=2.00MB | tuple-ids=1 row-size=16B cardinality=1563438 | 00:SCAN HDFS [tpch_parquet.lineitem] @@ -1152,23 +1152,23 @@ select max(tinyint_col) over(partition by int_col) from functional.alltypes ---- PLAN Max Per-Host Resource Reservation: Memory=10.00MB -Per-Host Resource Estimates: Memory=18.00MB +Per-Host Resource Estimates: Memory=26.00MB Codegen disabled by planner F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=18.00MB mem-reservation=10.00MB +| Per-Host Resources: mem-estimate=26.00MB mem-reservation=10.00MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | 02:ANALYTIC | functions: max(tinyint_col) | partition by: int_col -| mem-estimate=0B mem-reservation=4.00MB spill-buffer=2.00MB +| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB | tuple-ids=3,2 row-size=6B cardinality=7300 | 01:SORT | order by: int_col ASC NULLS FIRST -| mem-estimate=2.00MB mem-reservation=6.00MB spill-buffer=2.00MB +| mem-estimate=6.00MB mem-reservation=6.00MB spill-buffer=2.00MB | tuple-ids=3 row-size=5B cardinality=7300 | 00:SCAN HDFS [functional.alltypes] @@ -1180,7 +1180,7 @@ PLAN-ROOT SINK tuple-ids=0 row-size=5B cardinality=7300 ---- DISTRIBUTEDPLAN Max Per-Host Resource Reservation: Memory=10.00MB -Per-Host Resource Estimates: Memory=18.00MB +Per-Host Resource Estimates: Memory=26.00MB Codegen disabled by planner F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 @@ -1193,16 +1193,16 @@ PLAN-ROOT SINK | tuple-ids=3,2 row-size=6B cardinality=7300 | F01:PLAN FRAGMENT [HASH(int_col)] hosts=3 instances=3 -Per-Host Resources: mem-estimate=2.00MB mem-reservation=10.00MB +Per-Host Resources: mem-estimate=10.00MB mem-reservation=10.00MB 02:ANALYTIC | functions: max(tinyint_col) | partition by: int_col -| mem-estimate=0B mem-reservation=4.00MB spill-buffer=2.00MB +| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB | tuple-ids=3,2 row-size=6B cardinality=7300 | 01:SORT | order by: int_col ASC NULLS FIRST -| mem-estimate=2.00MB mem-reservation=6.00MB spill-buffer=2.00MB +| mem-estimate=6.00MB mem-reservation=6.00MB spill-buffer=2.00MB | tuple-ids=3 row-size=5B cardinality=7300 | 03:EXCHANGE [HASH(int_col)] @@ -1220,7 +1220,7 @@ Per-Host Resources: mem-estimate=16.00MB mem-reservation=0B tuple-ids=0 row-size=5B cardinality=7300 ---- PARALLELPLANS Max Per-Host Resource Reservation: Memory=20.00MB -Per-Host Resource Estimates: Memory=36.00MB +Per-Host Resource Estimates: Memory=52.00MB Codegen disabled by planner F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 @@ -1233,16 +1233,16 @@ PLAN-ROOT SINK | tuple-ids=3,2 row-size=6B cardinality=7300 | F01:PLAN FRAGMENT [HASH(int_col)] hosts=3 instances=6 -Per-Host Resources: mem-estimate=4.00MB mem-reservation=20.00MB +Per-Host Resources: mem-estimate=20.00MB mem-reservation=20.00MB 02:ANALYTIC | functions: max(tinyint_col) | partition by: int_col -| mem-estimate=0B mem-reservation=4.00MB spill-buffer=2.00MB +| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB | tuple-ids=3,2 row-size=6B cardinality=7300 | 01:SORT | order by: int_col ASC NULLS FIRST -| mem-estimate=2.00MB mem-reservation=6.00MB spill-buffer=2.00MB +| mem-estimate=6.00MB mem-reservation=6.00MB spill-buffer=2.00MB | tuple-ids=3 row-size=5B cardinality=7300 | 03:EXCHANGE [HASH(int_col)] @@ -1267,10 +1267,10 @@ select *, row_number() over (order by o_totalprice) rnum_price, from tpch_parquet.orders ---- PLAN Max Per-Host Resource Reservation: Memory=36.00MB -Per-Host Resource Estimates: Memory=58.00MB +Per-Host Resource Estimates: Memory=70.00MB F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=58.00MB mem-reservation=36.00MB +| Per-Host Resources: mem-estimate=70.00MB mem-reservation=36.00MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | @@ -1278,7 +1278,7 @@ PLAN-ROOT SINK | functions: row_number() | order by: o_orderpriority ASC | window: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW -| mem-estimate=0B mem-reservation=4.00MB spill-buffer=2.00MB +| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB | tuple-ids=10,5 row-size=215B cardinality=1500000 | 05:SORT @@ -1290,7 +1290,7 @@ PLAN-ROOT SINK | functions: row_number() | order by: o_orderdate ASC | window: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW -| mem-estimate=0B mem-reservation=4.00MB spill-buffer=2.00MB +| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB | tuple-ids=8,4 row-size=207B cardinality=1500000 | 03:SORT @@ -1302,7 +1302,7 @@ PLAN-ROOT SINK | functions: row_number() | order by: o_totalprice ASC | window: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW -| mem-estimate=0B mem-reservation=4.00MB spill-buffer=2.00MB +| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB | tuple-ids=6,3 row-size=199B cardinality=1500000 | 01:SORT @@ -1319,10 +1319,10 @@ PLAN-ROOT SINK tuple-ids=0 row-size=191B cardinality=1500000 ---- DISTRIBUTEDPLAN Max Per-Host Resource Reservation: Memory=44.00MB -Per-Host Resource Estimates: Memory=94.00MB +Per-Host Resource Estimates: Memory=102.00MB F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=36.00MB mem-reservation=32.00MB +| Per-Host Resources: mem-estimate=44.00MB mem-reservation=32.00MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | @@ -1330,7 +1330,7 @@ PLAN-ROOT SINK | functions: row_number() | order by: o_orderpriority ASC | window: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW -| mem-estimate=0B mem-reservation=4.00MB spill-buffer=2.00MB +| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB | tuple-ids=10,5 row-size=215B cardinality=1500000 | 05:SORT @@ -1342,7 +1342,7 @@ PLAN-ROOT SINK | functions: row_number() | order by: o_orderdate ASC | window: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW -| mem-estimate=0B mem-reservation=4.00MB spill-buffer=2.00MB +| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB | tuple-ids=8,4 row-size=207B cardinality=1500000 | 03:SORT @@ -1354,7 +1354,7 @@ PLAN-ROOT SINK | functions: row_number() | order by: o_totalprice ASC | window: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW -| mem-estimate=0B mem-reservation=4.00MB spill-buffer=2.00MB +| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB | tuple-ids=6,3 row-size=199B cardinality=1500000 | 07:MERGING-EXCHANGE [UNPARTITIONED] @@ -1378,10 +1378,10 @@ Per-Host Resources: mem-estimate=58.00MB mem-reservation=12.00MB tuple-ids=0 row-size=191B cardinality=1500000 ---- PARALLELPLANS Max Per-Host Resource Reservation: Memory=56.00MB -Per-Host Resource Estimates: Memory=152.00MB +Per-Host Resource Estimates: Memory=160.00MB F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=36.00MB mem-reservation=32.00MB +| Per-Host Resources: mem-estimate=44.00MB mem-reservation=32.00MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | @@ -1389,7 +1389,7 @@ PLAN-ROOT SINK | functions: row_number() | order by: o_orderpriority ASC | window: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW -| mem-estimate=0B mem-reservation=4.00MB spill-buffer=2.00MB +| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB | tuple-ids=10,5 row-size=215B cardinality=1500000 | 05:SORT @@ -1401,7 +1401,7 @@ PLAN-ROOT SINK | functions: row_number() | order by: o_orderdate ASC | window: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW -| mem-estimate=0B mem-reservation=4.00MB spill-buffer=2.00MB +| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB | tuple-ids=8,4 row-size=207B cardinality=1500000 | 03:SORT @@ -1413,7 +1413,7 @@ PLAN-ROOT SINK | functions: row_number() | order by: o_totalprice ASC | window: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW -| mem-estimate=0B mem-reservation=4.00MB spill-buffer=2.00MB +| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB | tuple-ids=6,3 row-size=199B cardinality=1500000 | 07:MERGING-EXCHANGE [UNPARTITIONED] @@ -1450,10 +1450,10 @@ from tpch_parquet.lineitem join tpch_parquet.orders on l_orderkey = o_orderkey where l_shipmode = 'F' ---- PLAN Max Per-Host Resource Reservation: Memory=51.00MB -Per-Host Resource Estimates: Memory=135.17MB +Per-Host Resource Estimates: Memory=139.58MB F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=135.17MB mem-reservation=51.00MB +| Per-Host Resources: mem-estimate=139.58MB mem-reservation=51.00MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | @@ -1466,7 +1466,7 @@ PLAN-ROOT SINK | | hash predicates: l_orderkey = o_orderkey | | fk/pk conjuncts: l_orderkey = o_orderkey | | runtime filters: RF002 <- o_orderkey -| | mem-estimate=12.59MB mem-reservation=17.00MB spill-buffer=1.00MB +| | mem-estimate=17.00MB mem-reservation=17.00MB spill-buffer=1.00MB | | tuple-ids=5,6 row-size=99B cardinality=822530 | | | |--09:SCAN HDFS [tpch_parquet.orders] @@ -1493,7 +1493,7 @@ PLAN-ROOT SINK | | hash predicates: l_orderkey = o_orderkey | | fk/pk conjuncts: l_orderkey = o_orderkey | | runtime filters: RF001 <- o_orderkey -| | mem-estimate=10.20MB mem-reservation=17.00MB spill-buffer=1.00MB +| | mem-estimate=17.00MB mem-reservation=17.00MB spill-buffer=1.00MB | | tuple-ids=3,4 row-size=103B cardinality=1151542 | | | |--06:SCAN HDFS [tpch_parquet.orders] @@ -1525,7 +1525,7 @@ PLAN-ROOT SINK | hash predicates: l_orderkey = o_orderkey | fk/pk conjuncts: l_orderkey = o_orderkey | runtime filters: RF000 <- o_orderkey -| mem-estimate=12.59MB mem-reservation=17.00MB spill-buffer=1.00MB +| mem-estimate=17.00MB mem-reservation=17.00MB spill-buffer=1.00MB | tuple-ids=0,1 row-size=86B cardinality=575772 | |--02:SCAN HDFS [tpch_parquet.orders] @@ -1549,7 +1549,7 @@ PLAN-ROOT SINK tuple-ids=0 row-size=78B cardinality=600122 ---- DISTRIBUTEDPLAN Max Per-Host Resource Reservation: Memory=38.25MB -Per-Host Resource Estimates: Memory=339.36MB +Per-Host Resource Estimates: Memory=343.83MB F09:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 | Per-Host Resources: mem-estimate=0B mem-reservation=0B @@ -1561,7 +1561,7 @@ PLAN-ROOT SINK | tuple-ids=7 row-size=70B cardinality=2549844 | F08:PLAN FRAGMENT [RANDOM] hosts=3 instances=3 -Per-Host Resources: mem-estimate=92.59MB mem-reservation=34.00MB +Per-Host Resources: mem-estimate=97.00MB mem-reservation=34.00MB 00:UNION | pass-through-operands: 14 | mem-estimate=0B mem-reservation=0B @@ -1571,7 +1571,7 @@ Per-Host Resources: mem-estimate=92.59MB mem-reservation=34.00MB | | hash predicates: l_orderkey = o_orderkey | | fk/pk conjuncts: l_orderkey = o_orderkey | | runtime filters: RF002 <- o_orderkey -| | mem-estimate=12.59MB mem-reservation=17.00MB spill-buffer=1.00MB +| | mem-estimate=17.00MB mem-reservation=17.00MB spill-buffer=1.00MB | | tuple-ids=5,6 row-size=99B cardinality=822530 | | | |--16:EXCHANGE [BROADCAST] @@ -1604,7 +1604,7 @@ Per-Host Resources: mem-estimate=92.59MB mem-reservation=34.00MB | | hash predicates: l_orderkey = o_orderkey | | fk/pk conjuncts: l_orderkey = o_orderkey | | runtime filters: RF001 <- o_orderkey -| | mem-estimate=10.20MB mem-reservation=17.00MB spill-buffer=1.00MB +| | mem-estimate=17.00MB mem-reservation=17.00MB spill-buffer=1.00MB | | tuple-ids=3,4 row-size=103B cardinality=1151542 | | | |--15:EXCHANGE [BROADCAST] @@ -1643,7 +1643,7 @@ Per-Host Resources: mem-estimate=92.59MB mem-reservation=34.00MB | tuple-ids=2 row-size=70B cardinality=575772 | F02:PLAN FRAGMENT [HASH(l_orderkey)] hosts=3 instances=3 -Per-Host Resources: mem-estimate=46.78MB mem-reservation=4.25MB +Per-Host Resources: mem-estimate=46.83MB mem-reservation=4.25MB 04:AGGREGATE [STREAMING] | group by: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_comment | mem-estimate=42.58MB mem-reservation=0B spill-buffer=2.00MB @@ -1653,7 +1653,7 @@ Per-Host Resources: mem-estimate=46.78MB mem-reservation=4.25MB | hash predicates: l_orderkey = o_orderkey | fk/pk conjuncts: l_orderkey = o_orderkey | runtime filters: RF000 <- o_orderkey -| mem-estimate=4.20MB mem-reservation=4.25MB spill-buffer=256.00KB +| mem-estimate=4.25MB mem-reservation=4.25MB spill-buffer=256.00KB | tuple-ids=0,1 row-size=86B cardinality=575772 | |--12:EXCHANGE [HASH(o_orderkey)] @@ -1689,7 +1689,7 @@ Per-Host Resources: mem-estimate=80.00MB mem-reservation=0B tuple-ids=0 row-size=78B cardinality=600122 ---- PARALLELPLANS Max Per-Host Resource Reservation: Memory=72.25MB -Per-Host Resource Estimates: Memory=674.53MB +Per-Host Resource Estimates: Memory=683.41MB F09:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 | Per-Host Resources: mem-estimate=0B mem-reservation=0B @@ -1701,7 +1701,7 @@ PLAN-ROOT SINK | tuple-ids=7 row-size=70B cardinality=2549844 | F08:PLAN FRAGMENT [RANDOM] hosts=3 instances=6 -Per-Host Resources: mem-estimate=185.18MB mem-reservation=68.00MB +Per-Host Resources: mem-estimate=194.00MB mem-reservation=68.00MB 00:UNION | pass-through-operands: 14 | mem-estimate=0B mem-reservation=0B @@ -1712,7 +1712,7 @@ Per-Host Resources: mem-estimate=185.18MB mem-reservation=68.00MB | | hash predicates: l_orderkey = o_orderkey | | fk/pk conjuncts: l_orderkey = o_orderkey | | runtime filters: RF002 <- o_orderkey -| | mem-estimate=12.59MB mem-reservation=17.00MB spill-buffer=1.00MB +| | mem-estimate=17.00MB mem-reservation=17.00MB spill-buffer=1.00MB | | tuple-ids=5,6 row-size=99B cardinality=822530 | | | |--F11:PLAN FRAGMENT [RANDOM] hosts=2 instances=4 @@ -1753,7 +1753,7 @@ Per-Host Resources: mem-estimate=185.18MB mem-reservation=68.00MB | | hash predicates: l_orderkey = o_orderkey | | fk/pk conjuncts: l_orderkey = o_orderkey | | runtime filters: RF001 <- o_orderkey -| | mem-estimate=10.20MB mem-reservation=17.00MB spill-buffer=1.00MB +| | mem-estimate=17.00MB mem-reservation=17.00MB spill-buffer=1.00MB | | tuple-ids=3,4 row-size=103B cardinality=1151542 | | | |--F10:PLAN FRAGMENT [RANDOM] hosts=2 instances=4 @@ -1799,7 +1799,7 @@ Per-Host Resources: mem-estimate=185.18MB mem-reservation=68.00MB | tuple-ids=2 row-size=70B cardinality=575772 | F02:PLAN FRAGMENT [HASH(l_orderkey)] hosts=3 instances=6 -Per-Host Resources: mem-estimate=89.35MB mem-reservation=4.25MB +Per-Host Resources: mem-estimate=89.41MB mem-reservation=4.25MB 04:AGGREGATE [STREAMING] | group by: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_comment | mem-estimate=42.58MB mem-reservation=0B spill-buffer=2.00MB @@ -1810,7 +1810,7 @@ Per-Host Resources: mem-estimate=89.35MB mem-reservation=4.25MB | hash predicates: l_orderkey = o_orderkey | fk/pk conjuncts: l_orderkey = o_orderkey | runtime filters: RF000 <- o_orderkey -| mem-estimate=2.10MB mem-reservation=2.12MB spill-buffer=128.00KB +| mem-estimate=2.12MB mem-reservation=2.12MB spill-buffer=128.00KB | tuple-ids=0,1 row-size=86B cardinality=575772 | |--F12:PLAN FRAGMENT [HASH(l_orderkey)] hosts=2 instances=4 @@ -1889,10 +1889,10 @@ order by limit 100 ---- PLAN Max Per-Host Resource Reservation: Memory=80.75MB -Per-Host Resource Estimates: Memory=391.29MB +Per-Host Resource Estimates: Memory=393.18MB F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=391.29MB mem-reservation=80.75MB +| Per-Host Resources: mem-estimate=393.18MB mem-reservation=80.75MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | @@ -1910,7 +1910,7 @@ PLAN-ROOT SINK 07:HASH JOIN [LEFT SEMI JOIN] | hash predicates: o_orderkey = l_orderkey | runtime filters: RF000 <- l_orderkey -| mem-estimate=3.94MB mem-reservation=4.25MB spill-buffer=256.00KB +| mem-estimate=4.25MB mem-reservation=4.25MB spill-buffer=256.00KB | tuple-ids=2,1,0 row-size=108B cardinality=575772 | |--04:AGGREGATE [FINALIZE] @@ -1932,7 +1932,7 @@ PLAN-ROOT SINK | hash predicates: o_custkey = c_custkey | fk/pk conjuncts: o_custkey = c_custkey | runtime filters: RF001 <- c_custkey -| mem-estimate=6.61MB mem-reservation=8.50MB spill-buffer=512.00KB +| mem-estimate=8.50MB mem-reservation=8.50MB spill-buffer=512.00KB | tuple-ids=2,1,0 row-size=108B cardinality=5757710 | |--00:SCAN HDFS [tpch.customer] @@ -1969,7 +1969,7 @@ PLAN-ROOT SINK tuple-ids=2 row-size=16B cardinality=6001215 ---- DISTRIBUTEDPLAN Max Per-Host Resource Reservation: Memory=82.88MB -Per-Host Resource Estimates: Memory=500.32MB +Per-Host Resource Estimates: Memory=510.80MB F07:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 | Per-Host Resources: mem-estimate=0B mem-reservation=0B @@ -2000,7 +2000,7 @@ Per-Host Resources: mem-estimate=60.41MB mem-reservation=34.00MB | tuple-ids=6 row-size=100B cardinality=575772 | F02:PLAN FRAGMENT [HASH(l_orderkey)] hosts=3 instances=3 -Per-Host Resources: mem-estimate=104.55MB mem-reservation=48.88MB +Per-Host Resources: mem-estimate=115.03MB mem-reservation=48.88MB 08:AGGREGATE [STREAMING] | output: sum(l_quantity) | group by: c_name, c_custkey, o_orderkey, o_orderdate, o_totalprice @@ -2010,7 +2010,7 @@ Per-Host Resources: mem-estimate=104.55MB mem-reservation=48.88MB 07:HASH JOIN [LEFT SEMI JOIN, PARTITIONED] | hash predicates: o_orderkey = l_orderkey | runtime filters: RF000 <- l_orderkey -| mem-estimate=1.31MB mem-reservation=2.12MB spill-buffer=128.00KB +| mem-estimate=2.12MB mem-reservation=2.12MB spill-buffer=128.00KB | tuple-ids=2,1,0 row-size=108B cardinality=575772 | |--14:AGGREGATE [FINALIZE] @@ -2044,7 +2044,7 @@ Per-Host Resources: mem-estimate=104.55MB mem-reservation=48.88MB | hash predicates: o_custkey = c_custkey | fk/pk conjuncts: o_custkey = c_custkey | runtime filters: RF001 <- c_custkey -| mem-estimate=6.61MB mem-reservation=8.50MB spill-buffer=512.00KB +| mem-estimate=8.50MB mem-reservation=8.50MB spill-buffer=512.00KB | tuple-ids=2,1,0 row-size=108B cardinality=5757710 | |--12:EXCHANGE [BROADCAST] @@ -2065,7 +2065,7 @@ Per-Host Resources: mem-estimate=104.55MB mem-reservation=48.88MB | hash predicates: l_orderkey = o_orderkey | fk/pk conjuncts: l_orderkey = o_orderkey | runtime filters: RF002 <- o_orderkey -| mem-estimate=26.23MB mem-reservation=34.00MB spill-buffer=2.00MB +| mem-estimate=34.00MB mem-reservation=34.00MB spill-buffer=2.00MB | tuple-ids=2,1 row-size=66B cardinality=5757710 | |--11:EXCHANGE [HASH(o_orderkey)] @@ -2099,7 +2099,7 @@ Per-Host Resources: mem-estimate=88.00MB mem-reservation=0B tuple-ids=2 row-size=16B cardinality=6001215 ---- PARALLELPLANS Max Per-Host Resource Reservation: Memory=121.12MB -Per-Host Resource Estimates: Memory=953.10MB +Per-Host Resource Estimates: Memory=965.47MB F07:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 | Per-Host Resources: mem-estimate=0B mem-reservation=0B @@ -2130,7 +2130,7 @@ Per-Host Resources: mem-estimate=120.82MB mem-reservation=68.00MB | tuple-ids=6 row-size=100B cardinality=575772 | F02:PLAN FRAGMENT [HASH(l_orderkey)] hosts=3 instances=6 -Per-Host Resources: mem-estimate=161.56MB mem-reservation=53.12MB +Per-Host Resources: mem-estimate=173.93MB mem-reservation=53.12MB 08:AGGREGATE [STREAMING] | output: sum(l_quantity) | group by: c_name, c_custkey, o_orderkey, o_orderdate, o_totalprice @@ -2141,7 +2141,7 @@ Per-Host Resources: mem-estimate=161.56MB mem-reservation=53.12MB | hash-table-id=00 | hash predicates: o_orderkey = l_orderkey | runtime filters: RF000 <- l_orderkey -| mem-estimate=671.79KB mem-reservation=1.06MB spill-buffer=64.00KB +| mem-estimate=1.06MB mem-reservation=1.06MB spill-buffer=64.00KB | tuple-ids=2,1,0 row-size=108B cardinality=575772 | |--F08:PLAN FRAGMENT [HASH(l_orderkey)] hosts=3 instances=6 @@ -2183,7 +2183,7 @@ Per-Host Resources: mem-estimate=161.56MB mem-reservation=53.12MB | hash predicates: o_custkey = c_custkey | fk/pk conjuncts: o_custkey = c_custkey | runtime filters: RF001 <- c_custkey -| mem-estimate=6.61MB mem-reservation=8.50MB spill-buffer=512.00KB +| mem-estimate=8.50MB mem-reservation=8.50MB spill-buffer=512.00KB | tuple-ids=2,1,0 row-size=108B cardinality=5757710 | |--F09:PLAN FRAGMENT [HASH(l_orderkey)] hosts=1 instances=2 @@ -2212,7 +2212,7 @@ Per-Host Resources: mem-estimate=161.56MB mem-reservation=53.12MB | hash predicates: l_orderkey = o_orderkey | fk/pk conjuncts: l_orderkey = o_orderkey | runtime filters: RF002 <- o_orderkey -| mem-estimate=13.11MB mem-reservation=17.00MB spill-buffer=1.00MB +| mem-estimate=17.00MB mem-reservation=17.00MB spill-buffer=1.00MB | tuple-ids=2,1 row-size=66B cardinality=5757710 | |--F10:PLAN FRAGMENT [HASH(l_orderkey)] hosts=2 instances=4 @@ -2391,12 +2391,12 @@ from tpch_nested_parquet.customer c, order by o1.o_orderkey limit 100) v ---- PLAN Max Per-Host Resource Reservation: Memory=69.06MB -Per-Host Resource Estimates: Memory=344.00MB +Per-Host Resource Estimates: Memory=345.06MB WARNING: The following tables are missing relevant table and/or column statistics. tpch_nested_parquet.customer F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=344.00MB mem-reservation=69.06MB +| Per-Host Resources: mem-estimate=345.06MB mem-reservation=69.06MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | @@ -2431,7 +2431,7 @@ PLAN-ROOT SINK | 05:HASH JOIN [INNER JOIN] | | hash predicates: o1.o_orderkey = o2.o_orderkey | | fk/pk conjuncts: assumed fk/pk -| | mem-estimate=0B mem-reservation=1.06MB spill-buffer=64.00KB +| | mem-estimate=1.06MB mem-reservation=1.06MB spill-buffer=64.00KB | | tuple-ids=1,2 row-size=32B cardinality=10 | | | |--04:UNNEST [c.c_orders o2] @@ -2453,7 +2453,7 @@ PLAN-ROOT SINK tuple-ids=0 row-size=66B cardinality=150000 ---- DISTRIBUTEDPLAN Max Per-Host Resource Reservation: Memory=69.06MB -Per-Host Resource Estimates: Memory=472.00MB +Per-Host Resource Estimates: Memory=473.06MB WARNING: The following tables are missing relevant table and/or column statistics. tpch_nested_parquet.customer @@ -2478,7 +2478,7 @@ Per-Host Resources: mem-estimate=128.00MB mem-reservation=34.00MB | tuple-ids=6 row-size=58B cardinality=1500000 | F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3 -Per-Host Resources: mem-estimate=344.00MB mem-reservation=35.06MB +Per-Host Resources: mem-estimate=345.06MB mem-reservation=35.06MB 09:AGGREGATE [STREAMING] | group by: c_name, o1.o_orderkey, o2.o_orderstatus | mem-estimate=128.00MB mem-reservation=0B spill-buffer=2.00MB @@ -2510,7 +2510,7 @@ Per-Host Resources: mem-estimate=344.00MB mem-reservation=35.06MB | 05:HASH JOIN [INNER JOIN] | | hash predicates: o1.o_orderkey = o2.o_orderkey | | fk/pk conjuncts: assumed fk/pk -| | mem-estimate=0B mem-reservation=1.06MB spill-buffer=64.00KB +| | mem-estimate=1.06MB mem-reservation=1.06MB spill-buffer=64.00KB | | tuple-ids=1,2 row-size=32B cardinality=10 | | | |--04:UNNEST [c.c_orders o2] @@ -2532,7 +2532,7 @@ Per-Host Resources: mem-estimate=344.00MB mem-reservation=35.06MB tuple-ids=0 row-size=66B cardinality=150000 ---- PARALLELPLANS Max Per-Host Resource Reservation: Memory=138.12MB -Per-Host Resource Estimates: Memory=944.00MB +Per-Host Resource Estimates: Memory=946.13MB WARNING: The following tables are missing relevant table and/or column statistics. tpch_nested_parquet.customer @@ -2557,7 +2557,7 @@ Per-Host Resources: mem-estimate=256.00MB mem-reservation=68.00MB | tuple-ids=6 row-size=58B cardinality=1500000 | F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6 -Per-Host Resources: mem-estimate=688.00MB mem-reservation=70.12MB +Per-Host Resources: mem-estimate=690.13MB mem-reservation=70.12MB 09:AGGREGATE [STREAMING] | group by: c_name, o1.o_orderkey, o2.o_orderstatus | mem-estimate=128.00MB mem-reservation=0B spill-buffer=2.00MB @@ -2589,7 +2589,7 @@ Per-Host Resources: mem-estimate=688.00MB mem-reservation=70.12MB | 05:HASH JOIN [INNER JOIN] | | hash predicates: o1.o_orderkey = o2.o_orderkey | | fk/pk conjuncts: assumed fk/pk -| | mem-estimate=0B mem-reservation=1.06MB spill-buffer=64.00KB +| | mem-estimate=1.06MB mem-reservation=1.06MB spill-buffer=64.00KB | | tuple-ids=1,2 row-size=32B cardinality=10 | | | |--04:UNNEST [c.c_orders o2] @@ -2620,12 +2620,12 @@ from tpch_nested_parquet.customer c, from c.c_orders) v; ---- PLAN Max Per-Host Resource Reservation: Memory=48.00MB -Per-Host Resource Estimates: Memory=94.00MB +Per-Host Resource Estimates: Memory=136.00MB WARNING: The following tables are missing relevant table and/or column statistics. tpch_nested_parquet.customer F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=94.00MB mem-reservation=48.00MB +| Per-Host Resources: mem-estimate=136.00MB mem-reservation=48.00MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | @@ -2646,36 +2646,36 @@ PLAN-ROOT SINK | | functions: row_number() | | order by: o_orderpriority ASC | | window: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW -| | mem-estimate=0B mem-reservation=4.00MB spill-buffer=2.00MB +| | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB | | tuple-ids=12,7 row-size=164B cardinality=10 | | | 08:SORT | | order by: o_orderpriority ASC -| | mem-estimate=2.00MB mem-reservation=12.00MB spill-buffer=2.00MB +| | mem-estimate=12.00MB mem-reservation=12.00MB spill-buffer=2.00MB | | tuple-ids=12 row-size=156B cardinality=10 | | | 07:ANALYTIC | | functions: row_number() | | order by: o_orderdate ASC | | window: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW -| | mem-estimate=0B mem-reservation=4.00MB spill-buffer=2.00MB +| | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB | | tuple-ids=10,6 row-size=156B cardinality=10 | | | 06:SORT | | order by: o_orderdate ASC -| | mem-estimate=2.00MB mem-reservation=12.00MB spill-buffer=2.00MB +| | mem-estimate=12.00MB mem-reservation=12.00MB spill-buffer=2.00MB | | tuple-ids=10 row-size=148B cardinality=10 | | | 05:ANALYTIC | | functions: row_number() | | order by: o_totalprice ASC | | window: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW -| | mem-estimate=0B mem-reservation=4.00MB spill-buffer=2.00MB +| | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB | | tuple-ids=8,5 row-size=148B cardinality=10 | | | 04:SORT | | order by: o_totalprice ASC -| | mem-estimate=2.00MB mem-reservation=12.00MB spill-buffer=2.00MB +| | mem-estimate=12.00MB mem-reservation=12.00MB spill-buffer=2.00MB | | tuple-ids=8 row-size=140B cardinality=10 | | | 03:UNNEST [c.c_orders] @@ -2692,7 +2692,7 @@ PLAN-ROOT SINK tuple-ids=0 row-size=254B cardinality=150000 ---- DISTRIBUTEDPLAN Max Per-Host Resource Reservation: Memory=48.00MB -Per-Host Resource Estimates: Memory=94.00MB +Per-Host Resource Estimates: Memory=136.00MB WARNING: The following tables are missing relevant table and/or column statistics. tpch_nested_parquet.customer @@ -2706,7 +2706,7 @@ PLAN-ROOT SINK | tuple-ids=12,7,0 row-size=418B cardinality=1500000 | F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3 -Per-Host Resources: mem-estimate=94.00MB mem-reservation=48.00MB +Per-Host Resources: mem-estimate=136.00MB mem-reservation=48.00MB 01:SUBPLAN | mem-estimate=0B mem-reservation=0B | tuple-ids=12,7,0 row-size=418B cardinality=1500000 @@ -2724,36 +2724,36 @@ Per-Host Resources: mem-estimate=94.00MB mem-reservation=48.00MB | | functions: row_number() | | order by: o_orderpriority ASC | | window: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW -| | mem-estimate=0B mem-reservation=4.00MB spill-buffer=2.00MB +| | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB | | tuple-ids=12,7 row-size=164B cardinality=10 | | | 08:SORT | | order by: o_orderpriority ASC -| | mem-estimate=2.00MB mem-reservation=12.00MB spill-buffer=2.00MB +| | mem-estimate=12.00MB mem-reservation=12.00MB spill-buffer=2.00MB | | tuple-ids=12 row-size=156B cardinality=10 | | | 07:ANALYTIC | | functions: row_number() | | order by: o_orderdate ASC | | window: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW -| | mem-estimate=0B mem-reservation=4.00MB spill-buffer=2.00MB +| | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB | | tuple-ids=10,6 row-size=156B cardinality=10 | | | 06:SORT | | order by: o_orderdate ASC -| | mem-estimate=2.00MB mem-reservation=12.00MB spill-buffer=2.00MB +| | mem-estimate=12.00MB mem-reservation=12.00MB spill-buffer=2.00MB | | tuple-ids=10 row-size=148B cardinality=10 | | | 05:ANALYTIC | | functions: row_number() | | order by: o_totalprice ASC | | window: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW -| | mem-estimate=0B mem-reservation=4.00MB spill-buffer=2.00MB +| | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB | | tuple-ids=8,5 row-size=148B cardinality=10 | | | 04:SORT | | order by: o_totalprice ASC -| | mem-estimate=2.00MB mem-reservation=12.00MB spill-buffer=2.00MB +| | mem-estimate=12.00MB mem-reservation=12.00MB spill-buffer=2.00MB | | tuple-ids=8 row-size=140B cardinality=10 | | | 03:UNNEST [c.c_orders] @@ -2770,7 +2770,7 @@ Per-Host Resources: mem-estimate=94.00MB mem-reservation=48.00MB tuple-ids=0 row-size=254B cardinality=150000 ---- PARALLELPLANS Max Per-Host Resource Reservation: Memory=96.00MB -Per-Host Resource Estimates: Memory=188.00MB +Per-Host Resource Estimates: Memory=272.00MB WARNING: The following tables are missing relevant table and/or column statistics. tpch_nested_parquet.customer @@ -2784,7 +2784,7 @@ PLAN-ROOT SINK | tuple-ids=12,7,0 row-size=418B cardinality=1500000 | F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6 -Per-Host Resources: mem-estimate=188.00MB mem-reservation=96.00MB +Per-Host Resources: mem-estimate=272.00MB mem-reservation=96.00MB 01:SUBPLAN | mem-estimate=0B mem-reservation=0B | tuple-ids=12,7,0 row-size=418B cardinality=1500000 @@ -2802,36 +2802,36 @@ Per-Host Resources: mem-estimate=188.00MB mem-reservation=96.00MB | | functions: row_number() | | order by: o_orderpriority ASC | | window: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW -| | mem-estimate=0B mem-reservation=4.00MB spill-buffer=2.00MB +| | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB | | tuple-ids=12,7 row-size=164B cardinality=10 | | | 08:SORT | | order by: o_orderpriority ASC -| | mem-estimate=2.00MB mem-reservation=12.00MB spill-buffer=2.00MB +| | mem-estimate=12.00MB mem-reservation=12.00MB spill-buffer=2.00MB | | tuple-ids=12 row-size=156B cardinality=10 | | | 07:ANALYTIC | | functions: row_number() | | order by: o_orderdate ASC | | window: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW -| | mem-estimate=0B mem-reservation=4.00MB spill-buffer=2.00MB +| | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB | | tuple-ids=10,6 row-size=156B cardinality=10 | | | 06:SORT | | order by: o_orderdate ASC -| | mem-estimate=2.00MB mem-reservation=12.00MB spill-buffer=2.00MB +| | mem-estimate=12.00MB mem-reservation=12.00MB spill-buffer=2.00MB | | tuple-ids=10 row-size=148B cardinality=10 | | | 05:ANALYTIC | | functions: row_number() | | order by: o_totalprice ASC | | window: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW -| | mem-estimate=0B mem-reservation=4.00MB spill-buffer=2.00MB +| | mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB | | tuple-ids=8,5 row-size=148B cardinality=10 | | | 04:SORT | | order by: o_totalprice ASC -| | mem-estimate=2.00MB mem-reservation=12.00MB spill-buffer=2.00MB +| | mem-estimate=12.00MB mem-reservation=12.00MB spill-buffer=2.00MB | | tuple-ids=8 row-size=140B cardinality=10 | | | 03:UNNEST [c.c_orders] @@ -2862,10 +2862,10 @@ join ( ) v1 on v1.k3 = t1.o_orderkey ---- PLAN Max Per-Host Resource Reservation: Memory=68.00MB -Per-Host Resource Estimates: Memory=172.59MB +Per-Host Resource Estimates: Memory=177.00MB F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=172.59MB mem-reservation=68.00MB +| Per-Host Resources: mem-estimate=177.00MB mem-reservation=68.00MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | @@ -2880,14 +2880,14 @@ PLAN-ROOT SINK | | hash predicates: t2.o_orderkey = t3.o_orderkey | | fk/pk conjuncts: t2.o_orderkey = t3.o_orderkey | | runtime filters: RF001 <- t3.o_orderkey -| | mem-estimate=25.18MB mem-reservation=34.00MB spill-buffer=2.00MB +| | mem-estimate=34.00MB mem-reservation=34.00MB spill-buffer=2.00MB | | tuple-ids=1,2,3 row-size=24B cardinality=1500000 | | | |--04:HASH JOIN [INNER JOIN] | | | hash predicates: t3.o_orderkey = t4.o_orderkey | | | fk/pk conjuncts: t3.o_orderkey = t4.o_orderkey | | | runtime filters: RF002 <- t4.o_orderkey -| | | mem-estimate=12.59MB mem-reservation=17.00MB spill-buffer=1.00MB +| | | mem-estimate=17.00MB mem-reservation=17.00MB spill-buffer=1.00MB | | | tuple-ids=2,3 row-size=16B cardinality=1500000 | | | | | |--03:SCAN HDFS [tpch_parquet.orders t4] @@ -2926,7 +2926,7 @@ PLAN-ROOT SINK tuple-ids=0 row-size=191B cardinality=1500000 ---- DISTRIBUTEDPLAN Max Per-Host Resource Reservation: Memory=59.50MB -Per-Host Resource Estimates: Memory=216.65MB +Per-Host Resource Estimates: Memory=223.27MB F05:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 | Per-Host Resources: mem-estimate=0B mem-reservation=0B @@ -2951,19 +2951,19 @@ Per-Host Resources: mem-estimate=77.77MB mem-reservation=34.00MB | | tuple-ids=1,2,3 row-size=24B cardinality=1500000 | | | F04:PLAN FRAGMENT [HASH(t3.o_orderkey)] hosts=2 instances=2 -| Per-Host Resources: mem-estimate=18.88MB mem-reservation=25.50MB +| Per-Host Resources: mem-estimate=25.50MB mem-reservation=25.50MB | 05:HASH JOIN [INNER JOIN, PARTITIONED] | | hash predicates: t2.o_orderkey = t3.o_orderkey | | fk/pk conjuncts: t2.o_orderkey = t3.o_orderkey | | runtime filters: RF001 <- t3.o_orderkey -| | mem-estimate=12.59MB mem-reservation=17.00MB spill-buffer=1.00MB +| | mem-estimate=17.00MB mem-reservation=17.00MB spill-buffer=1.00MB | | tuple-ids=1,2,3 row-size=24B cardinality=1500000 | | | |--04:HASH JOIN [INNER JOIN, PARTITIONED] | | | hash predicates: t3.o_orderkey = t4.o_orderkey | | | fk/pk conjuncts: t3.o_orderkey = t4.o_orderkey | | | runtime filters: RF002 <- t4.o_orderkey -| | | mem-estimate=6.29MB mem-reservation=8.50MB spill-buffer=512.00KB +| | | mem-estimate=8.50MB mem-reservation=8.50MB spill-buffer=512.00KB | | | tuple-ids=2,3 row-size=16B cardinality=1500000 | | | | | |--08:EXCHANGE [HASH(t4.o_orderkey)] @@ -3020,7 +3020,7 @@ Per-Host Resources: mem-estimate=77.77MB mem-reservation=34.00MB tuple-ids=0 row-size=191B cardinality=1500000 ---- PARALLELPLANS Max Per-Host Resource Reservation: Memory=93.50MB -Per-Host Resource Estimates: Memory=414.41MB +Per-Host Resource Estimates: Memory=421.03MB F05:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 | Per-Host Resources: mem-estimate=0B mem-reservation=0B @@ -3053,13 +3053,13 @@ Per-Host Resources: mem-estimate=155.53MB mem-reservation=68.00MB | | tuple-ids=1,2,3 row-size=24B cardinality=1500000 | | | F04:PLAN FRAGMENT [HASH(t3.o_orderkey)] hosts=2 instances=4 -| Per-Host Resources: mem-estimate=18.88MB mem-reservation=25.50MB +| Per-Host Resources: mem-estimate=25.50MB mem-reservation=25.50MB | 05:HASH JOIN [INNER JOIN, PARTITIONED] | | hash-table-id=01 | | hash predicates: t2.o_orderkey = t3.o_orderkey | | fk/pk conjuncts: t2.o_orderkey = t3.o_orderkey | | runtime filters: RF001 <- t3.o_orderkey -| | mem-estimate=6.29MB mem-reservation=8.50MB spill-buffer=512.00KB +| | mem-estimate=8.50MB mem-reservation=8.50MB spill-buffer=512.00KB | | tuple-ids=1,2,3 row-size=24B cardinality=1500000 | | | |--F07:PLAN FRAGMENT [HASH(t3.o_orderkey)] hosts=2 instances=4 @@ -3074,7 +3074,7 @@ Per-Host Resources: mem-estimate=155.53MB mem-reservation=68.00MB | | | hash predicates: t3.o_orderkey = t4.o_orderkey | | | fk/pk conjuncts: t3.o_orderkey = t4.o_orderkey | | | runtime filters: RF002 <- t4.o_orderkey -| | | mem-estimate=3.15MB mem-reservation=4.25MB spill-buffer=256.00KB +| | | mem-estimate=4.25MB mem-reservation=4.25MB spill-buffer=256.00KB | | | tuple-ids=2,3 row-size=16B cardinality=1500000 | | | | | |--F08:PLAN FRAGMENT [HASH(t3.o_orderkey)] hosts=2 instances=4 @@ -3388,11 +3388,11 @@ sum(smallint_col) over (partition by tinyint_col order by smallint_col from functional.alltypesagg ---- PLAN Max Per-Host Resource Reservation: Memory=18.00MB -Per-Host Resource Estimates: Memory=18.00MB +Per-Host Resource Estimates: Memory=34.00MB Codegen disabled by planner F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=18.00MB mem-reservation=18.00MB +| Per-Host Resources: mem-estimate=34.00MB mem-reservation=18.00MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | @@ -3401,7 +3401,7 @@ PLAN-ROOT SINK | partition by: tinyint_col | order by: smallint_col ASC | window: ROWS BETWEEN 1 FOLLOWING AND 2 FOLLOWING -| mem-estimate=0B mem-reservation=4.00MB spill-buffer=2.00MB +| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB | tuple-ids=5,2,3,4 row-size=27B cardinality=11000 | 03:ANALYTIC @@ -3409,7 +3409,7 @@ PLAN-ROOT SINK | partition by: tinyint_col | order by: smallint_col ASC | window: ROWS BETWEEN UNBOUNDED PRECEDING AND 1 FOLLOWING -| mem-estimate=0B mem-reservation=4.00MB spill-buffer=2.00MB +| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB | tuple-ids=5,2,3 row-size=19B cardinality=11000 | 02:ANALYTIC @@ -3417,12 +3417,12 @@ PLAN-ROOT SINK | partition by: tinyint_col | order by: smallint_col ASC | window: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW -| mem-estimate=0B mem-reservation=4.00MB spill-buffer=2.00MB +| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB | tuple-ids=5,2 row-size=11B cardinality=11000 | 01:SORT | order by: tinyint_col ASC NULLS FIRST, smallint_col ASC -| mem-estimate=2.00MB mem-reservation=6.00MB spill-buffer=2.00MB +| mem-estimate=6.00MB mem-reservation=6.00MB spill-buffer=2.00MB | tuple-ids=5 row-size=3B cardinality=11000 | 00:SCAN HDFS [functional.alltypesagg] http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/5ec40464/testdata/workloads/functional-planner/queries/PlannerTest/sort-expr-materialization.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/sort-expr-materialization.test b/testdata/workloads/functional-planner/queries/PlannerTest/sort-expr-materialization.test index 66f8167..47046b3 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/sort-expr-materialization.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/sort-expr-materialization.test @@ -2,14 +2,14 @@ select * from functional.alltypes order by random() ---- PLAN F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=130.00MB mem-reservation=12.00MB +| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.00MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | 01:SORT | order by: random() ASC | materialized: random() -| mem-estimate=2.00MB mem-reservation=12.00MB spill-buffer=2.00MB +| mem-estimate=12.00MB mem-reservation=12.00MB spill-buffer=2.00MB | tuple-ids=1 row-size=105B cardinality=7300 | 00:SCAN HDFS [functional.alltypes] @@ -24,14 +24,14 @@ PLAN-ROOT SINK select * from functional.alltypes order by abs(id) + abs(id) ---- PLAN F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=130.00MB mem-reservation=12.00MB +| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.00MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | 01:SORT | order by: abs(id) + abs(id) ASC | materialized: abs(id) + abs(id) -| mem-estimate=2.00MB mem-reservation=12.00MB spill-buffer=2.00MB +| mem-estimate=12.00MB mem-reservation=12.00MB spill-buffer=2.00MB | tuple-ids=1 row-size=105B cardinality=7300 | 00:SCAN HDFS [functional.alltypes] @@ -46,13 +46,13 @@ PLAN-ROOT SINK select * from functional.alltypes order by tinyint_col + 1 ---- PLAN F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=130.00MB mem-reservation=12.00MB +| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.00MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | 01:SORT | order by: tinyint_col + 1 ASC -| mem-estimate=2.00MB mem-reservation=12.00MB spill-buffer=2.00MB +| mem-estimate=12.00MB mem-reservation=12.00MB spill-buffer=2.00MB | tuple-ids=1 row-size=97B cardinality=7300 | 00:SCAN HDFS [functional.alltypes] @@ -68,14 +68,14 @@ select * from functional.alltypes order by dayofweek(timestamp_col), true, id + 1, string_col = date_string_col, id = tinyint_col ---- PLAN F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=130.00MB mem-reservation=12.00MB +| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.00MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | 01:SORT | order by: dayofweek(timestamp_col) ASC, TRUE ASC, id + 1 ASC, string_col = date_string_col ASC, id = tinyint_col ASC | materialized: dayofweek(timestamp_col), string_col = date_string_col -| mem-estimate=2.00MB mem-reservation=12.00MB spill-buffer=2.00MB +| mem-estimate=12.00MB mem-reservation=12.00MB spill-buffer=2.00MB | tuple-ids=1 row-size=102B cardinality=7300 | 00:SCAN HDFS [functional.alltypes] @@ -91,7 +91,7 @@ select last_value(id) over (order by to_date(timestamp_col), bool_col is null) from functional.alltypes ---- PLAN F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=130.00MB mem-reservation=16.00MB +| Per-Host Resources: mem-estimate=144.00MB mem-reservation=16.00MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | @@ -99,13 +99,13 @@ PLAN-ROOT SINK | functions: last_value(id) | order by: to_date(timestamp_col) ASC, bool_col IS NULL ASC | window: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW -| mem-estimate=0B mem-reservation=4.00MB spill-buffer=2.00MB +| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB | tuple-ids=3,2 row-size=41B cardinality=7300 | 01:SORT | order by: to_date(timestamp_col) ASC, bool_col IS NULL ASC | materialized: to_date(timestamp_col) -| mem-estimate=2.00MB mem-reservation=12.00MB spill-buffer=2.00MB +| mem-estimate=12.00MB mem-reservation=12.00MB spill-buffer=2.00MB | tuple-ids=3 row-size=37B cardinality=7300 | 00:SCAN HDFS [functional.alltypes] @@ -143,14 +143,14 @@ PLAN-ROOT SINK select * from functional.alltypes order by TestFn(double_col) ---- PLAN F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=130.00MB mem-reservation=12.00MB +| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.00MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | 01:SORT | order by: default.testfn(double_col) ASC | materialized: default.testfn(double_col) -| mem-estimate=2.00MB mem-reservation=12.00MB spill-buffer=2.00MB +| mem-estimate=12.00MB mem-reservation=12.00MB spill-buffer=2.00MB | tuple-ids=1 row-size=101B cardinality=7300 | 00:SCAN HDFS [functional.alltypes] @@ -165,14 +165,14 @@ PLAN-ROOT SINK select concat(date_string_col, string_col) c from functional.alltypes order by c ---- PLAN F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=130.00MB mem-reservation=12.00MB +| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.00MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | 01:SORT | order by: concat(date_string_col, string_col) ASC | materialized: concat(date_string_col, string_col) -| mem-estimate=2.00MB mem-reservation=12.00MB spill-buffer=2.00MB +| mem-estimate=12.00MB mem-reservation=12.00MB spill-buffer=2.00MB | tuple-ids=1 row-size=16B cardinality=7300 | 00:SCAN HDFS [functional.alltypes] http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/5ec40464/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test b/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test index 99ef97b..16e8cb2 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test @@ -4,7 +4,7 @@ from tpch_parquet.customer inner join tpch_parquet.nation on c_nationkey = n_nationkey ---- DISTRIBUTEDPLAN Max Per-Host Resource Reservation: Memory=1.06MB -Per-Host Resource Estimates: Memory=40.00MB +Per-Host Resource Estimates: Memory=41.06MB F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 | Per-Host Resources: mem-estimate=0B mem-reservation=0B @@ -16,12 +16,12 @@ PLAN-ROOT SINK | tuple-ids=0,1 row-size=355B cardinality=150000 | F00:PLAN FRAGMENT [RANDOM] hosts=1 instances=1 -Per-Host Resources: mem-estimate=24.00MB mem-reservation=1.06MB +Per-Host Resources: mem-estimate=25.06MB mem-reservation=1.06MB 02:HASH JOIN [INNER JOIN, BROADCAST] | hash predicates: c_nationkey = n_nationkey | fk/pk conjuncts: c_nationkey = n_nationkey | runtime filters: RF000 <- n_nationkey -| mem-estimate=3.15KB mem-reservation=1.06MB spill-buffer=64.00KB +| mem-estimate=1.06MB mem-reservation=1.06MB spill-buffer=64.00KB | tuple-ids=0,1 row-size=355B cardinality=150000 | |--03:EXCHANGE [BROADCAST] @@ -48,7 +48,7 @@ Per-Host Resources: mem-estimate=24.00MB mem-reservation=1.06MB tuple-ids=0 row-size=238B cardinality=150000 ---- PARALLELPLANS Max Per-Host Resource Reservation: Memory=2.12MB -Per-Host Resource Estimates: Memory=80.01MB +Per-Host Resource Estimates: Memory=82.12MB F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 | Per-Host Resources: mem-estimate=0B mem-reservation=0B @@ -60,13 +60,13 @@ PLAN-ROOT SINK | tuple-ids=0,1 row-size=355B cardinality=150000 | F00:PLAN FRAGMENT [RANDOM] hosts=1 instances=2 -Per-Host Resources: mem-estimate=48.01MB mem-reservation=2.12MB +Per-Host Resources: mem-estimate=50.12MB mem-reservation=2.12MB 02:HASH JOIN [INNER JOIN, BROADCAST] | hash-table-id=00 | hash predicates: c_nationkey = n_nationkey | fk/pk conjuncts: c_nationkey = n_nationkey | runtime filters: RF000 <- n_nationkey -| mem-estimate=3.15KB mem-reservation=1.06MB spill-buffer=64.00KB +| mem-estimate=1.06MB mem-reservation=1.06MB spill-buffer=64.00KB | tuple-ids=0,1 row-size=355B cardinality=150000 | |--F03:PLAN FRAGMENT [RANDOM] hosts=1 instances=2 @@ -202,7 +202,7 @@ from tpch_parquet.orders join /*+shuffle*/ tpch_parquet.customer on o_custkey = c_custkey ---- DISTRIBUTEDPLAN Max Per-Host Resource Reservation: Memory=34.00MB -Per-Host Resource Estimates: Memory=82.69MB +Per-Host Resource Estimates: Memory=98.00MB F03:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 | Per-Host Resources: mem-estimate=0B mem-reservation=0B @@ -214,12 +214,12 @@ PLAN-ROOT SINK | tuple-ids=0,1 row-size=428B cardinality=1500000 | F02:PLAN FRAGMENT [HASH(o_custkey)] hosts=2 instances=2 -Per-Host Resources: mem-estimate=18.69MB mem-reservation=34.00MB +Per-Host Resources: mem-estimate=34.00MB mem-reservation=34.00MB 02:HASH JOIN [INNER JOIN, PARTITIONED] | hash predicates: o_custkey = c_custkey | fk/pk conjuncts: o_custkey = c_custkey | runtime filters: RF000 <- c_custkey -| mem-estimate=18.69MB mem-reservation=34.00MB spill-buffer=2.00MB +| mem-estimate=34.00MB mem-reservation=34.00MB spill-buffer=2.00MB | tuple-ids=0,1 row-size=428B cardinality=1500000 | |--04:EXCHANGE [HASH(c_custkey)] @@ -252,7 +252,7 @@ Per-Host Resources: mem-estimate=40.00MB mem-reservation=0B tuple-ids=0 row-size=191B cardinality=1500000 ---- PARALLELPLANS Max Per-Host Resource Reservation: Memory=34.00MB -Per-Host Resource Estimates: Memory=146.69MB +Per-Host Resource Estimates: Memory=162.00MB F03:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 | Per-Host Resources: mem-estimate=0B mem-reservation=0B @@ -264,13 +264,13 @@ PLAN-ROOT SINK | tuple-ids=0,1 row-size=428B cardinality=1500000 | F02:PLAN FRAGMENT [HASH(o_custkey)] hosts=2 instances=4 -Per-Host Resources: mem-estimate=18.69MB mem-reservation=34.00MB +Per-Host Resources: mem-estimate=34.00MB mem-reservation=34.00MB 02:HASH JOIN [INNER JOIN, PARTITIONED] | hash-table-id=00 | hash predicates: o_custkey = c_custkey | fk/pk conjuncts: o_custkey = c_custkey | runtime filters: RF000 <- c_custkey -| mem-estimate=9.35MB mem-reservation=17.00MB spill-buffer=1.00MB +| mem-estimate=17.00MB mem-reservation=17.00MB spill-buffer=1.00MB | tuple-ids=0,1 row-size=428B cardinality=1500000 | |--F04:PLAN FRAGMENT [HASH(o_custkey)] hosts=1 instances=2 @@ -604,7 +604,7 @@ group by 1, 2 having count(*) = 1 ---- DISTRIBUTEDPLAN Max Per-Host Resource Reservation: Memory=51.00MB -Per-Host Resource Estimates: Memory=205.28MB +Per-Host Resource Estimates: Memory=225.12MB F04:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 | Per-Host Resources: mem-estimate=0B mem-reservation=0B @@ -616,12 +616,12 @@ PLAN-ROOT SINK | tuple-ids=2 row-size=33B cardinality=4690314 | F03:PLAN FRAGMENT [HASH(l_orderkey,o_orderstatus)] hosts=3 instances=3 -Per-Host Resources: mem-estimate=18.04MB mem-reservation=34.00MB +Per-Host Resources: mem-estimate=34.00MB mem-reservation=34.00MB 07:AGGREGATE [FINALIZE] | output: count:merge(*) | group by: l_orderkey, o_orderstatus | having: count(*) = 1 -| mem-estimate=18.04MB mem-reservation=34.00MB spill-buffer=2.00MB +| mem-estimate=34.00MB mem-reservation=34.00MB spill-buffer=2.00MB | tuple-ids=2 row-size=33B cardinality=4690314 | 06:EXCHANGE [HASH(l_orderkey,o_orderstatus)] @@ -629,7 +629,7 @@ Per-Host Resources: mem-estimate=18.04MB mem-reservation=34.00MB | tuple-ids=2 row-size=33B cardinality=4690314 | F02:PLAN FRAGMENT [HASH(l_orderkey)] hosts=3 instances=3 -Per-Host Resources: mem-estimate=67.24MB mem-reservation=17.00MB +Per-Host Resources: mem-estimate=71.12MB mem-reservation=17.00MB 03:AGGREGATE [STREAMING] | output: count(*) | group by: l_orderkey, o_orderstatus @@ -640,7 +640,7 @@ Per-Host Resources: mem-estimate=67.24MB mem-reservation=17.00MB | hash predicates: l_orderkey = o_orderkey | fk/pk conjuncts: l_orderkey = o_orderkey | runtime filters: RF000 <- o_orderkey -| mem-estimate=13.11MB mem-reservation=17.00MB spill-buffer=1.00MB +| mem-estimate=17.00MB mem-reservation=17.00MB spill-buffer=1.00MB | tuple-ids=0,1 row-size=33B cardinality=5757710 | |--05:EXCHANGE [HASH(o_orderkey)] @@ -673,7 +673,7 @@ Per-Host Resources: mem-estimate=80.00MB mem-reservation=0B tuple-ids=0 row-size=8B cardinality=6001215 ---- PARALLELPLANS Max Per-Host Resource Reservation: Memory=51.00MB -Per-Host Resource Estimates: Memory=327.24MB +Per-Host Resource Estimates: Memory=345.12MB F04:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 | Per-Host Resources: mem-estimate=0B mem-reservation=0B @@ -685,12 +685,12 @@ PLAN-ROOT SINK | tuple-ids=2 row-size=33B cardinality=4690314 | F03:PLAN FRAGMENT [HASH(l_orderkey,o_orderstatus)] hosts=3 instances=6 -Per-Host Resources: mem-estimate=20.00MB mem-reservation=34.00MB +Per-Host Resources: mem-estimate=34.00MB mem-reservation=34.00MB 07:AGGREGATE [FINALIZE] | output: count:merge(*) | group by: l_orderkey, o_orderstatus | having: count(*) = 1 -| mem-estimate=10.00MB mem-reservation=17.00MB spill-buffer=1.00MB +| mem-estimate=17.00MB mem-reservation=17.00MB spill-buffer=1.00MB | tuple-ids=2 row-size=33B cardinality=4690314 | 06:EXCHANGE [HASH(l_orderkey,o_orderstatus)] @@ -698,7 +698,7 @@ Per-Host Resources: mem-estimate=20.00MB mem-reservation=34.00MB | tuple-ids=2 row-size=33B cardinality=4690314 | F02:PLAN FRAGMENT [HASH(l_orderkey)] hosts=3 instances=6 -Per-Host Resources: mem-estimate=67.24MB mem-reservation=17.00MB +Per-Host Resources: mem-estimate=71.12MB mem-reservation=17.00MB 03:AGGREGATE [STREAMING] | output: count(*) | group by: l_orderkey, o_orderstatus @@ -710,7 +710,7 @@ Per-Host Resources: mem-estimate=67.24MB mem-reservation=17.00MB | hash predicates: l_orderkey = o_orderkey | fk/pk conjuncts: l_orderkey = o_orderkey | runtime filters: RF000 <- o_orderkey -| mem-estimate=6.56MB mem-reservation=8.50MB spill-buffer=512.00KB +| mem-estimate=8.50MB mem-reservation=8.50MB spill-buffer=512.00KB | tuple-ids=0,1 row-size=33B cardinality=5757710 | |--F05:PLAN FRAGMENT [HASH(l_orderkey)] hosts=2 instances=4 http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/5ec40464/testdata/workloads/functional-planner/queries/PlannerTest/tablesample.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/tablesample.test b/testdata/workloads/functional-planner/queries/PlannerTest/tablesample.test index 4c208a4..7e78f73 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/tablesample.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/tablesample.test @@ -154,14 +154,14 @@ select id from functional.alltypes t1 where exists ( where t1.id = t2.id) ---- PLAN F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=160.00MB mem-reservation=1.06MB +| Per-Host Resources: mem-estimate=161.06MB mem-reservation=1.06MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | 02:HASH JOIN [LEFT SEMI JOIN] | hash predicates: t1.id = t2.id | runtime filters: RF000 <- t2.id -| mem-estimate=44B mem-reservation=1.06MB spill-buffer=64.00KB +| mem-estimate=1.06MB mem-reservation=1.06MB spill-buffer=64.00KB | tuple-ids=0 row-size=4B cardinality=10 | |--01:SCAN HDFS [functional.alltypessmall t2]
