http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ed87c406/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test b/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test index 73129ec..bf2bc52 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test @@ -415,7 +415,7 @@ Per-Host Resources: mem-estimate=106.24MB mem-reservation=0B mem-estimate=80.00MB mem-reservation=0B tuple-ids=0 row-size=8B cardinality=6001215 ---- PARALLELPLANS -Max Per-Host Resource Reservation: Memory=8.50MB +Max Per-Host Resource Reservation: Memory=9.50MB Per-Host Resource Estimates: Memory=232.48MB F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 @@ -428,11 +428,11 @@ PLAN-ROOT SINK | tuple-ids=1 row-size=16B cardinality=1563438 | F01:PLAN FRAGMENT [HASH(l_orderkey)] hosts=3 instances=6 -Per-Host Resources: mem-estimate=20.00MB mem-reservation=8.50MB +Per-Host Resources: mem-estimate=20.00MB mem-reservation=9.50MB 03:AGGREGATE [FINALIZE] | output: count:merge(*) | group by: l_orderkey -| mem-estimate=10.00MB mem-reservation=4.25MB spill-buffer=256.00KB +| mem-estimate=10.00MB mem-reservation=4.75MB spill-buffer=256.00KB | tuple-ids=1 row-size=16B cardinality=1563438 | 02:EXCHANGE [HASH(l_orderkey)] @@ -1548,8 +1548,8 @@ PLAN-ROOT SINK mem-estimate=80.00MB mem-reservation=0B tuple-ids=0 row-size=78B cardinality=600122 ---- DISTRIBUTEDPLAN -Max Per-Host Resource Reservation: Memory=38.25MB -Per-Host Resource Estimates: Memory=343.83MB +Max Per-Host Resource Reservation: Memory=38.75MB +Per-Host Resource Estimates: Memory=344.33MB F09:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 | Per-Host Resources: mem-estimate=0B mem-reservation=0B @@ -1643,7 +1643,7 @@ Per-Host Resources: mem-estimate=97.00MB mem-reservation=34.00MB | tuple-ids=2 row-size=70B cardinality=575772 | F02:PLAN FRAGMENT [HASH(l_orderkey)] hosts=3 instances=3 -Per-Host Resources: mem-estimate=46.83MB mem-reservation=4.25MB +Per-Host Resources: mem-estimate=47.33MB mem-reservation=4.75MB 04:AGGREGATE [STREAMING] | group by: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_comment | mem-estimate=42.58MB mem-reservation=0B spill-buffer=2.00MB @@ -1653,7 +1653,7 @@ Per-Host Resources: mem-estimate=46.83MB mem-reservation=4.25MB | hash predicates: l_orderkey = o_orderkey | fk/pk conjuncts: l_orderkey = o_orderkey | runtime filters: RF000 <- o_orderkey -| mem-estimate=4.25MB mem-reservation=4.25MB spill-buffer=256.00KB +| mem-estimate=4.75MB mem-reservation=4.75MB spill-buffer=256.00KB | tuple-ids=0,1 row-size=86B cardinality=575772 | |--12:EXCHANGE [HASH(o_orderkey)] @@ -1688,8 +1688,8 @@ Per-Host Resources: mem-estimate=80.00MB mem-reservation=0B mem-estimate=80.00MB mem-reservation=0B tuple-ids=0 row-size=78B cardinality=600122 ---- PARALLELPLANS -Max Per-Host Resource Reservation: Memory=72.25MB -Per-Host Resource Estimates: Memory=683.41MB +Max Per-Host Resource Reservation: Memory=73.75MB +Per-Host Resource Estimates: Memory=684.91MB F09:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 | Per-Host Resources: mem-estimate=0B mem-reservation=0B @@ -1799,7 +1799,7 @@ Per-Host Resources: mem-estimate=194.00MB mem-reservation=68.00MB | tuple-ids=2 row-size=70B cardinality=575772 | F02:PLAN FRAGMENT [HASH(l_orderkey)] hosts=3 instances=6 -Per-Host Resources: mem-estimate=89.41MB mem-reservation=4.25MB +Per-Host Resources: mem-estimate=90.91MB mem-reservation=5.75MB 04:AGGREGATE [STREAMING] | group by: l_orderkey, l_partkey, l_suppkey, l_linenumber, l_comment | mem-estimate=42.58MB mem-reservation=0B spill-buffer=2.00MB @@ -1810,7 +1810,7 @@ Per-Host Resources: mem-estimate=89.41MB mem-reservation=4.25MB | hash predicates: l_orderkey = o_orderkey | fk/pk conjuncts: l_orderkey = o_orderkey | runtime filters: RF000 <- o_orderkey -| mem-estimate=2.12MB mem-reservation=2.12MB spill-buffer=128.00KB +| mem-estimate=2.88MB mem-reservation=2.88MB spill-buffer=128.00KB | tuple-ids=0,1 row-size=86B cardinality=575772 | |--F12:PLAN FRAGMENT [HASH(l_orderkey)] hosts=2 instances=4 @@ -1888,11 +1888,11 @@ order by o_orderdate limit 100 ---- PLAN -Max Per-Host Resource Reservation: Memory=80.75MB +Max Per-Host Resource Reservation: Memory=81.25MB Per-Host Resource Estimates: Memory=393.18MB F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=393.18MB mem-reservation=80.75MB +| Per-Host Resources: mem-estimate=393.18MB mem-reservation=81.25MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | @@ -1910,14 +1910,14 @@ PLAN-ROOT SINK 07:HASH JOIN [LEFT SEMI JOIN] | hash predicates: o_orderkey = l_orderkey | runtime filters: RF000 <- l_orderkey -| mem-estimate=4.25MB mem-reservation=4.25MB spill-buffer=256.00KB +| mem-estimate=4.75MB mem-reservation=4.75MB spill-buffer=256.00KB | tuple-ids=2,1,0 row-size=108B cardinality=575772 | |--04:AGGREGATE [FINALIZE] | | output: sum(l_quantity) | | group by: l_orderkey | | having: sum(l_quantity) > 300 -| | mem-estimate=10.00MB mem-reservation=4.25MB spill-buffer=256.00KB +| | mem-estimate=10.00MB mem-reservation=4.75MB spill-buffer=256.00KB | | tuple-ids=4 row-size=24B cardinality=156344 | | | 03:SCAN HDFS [tpch.lineitem] @@ -1968,8 +1968,8 @@ PLAN-ROOT SINK mem-estimate=88.00MB mem-reservation=0B tuple-ids=2 row-size=16B cardinality=6001215 ---- DISTRIBUTEDPLAN -Max Per-Host Resource Reservation: Memory=82.88MB -Per-Host Resource Estimates: Memory=510.80MB +Max Per-Host Resource Reservation: Memory=84.12MB +Per-Host Resource Estimates: Memory=511.55MB F07:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 | Per-Host Resources: mem-estimate=0B mem-reservation=0B @@ -2000,7 +2000,7 @@ Per-Host Resources: mem-estimate=60.41MB mem-reservation=34.00MB | tuple-ids=6 row-size=100B cardinality=575772 | F02:PLAN FRAGMENT [HASH(l_orderkey)] hosts=3 instances=3 -Per-Host Resources: mem-estimate=115.03MB mem-reservation=48.88MB +Per-Host Resources: mem-estimate=115.78MB mem-reservation=50.12MB 08:AGGREGATE [STREAMING] | output: sum(l_quantity) | group by: c_name, c_custkey, o_orderkey, o_orderdate, o_totalprice @@ -2010,14 +2010,14 @@ Per-Host Resources: mem-estimate=115.03MB mem-reservation=48.88MB 07:HASH JOIN [LEFT SEMI JOIN, PARTITIONED] | hash predicates: o_orderkey = l_orderkey | runtime filters: RF000 <- l_orderkey -| mem-estimate=2.12MB mem-reservation=2.12MB spill-buffer=128.00KB +| mem-estimate=2.88MB mem-reservation=2.88MB spill-buffer=128.00KB | tuple-ids=2,1,0 row-size=108B cardinality=575772 | |--14:AGGREGATE [FINALIZE] | | output: sum:merge(l_quantity) | | group by: l_orderkey | | having: sum(l_quantity) > 300 -| | mem-estimate=10.00MB mem-reservation=4.25MB spill-buffer=256.00KB +| | mem-estimate=10.00MB mem-reservation=4.75MB spill-buffer=256.00KB | | tuple-ids=4 row-size=24B cardinality=156344 | | | 13:EXCHANGE [HASH(l_orderkey)] @@ -2098,8 +2098,8 @@ Per-Host Resources: mem-estimate=88.00MB mem-reservation=0B mem-estimate=88.00MB mem-reservation=0B tuple-ids=2 row-size=16B cardinality=6001215 ---- PARALLELPLANS -Max Per-Host Resource Reservation: Memory=121.12MB -Per-Host Resource Estimates: Memory=965.47MB +Max Per-Host Resource Reservation: Memory=122.88MB +Per-Host Resource Estimates: Memory=967.22MB F07:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 | Per-Host Resources: mem-estimate=0B mem-reservation=0B @@ -2130,7 +2130,7 @@ Per-Host Resources: mem-estimate=120.82MB mem-reservation=68.00MB | tuple-ids=6 row-size=100B cardinality=575772 | F02:PLAN FRAGMENT [HASH(l_orderkey)] hosts=3 instances=6 -Per-Host Resources: mem-estimate=173.93MB mem-reservation=53.12MB +Per-Host Resources: mem-estimate=175.68MB mem-reservation=54.88MB 08:AGGREGATE [STREAMING] | output: sum(l_quantity) | group by: c_name, c_custkey, o_orderkey, o_orderdate, o_totalprice @@ -2141,7 +2141,7 @@ Per-Host Resources: mem-estimate=173.93MB mem-reservation=53.12MB | hash-table-id=00 | hash predicates: o_orderkey = l_orderkey | runtime filters: RF000 <- l_orderkey -| mem-estimate=1.06MB mem-reservation=1.06MB spill-buffer=64.00KB +| mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB | tuple-ids=2,1,0 row-size=108B cardinality=575772 | |--F08:PLAN FRAGMENT [HASH(l_orderkey)] hosts=3 instances=6 @@ -2155,7 +2155,7 @@ Per-Host Resources: mem-estimate=173.93MB mem-reservation=53.12MB | | output: sum:merge(l_quantity) | | group by: l_orderkey | | having: sum(l_quantity) > 300 -| | mem-estimate=10.00MB mem-reservation=4.25MB spill-buffer=256.00KB +| | mem-estimate=10.00MB mem-reservation=4.75MB spill-buffer=256.00KB | | tuple-ids=4 row-size=24B cardinality=156344 | | | 13:EXCHANGE [HASH(l_orderkey)] @@ -2390,13 +2390,13 @@ from tpch_nested_parquet.customer c, join c.c_orders o2 on o1.o_orderkey = o2.o_orderkey order by o1.o_orderkey limit 100) v ---- PLAN -Max Per-Host Resource Reservation: Memory=69.06MB -Per-Host Resource Estimates: Memory=345.06MB +Max Per-Host Resource Reservation: Memory=69.94MB +Per-Host Resource Estimates: Memory=345.94MB WARNING: The following tables are missing relevant table and/or column statistics. tpch_nested_parquet.customer F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=345.06MB mem-reservation=69.06MB +| Per-Host Resources: mem-estimate=345.94MB mem-reservation=69.94MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | @@ -2431,7 +2431,7 @@ PLAN-ROOT SINK | 05:HASH JOIN [INNER JOIN] | | hash predicates: o1.o_orderkey = o2.o_orderkey | | fk/pk conjuncts: assumed fk/pk -| | mem-estimate=1.06MB mem-reservation=1.06MB spill-buffer=64.00KB +| | mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB | | tuple-ids=1,2 row-size=32B cardinality=10 | | | |--04:UNNEST [c.c_orders o2] @@ -2452,8 +2452,8 @@ PLAN-ROOT SINK mem-estimate=88.00MB mem-reservation=0B tuple-ids=0 row-size=66B cardinality=150000 ---- DISTRIBUTEDPLAN -Max Per-Host Resource Reservation: Memory=69.06MB -Per-Host Resource Estimates: Memory=473.06MB +Max Per-Host Resource Reservation: Memory=69.94MB +Per-Host Resource Estimates: Memory=473.94MB WARNING: The following tables are missing relevant table and/or column statistics. tpch_nested_parquet.customer @@ -2478,7 +2478,7 @@ Per-Host Resources: mem-estimate=128.00MB mem-reservation=34.00MB | tuple-ids=6 row-size=58B cardinality=1500000 | F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3 -Per-Host Resources: mem-estimate=345.06MB mem-reservation=35.06MB +Per-Host Resources: mem-estimate=345.94MB mem-reservation=35.94MB 09:AGGREGATE [STREAMING] | group by: c_name, o1.o_orderkey, o2.o_orderstatus | mem-estimate=128.00MB mem-reservation=0B spill-buffer=2.00MB @@ -2510,7 +2510,7 @@ Per-Host Resources: mem-estimate=345.06MB mem-reservation=35.06MB | 05:HASH JOIN [INNER JOIN] | | hash predicates: o1.o_orderkey = o2.o_orderkey | | fk/pk conjuncts: assumed fk/pk -| | mem-estimate=1.06MB mem-reservation=1.06MB spill-buffer=64.00KB +| | mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB | | tuple-ids=1,2 row-size=32B cardinality=10 | | | |--04:UNNEST [c.c_orders o2] @@ -2531,8 +2531,8 @@ Per-Host Resources: mem-estimate=345.06MB mem-reservation=35.06MB mem-estimate=88.00MB mem-reservation=0B tuple-ids=0 row-size=66B cardinality=150000 ---- PARALLELPLANS -Max Per-Host Resource Reservation: Memory=138.12MB -Per-Host Resource Estimates: Memory=946.13MB +Max Per-Host Resource Reservation: Memory=139.88MB +Per-Host Resource Estimates: Memory=947.88MB WARNING: The following tables are missing relevant table and/or column statistics. tpch_nested_parquet.customer @@ -2557,7 +2557,7 @@ Per-Host Resources: mem-estimate=256.00MB mem-reservation=68.00MB | tuple-ids=6 row-size=58B cardinality=1500000 | F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6 -Per-Host Resources: mem-estimate=690.13MB mem-reservation=70.12MB +Per-Host Resources: mem-estimate=691.88MB mem-reservation=71.88MB 09:AGGREGATE [STREAMING] | group by: c_name, o1.o_orderkey, o2.o_orderstatus | mem-estimate=128.00MB mem-reservation=0B spill-buffer=2.00MB @@ -2589,7 +2589,7 @@ Per-Host Resources: mem-estimate=690.13MB mem-reservation=70.12MB | 05:HASH JOIN [INNER JOIN] | | hash predicates: o1.o_orderkey = o2.o_orderkey | | fk/pk conjuncts: assumed fk/pk -| | mem-estimate=1.06MB mem-reservation=1.06MB spill-buffer=64.00KB +| | mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB | | tuple-ids=1,2 row-size=32B cardinality=10 | | | |--04:UNNEST [c.c_orders o2] @@ -3019,8 +3019,8 @@ Per-Host Resources: mem-estimate=77.77MB mem-reservation=34.00MB mem-estimate=40.00MB mem-reservation=0B tuple-ids=0 row-size=191B cardinality=1500000 ---- PARALLELPLANS -Max Per-Host Resource Reservation: Memory=93.50MB -Per-Host Resource Estimates: Memory=421.03MB +Max Per-Host Resource Reservation: Memory=94.50MB +Per-Host Resource Estimates: Memory=422.03MB F05:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 | Per-Host Resources: mem-estimate=0B mem-reservation=0B @@ -3053,7 +3053,7 @@ Per-Host Resources: mem-estimate=155.53MB mem-reservation=68.00MB | | tuple-ids=1,2,3 row-size=24B cardinality=1500000 | | | F04:PLAN FRAGMENT [HASH(t3.o_orderkey)] hosts=2 instances=4 -| Per-Host Resources: mem-estimate=25.50MB mem-reservation=25.50MB +| Per-Host Resources: mem-estimate=26.50MB mem-reservation=26.50MB | 05:HASH JOIN [INNER JOIN, PARTITIONED] | | hash-table-id=01 | | hash predicates: t2.o_orderkey = t3.o_orderkey @@ -3074,7 +3074,7 @@ Per-Host Resources: mem-estimate=155.53MB mem-reservation=68.00MB | | | hash predicates: t3.o_orderkey = t4.o_orderkey | | | fk/pk conjuncts: t3.o_orderkey = t4.o_orderkey | | | runtime filters: RF002 <- t4.o_orderkey -| | | mem-estimate=4.25MB mem-reservation=4.25MB spill-buffer=256.00KB +| | | mem-estimate=4.75MB mem-reservation=4.75MB spill-buffer=256.00KB | | | tuple-ids=2,3 row-size=16B cardinality=1500000 | | | | | |--F08:PLAN FRAGMENT [HASH(t3.o_orderkey)] hosts=2 instances=4
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ed87c406/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test b/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test index 16e8cb2..340c3ec 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test @@ -3,8 +3,8 @@ select straight_join * from tpch_parquet.customer inner join tpch_parquet.nation on c_nationkey = n_nationkey ---- DISTRIBUTEDPLAN -Max Per-Host Resource Reservation: Memory=1.06MB -Per-Host Resource Estimates: Memory=41.06MB +Max Per-Host Resource Reservation: Memory=1.94MB +Per-Host Resource Estimates: Memory=41.94MB F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 | Per-Host Resources: mem-estimate=0B mem-reservation=0B @@ -16,12 +16,12 @@ PLAN-ROOT SINK | tuple-ids=0,1 row-size=355B cardinality=150000 | F00:PLAN FRAGMENT [RANDOM] hosts=1 instances=1 -Per-Host Resources: mem-estimate=25.06MB mem-reservation=1.06MB +Per-Host Resources: mem-estimate=25.94MB mem-reservation=1.94MB 02:HASH JOIN [INNER JOIN, BROADCAST] | hash predicates: c_nationkey = n_nationkey | fk/pk conjuncts: c_nationkey = n_nationkey | runtime filters: RF000 <- n_nationkey -| mem-estimate=1.06MB mem-reservation=1.06MB spill-buffer=64.00KB +| mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB | tuple-ids=0,1 row-size=355B cardinality=150000 | |--03:EXCHANGE [BROADCAST] @@ -47,8 +47,8 @@ Per-Host Resources: mem-estimate=25.06MB mem-reservation=1.06MB mem-estimate=24.00MB mem-reservation=0B tuple-ids=0 row-size=238B cardinality=150000 ---- PARALLELPLANS -Max Per-Host Resource Reservation: Memory=2.12MB -Per-Host Resource Estimates: Memory=82.12MB +Max Per-Host Resource Reservation: Memory=3.88MB +Per-Host Resource Estimates: Memory=83.88MB F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 | Per-Host Resources: mem-estimate=0B mem-reservation=0B @@ -60,13 +60,13 @@ PLAN-ROOT SINK | tuple-ids=0,1 row-size=355B cardinality=150000 | F00:PLAN FRAGMENT [RANDOM] hosts=1 instances=2 -Per-Host Resources: mem-estimate=50.12MB mem-reservation=2.12MB +Per-Host Resources: mem-estimate=51.88MB mem-reservation=3.88MB 02:HASH JOIN [INNER JOIN, BROADCAST] | hash-table-id=00 | hash predicates: c_nationkey = n_nationkey | fk/pk conjuncts: c_nationkey = n_nationkey | runtime filters: RF000 <- n_nationkey -| mem-estimate=1.06MB mem-reservation=1.06MB spill-buffer=64.00KB +| mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB | tuple-ids=0,1 row-size=355B cardinality=150000 | |--F03:PLAN FRAGMENT [RANDOM] hosts=1 instances=2 @@ -516,7 +516,7 @@ select c_nationkey, avg(c_acctbal) from tpch_parquet.customer group by c_nationkey ---- DISTRIBUTEDPLAN -Max Per-Host Resource Reservation: Memory=1.06MB +Max Per-Host Resource Reservation: Memory=1.94MB Per-Host Resource Estimates: Memory=44.00MB F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 @@ -529,11 +529,11 @@ PLAN-ROOT SINK | tuple-ids=2 row-size=10B cardinality=25 | F01:PLAN FRAGMENT [HASH(c_nationkey)] hosts=1 instances=1 -Per-Host Resources: mem-estimate=10.00MB mem-reservation=1.06MB +Per-Host Resources: mem-estimate=10.00MB mem-reservation=1.94MB 03:AGGREGATE [FINALIZE] | output: avg:merge(c_acctbal) | group by: c_nationkey -| mem-estimate=10.00MB mem-reservation=1.06MB spill-buffer=64.00KB +| mem-estimate=10.00MB mem-reservation=1.94MB spill-buffer=64.00KB | tuple-ids=2 row-size=10B cardinality=25 | 02:EXCHANGE [HASH(c_nationkey)] @@ -556,7 +556,7 @@ Per-Host Resources: mem-estimate=34.00MB mem-reservation=0B mem-estimate=24.00MB mem-reservation=0B tuple-ids=0 row-size=10B cardinality=150000 ---- PARALLELPLANS -Max Per-Host Resource Reservation: Memory=2.12MB +Max Per-Host Resource Reservation: Memory=3.88MB Per-Host Resource Estimates: Memory=88.00MB F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 @@ -569,11 +569,11 @@ PLAN-ROOT SINK | tuple-ids=2 row-size=10B cardinality=25 | F01:PLAN FRAGMENT [HASH(c_nationkey)] hosts=1 instances=2 -Per-Host Resources: mem-estimate=20.00MB mem-reservation=2.12MB +Per-Host Resources: mem-estimate=20.00MB mem-reservation=3.88MB 03:AGGREGATE [FINALIZE] | output: avg:merge(c_acctbal) | group by: c_nationkey -| mem-estimate=10.00MB mem-reservation=1.06MB spill-buffer=64.00KB +| mem-estimate=10.00MB mem-reservation=1.94MB spill-buffer=64.00KB | tuple-ids=2 row-size=10B cardinality=25 | 02:EXCHANGE [HASH(c_nationkey)] http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ed87c406/testdata/workloads/functional-planner/queries/PlannerTest/tablesample.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/tablesample.test b/testdata/workloads/functional-planner/queries/PlannerTest/tablesample.test index 7e78f73..a4c06e0 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/tablesample.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/tablesample.test @@ -154,14 +154,14 @@ select id from functional.alltypes t1 where exists ( where t1.id = t2.id) ---- PLAN F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 -| Per-Host Resources: mem-estimate=161.06MB mem-reservation=1.06MB +| Per-Host Resources: mem-estimate=161.94MB mem-reservation=1.94MB PLAN-ROOT SINK | mem-estimate=0B mem-reservation=0B | 02:HASH JOIN [LEFT SEMI JOIN] | hash predicates: t1.id = t2.id | runtime filters: RF000 <- t2.id -| mem-estimate=1.06MB mem-reservation=1.06MB spill-buffer=64.00KB +| mem-estimate=1.94MB mem-reservation=1.94MB spill-buffer=64.00KB | tuple-ids=0 row-size=4B cardinality=10 | |--01:SCAN HDFS [functional.alltypessmall t2] http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ed87c406/testdata/workloads/functional-query/queries/QueryTest/set.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/set.test b/testdata/workloads/functional-query/queries/QueryTest/set.test index 12b8df6..a7004e0 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/set.test +++ b/testdata/workloads/functional-query/queries/QueryTest/set.test @@ -260,3 +260,34 @@ explain select count(distinct double_col) from functional.alltypesagg; # IMPALA-5591: This shouldn't throw an error. set scratch_limit=-1; ==== +---- QUERY +# Power of two max_row_size. +set max_row_size=8m; +set; +---- RESULTS: VERIFY_IS_SUBSET +'MAX_ROW_SIZE','8388608' +---- TYPES +STRING,STRING +==== +---- QUERY +# Non power of two max_row_size. +set max_row_size=12345; +set; +---- RESULTS: VERIFY_IS_SUBSET +'MAX_ROW_SIZE','12345' +---- TYPES +STRING,STRING +==== +---- QUERY +set max_row_size=8m; +==== +---- QUERY +set max_row_size=-1; +---- CATCH +Max row size must be a positive number of bytes: -1 +==== +---- QUERY +set max_row_size=0; +---- CATCH +Max row size must be a positive number of bytes: 0 +==== http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ed87c406/testdata/workloads/functional-query/queries/QueryTest/spilling-aggs.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/spilling-aggs.test b/testdata/workloads/functional-query/queries/QueryTest/spilling-aggs.test index 34628c0..b7345f2 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/spilling-aggs.test +++ b/testdata/workloads/functional-query/queries/QueryTest/spilling-aggs.test @@ -169,6 +169,7 @@ row_regex: .*SpilledPartitions: .* \([1-9][0-9]*\) ---- QUERY # Test aggregation with minimum required reservation to exercise IMPALA-2708. # Merge aggregation requires 17 buffers. The buffer size is 256k for this test. +set max_row_size=256k; set buffer_pool_limit=4352k; select count(*) from (select distinct * from orders) t http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ed87c406/testdata/workloads/functional-query/queries/QueryTest/spilling-large-rows.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/spilling-large-rows.test b/testdata/workloads/functional-query/queries/QueryTest/spilling-large-rows.test new file mode 100644 index 0000000..9b61986 --- /dev/null +++ b/testdata/workloads/functional-query/queries/QueryTest/spilling-large-rows.test @@ -0,0 +1,188 @@ +==== +---- QUERY +# Create a temporary table with 10MB strings for the following tests. +create table bigstrs stored as parquet as +select *, repeat(string_col, 10000000) as bigstr +from functional.alltypes +order by id +limit 25 +---- RESULTS +'Inserted 25 row(s)' +==== +---- QUERY +# Row is too big to process in agg. +select id, count(distinct bigstr) +from bigstrs +group by id +---- CATCH +Row of size 9.54 MB could not be materialized in plan node +==== +---- QUERY +# Agg should be able to process the large strings if we increase the row size. +set max_row_size=10m; +select id, count(distinct bigstr) +from bigstrs +group by id +order by id +---- TYPES +int,bigint +---- RESULTS +0,1 +1,1 +2,1 +3,1 +4,1 +5,1 +6,1 +7,1 +8,1 +9,1 +10,1 +11,1 +12,1 +13,1 +14,1 +15,1 +16,1 +17,1 +18,1 +19,1 +20,1 +21,1 +22,1 +23,1 +24,1 +==== +---- QUERY +# Row is too big to process in right side of hash join. +select straight_join at.id, bs.id, at.string_col +from functional.alltypes at + join bigstrs bs on repeat(at.string_col, 10000) = substring(bs.bigstr, 5000000, 10000) and at.id = bs.id +where at.id < 100 +---- CATCH +Row of size 9.54 MB could not be materialized in plan node with id 2. Increase the max_row_size query option (currently 512.00 KB) to process larger rows. +==== +---- QUERY +# Row is too big to process in right side of hash join. +set max_row_size=18m; +select straight_join at.id, bs.id, at.string_col +from functional.alltypes at + join bigstrs bs on repeat(at.string_col, 10000) = substring(bs.bigstr, 5000000, 10000) and at.id = bs.id +where at.id < 100 +---- TYPES +int,int,string +---- RESULTS +0,0,'0' +1,1,'1' +2,2,'2' +3,3,'3' +4,4,'4' +5,5,'5' +6,6,'6' +7,7,'7' +8,8,'8' +9,9,'9' +10,10,'0' +11,11,'1' +12,12,'2' +13,13,'3' +14,14,'4' +15,15,'5' +16,16,'6' +17,17,'7' +18,18,'8' +19,19,'9' +20,20,'0' +21,21,'1' +22,22,'2' +23,23,'3' +24,24,'4' +==== +---- QUERY +# Row is too big to process in sort. +select id, substr(bigstr, 1, 5) +from bigstrs +order by bigstr, id +---- CATCH +Row of size 9.54 MB could not be materialized in plan node with id 1. Increase the max_row_size query option (currently 512.00 KB) to process larger rows. +==== +---- QUERY +# Sort should be able to process the large strings if we increase the row size. +set max_row_size=10m; +select id, substr(bigstr, 1, 5) +from bigstrs +where id < 15 +order by bigstr, id +---- TYPES +int,string +---- RESULTS +0,'00000' +10,'00000' +1,'11111' +11,'11111' +2,'22222' +12,'22222' +3,'33333' +13,'33333' +4,'44444' +14,'44444' +5,'55555' +6,'66666' +7,'77777' +8,'88888' +9,'99999' +==== +---- QUERY +# Row is too big to process in sort or analytic. +SELECT id, int_col, substring(bigstr, 1, 10), substring(bigstr, 9999999, 1), rank +FROM ( + SELECT id, int_col, bigstr, Rank() OVER ( + ORDER BY int_col + ) AS rank + FROM bigstrs + ) a +ORDER BY id +---- CATCH +Row of size 9.54 MB could not be materialized in plan node with id 1. Increase the max_row_size query option (currently 512.00 KB) to process larger rows. +==== +---- QUERY +# Sort and analytic should be able to process the large strings if we increase the row +# size. +set max_row_size=10m; +SELECT id, int_col, substring(bigstr, 1, 10), substring(bigstr, 9999999, 1), rank +FROM ( + SELECT id, int_col, bigstr, Rank() OVER ( + ORDER BY int_col + ) AS rank + FROM bigstrs + ) a +ORDER BY id +---- TYPES +int,int,string,string,bigint +---- RESULTS +0,0,'0000000000','0',1 +1,1,'1111111111','1',4 +2,2,'2222222222','2',7 +3,3,'3333333333','3',10 +4,4,'4444444444','4',13 +5,5,'5555555555','5',16 +6,6,'6666666666','6',18 +7,7,'7777777777','7',20 +8,8,'8888888888','8',22 +9,9,'9999999999','9',24 +10,0,'0000000000','0',1 +11,1,'1111111111','1',4 +12,2,'2222222222','2',7 +13,3,'3333333333','3',10 +14,4,'4444444444','4',13 +15,5,'5555555555','5',16 +16,6,'6666666666','6',18 +17,7,'7777777777','7',20 +18,8,'8888888888','8',22 +19,9,'9999999999','9',24 +20,0,'0000000000','0',1 +21,1,'1111111111','1',4 +22,2,'2222222222','2',7 +23,3,'3333333333','3',10 +24,4,'4444444444','4',13 +==== http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ed87c406/tests/query_test/test_spilling.py ---------------------------------------------------------------------- diff --git a/tests/query_test/test_spilling.py b/tests/query_test/test_spilling.py index ee366d4..0ca0971 100644 --- a/tests/query_test/test_spilling.py +++ b/tests/query_test/test_spilling.py @@ -52,6 +52,11 @@ class TestSpilling(ImpalaTestSuite): def test_spilling_aggs(self, vector): self.run_test_case('QueryTest/spilling-aggs', vector) + def test_spilling_large_rows(self, vector, unique_database): + """Test that we can process large rows in spilling operators, with or without + spilling to disk""" + self.run_test_case('QueryTest/spilling-large-rows', vector, unique_database) + def test_spilling_sorts_exhaustive(self, vector): if self.exploration_strategy() != 'exhaustive': pytest.skip("only run large sorts on exhaustive")
