Repository: impala Updated Branches: refs/heads/2.x c8c694779 -> 83a70a7ae
http://git-wip-us.apache.org/repos/asf/impala/blob/83a70a7a/testdata/workloads/functional-planner/queries/PlannerTest/sort-expr-materialization.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/sort-expr-materialization.test b/testdata/workloads/functional-planner/queries/PlannerTest/sort-expr-materialization.test index 7b70a9d..4925f8f 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/sort-expr-materialization.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/sort-expr-materialization.test @@ -18,7 +18,7 @@ PLAN-ROOT SINK table: rows=7300 size=478.45KB partitions: 24/24 rows=7300 columns: all - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=310 mem-estimate=128.00MB mem-reservation=32.00KB tuple-ids=0 row-size=97B cardinality=7300 ==== @@ -42,7 +42,7 @@ PLAN-ROOT SINK table: rows=7300 size=478.45KB partitions: 24/24 rows=7300 columns: all - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=310 mem-estimate=128.00MB mem-reservation=32.00KB tuple-ids=0 row-size=97B cardinality=7300 ==== @@ -65,7 +65,7 @@ PLAN-ROOT SINK table: rows=7300 size=478.45KB partitions: 24/24 rows=7300 columns: all - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=310 mem-estimate=128.00MB mem-reservation=32.00KB tuple-ids=0 row-size=97B cardinality=7300 ==== @@ -90,7 +90,7 @@ PLAN-ROOT SINK table: rows=7300 size=478.45KB partitions: 24/24 rows=7300 columns: all - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=310 mem-estimate=128.00MB mem-reservation=32.00KB tuple-ids=0 row-size=97B cardinality=7300 ==== @@ -122,7 +122,7 @@ PLAN-ROOT SINK table: rows=7300 size=478.45KB partitions: 24/24 rows=7300 columns: all - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=310 mem-estimate=128.00MB mem-reservation=32.00KB tuple-ids=0 row-size=21B cardinality=7300 ==== @@ -147,7 +147,7 @@ PLAN-ROOT SINK table: rows=7300 size=478.45KB partitions: 24/24 rows=7300 columns: all - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=310 mem-estimate=128.00MB mem-reservation=32.00KB tuple-ids=0 row-size=29B cardinality=7300 ==== @@ -171,7 +171,7 @@ PLAN-ROOT SINK table: rows=7300 size=478.45KB partitions: 24/24 rows=7300 columns: all - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=310 mem-estimate=128.00MB mem-reservation=32.00KB tuple-ids=0 row-size=97B cardinality=7300 ==== @@ -195,7 +195,7 @@ PLAN-ROOT SINK table: rows=7300 size=478.45KB partitions: 24/24 rows=7300 columns: all - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=310 mem-estimate=128.00MB mem-reservation=32.00KB tuple-ids=0 row-size=41B cardinality=7300 ==== @@ -236,7 +236,7 @@ PLAN-ROOT SINK table: rows=8 size=460B partitions: 4/4 rows=8 columns: all - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=2 mem-estimate=32.00MB mem-reservation=8.00KB tuple-ids=0 row-size=12B cardinality=8 ==== http://git-wip-us.apache.org/repos/asf/impala/blob/83a70a7a/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test b/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test index f57e578..f7dfcff 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test @@ -31,11 +31,11 @@ Per-Host Resources: mem-estimate=26.94MB mem-reservation=18.94MB runtime-filters | F01:PLAN FRAGMENT [RANDOM] hosts=1 instances=1 | Per-Host Resources: mem-estimate=16.00MB mem-reservation=32.00KB | 01:SCAN HDFS [tpch_parquet.nation, RANDOM] -| partitions=1/1 files=1 size=2.74KB +| partitions=1/1 files=1 size=2.75KB | stored statistics: -| table: rows=25 size=2.74KB +| table: rows=25 size=2.75KB | columns: all -| extrapolated-rows=disabled +| extrapolated-rows=disabled max-scan-range-rows=25 | mem-estimate=16.00MB mem-reservation=32.00KB | tuple-ids=1 row-size=117B cardinality=25 | @@ -45,7 +45,7 @@ Per-Host Resources: mem-estimate=26.94MB mem-reservation=18.94MB runtime-filters stored statistics: table: rows=150000 size=12.31MB columns: all - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=150000 mem-estimate=24.00MB mem-reservation=16.00MB tuple-ids=0 row-size=238B cardinality=150000 ---- PARALLELPLANS @@ -85,11 +85,11 @@ Per-Host Resources: mem-estimate=53.88MB mem-reservation=37.88MB runtime-filters | F01:PLAN FRAGMENT [RANDOM] hosts=1 instances=2 | Per-Host Resources: mem-estimate=32.00MB mem-reservation=64.00KB | 01:SCAN HDFS [tpch_parquet.nation, RANDOM] -| partitions=1/1 files=1 size=2.74KB +| partitions=1/1 files=1 size=2.75KB | stored statistics: -| table: rows=25 size=2.74KB +| table: rows=25 size=2.75KB | columns: all -| extrapolated-rows=disabled +| extrapolated-rows=disabled max-scan-range-rows=25 | mem-estimate=16.00MB mem-reservation=32.00KB | tuple-ids=1 row-size=117B cardinality=25 | @@ -99,7 +99,7 @@ Per-Host Resources: mem-estimate=53.88MB mem-reservation=37.88MB runtime-filters stored statistics: table: rows=150000 size=12.31MB columns: all - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=150000 mem-estimate=24.00MB mem-reservation=16.00MB tuple-ids=0 row-size=238B cardinality=150000 ==== @@ -108,8 +108,8 @@ select straight_join * from tpch_parquet.lineitem left join tpch_parquet.orders on l_orderkey = o_orderkey ---- DISTRIBUTEDPLAN -Max Per-Host Resource Reservation: Memory=154.00MB -Per-Host Resource Estimates: Memory=428.41MB +Max Per-Host Resource Reservation: Memory=98.00MB +Per-Host Resource Estimates: Memory=420.41MB F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 | Per-Host Resources: mem-estimate=0B mem-reservation=0B @@ -121,7 +121,7 @@ PLAN-ROOT SINK | tuple-ids=0,1N row-size=454B cardinality=6001215 | F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3 -Per-Host Resources: mem-estimate=380.41MB mem-reservation=106.00MB +Per-Host Resources: mem-estimate=380.41MB mem-reservation=74.00MB 02:HASH JOIN [LEFT OUTER JOIN, BROADCAST] | hash predicates: l_orderkey = o_orderkey | fk/pk conjuncts: l_orderkey = o_orderkey @@ -133,27 +133,27 @@ Per-Host Resources: mem-estimate=380.41MB mem-reservation=106.00MB | | tuple-ids=1 row-size=191B cardinality=1500000 | | | F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=2 -| Per-Host Resources: mem-estimate=48.00MB mem-reservation=48.00MB +| Per-Host Resources: mem-estimate=40.00MB mem-reservation=24.00MB | 01:SCAN HDFS [tpch_parquet.orders, RANDOM] | partitions=1/1 files=2 size=54.07MB | stored statistics: | table: rows=1500000 size=54.07MB | columns: all -| extrapolated-rows=disabled -| mem-estimate=48.00MB mem-reservation=48.00MB +| extrapolated-rows=disabled max-scan-range-rows=1177135 +| mem-estimate=40.00MB mem-reservation=24.00MB | tuple-ids=1 row-size=191B cardinality=1500000 | 00:SCAN HDFS [tpch_parquet.lineitem, RANDOM] - partitions=1/1 files=3 size=193.72MB + partitions=1/1 files=3 size=193.71MB stored statistics: - table: rows=6001215 size=193.72MB + table: rows=6001215 size=193.71MB columns: all - extrapolated-rows=disabled - mem-estimate=80.00MB mem-reservation=72.00MB + extrapolated-rows=disabled max-scan-range-rows=2141802 + mem-estimate=80.00MB mem-reservation=40.00MB tuple-ids=0 row-size=263B cardinality=6001215 ---- PARALLELPLANS -Max Per-Host Resource Reservation: Memory=308.00MB -Per-Host Resource Estimates: Memory=856.83MB +Max Per-Host Resource Reservation: Memory=196.00MB +Per-Host Resource Estimates: Memory=840.83MB F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 | Per-Host Resources: mem-estimate=0B mem-reservation=0B @@ -165,7 +165,7 @@ PLAN-ROOT SINK | tuple-ids=0,1N row-size=454B cardinality=6001215 | F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6 -Per-Host Resources: mem-estimate=760.83MB mem-reservation=212.00MB +Per-Host Resources: mem-estimate=760.83MB mem-reservation=148.00MB 02:HASH JOIN [LEFT OUTER JOIN, BROADCAST] | hash-table-id=00 | hash predicates: l_orderkey = o_orderkey @@ -185,23 +185,23 @@ Per-Host Resources: mem-estimate=760.83MB mem-reservation=212.00MB | | tuple-ids=1 row-size=191B cardinality=1500000 | | | F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=4 -| Per-Host Resources: mem-estimate=96.00MB mem-reservation=96.00MB +| Per-Host Resources: mem-estimate=80.00MB mem-reservation=48.00MB | 01:SCAN HDFS [tpch_parquet.orders, RANDOM] | partitions=1/1 files=2 size=54.07MB | stored statistics: | table: rows=1500000 size=54.07MB | columns: all -| extrapolated-rows=disabled -| mem-estimate=48.00MB mem-reservation=48.00MB +| extrapolated-rows=disabled max-scan-range-rows=1177135 +| mem-estimate=40.00MB mem-reservation=24.00MB | tuple-ids=1 row-size=191B cardinality=1500000 | 00:SCAN HDFS [tpch_parquet.lineitem, RANDOM] - partitions=1/1 files=3 size=193.72MB + partitions=1/1 files=3 size=193.71MB stored statistics: - table: rows=6001215 size=193.72MB + table: rows=6001215 size=193.71MB columns: all - extrapolated-rows=disabled - mem-estimate=80.00MB mem-reservation=72.00MB + extrapolated-rows=disabled max-scan-range-rows=2141802 + mem-estimate=80.00MB mem-reservation=40.00MB tuple-ids=0 row-size=263B cardinality=6001215 ==== # Shuffle join with mid-sized input. @@ -209,8 +209,8 @@ select straight_join * from tpch_parquet.orders join /*+shuffle*/ tpch_parquet.customer on o_custkey = c_custkey ---- DISTRIBUTEDPLAN -Max Per-Host Resource Reservation: Memory=100.00MB -Per-Host Resource Estimates: Memory=108.00MB +Max Per-Host Resource Reservation: Memory=76.00MB +Per-Host Resource Estimates: Memory=100.00MB F03:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 | Per-Host Resources: mem-estimate=0B mem-reservation=0B @@ -241,7 +241,7 @@ Per-Host Resources: mem-estimate=35.00MB mem-reservation=35.00MB runtime-filters | stored statistics: | table: rows=150000 size=12.31MB | columns: all -| extrapolated-rows=disabled +| extrapolated-rows=disabled max-scan-range-rows=150000 | mem-estimate=24.00MB mem-reservation=16.00MB | tuple-ids=1 row-size=238B cardinality=150000 | @@ -250,19 +250,19 @@ Per-Host Resources: mem-estimate=35.00MB mem-reservation=35.00MB runtime-filters | tuple-ids=0 row-size=191B cardinality=1500000 | F00:PLAN FRAGMENT [RANDOM] hosts=2 instances=2 -Per-Host Resources: mem-estimate=49.00MB mem-reservation=49.00MB runtime-filters-memory=1.00MB +Per-Host Resources: mem-estimate=41.00MB mem-reservation=25.00MB runtime-filters-memory=1.00MB 00:SCAN HDFS [tpch_parquet.orders, RANDOM] partitions=1/1 files=2 size=54.07MB runtime filters: RF000[bloom] -> o_custkey stored statistics: table: rows=1500000 size=54.07MB columns: all - extrapolated-rows=disabled - mem-estimate=48.00MB mem-reservation=48.00MB + extrapolated-rows=disabled max-scan-range-rows=1177135 + mem-estimate=40.00MB mem-reservation=24.00MB tuple-ids=0 row-size=191B cardinality=1500000 ---- PARALLELPLANS -Max Per-Host Resource Reservation: Memory=166.00MB -Per-Host Resource Estimates: Memory=182.00MB +Max Per-Host Resource Reservation: Memory=118.00MB +Per-Host Resource Estimates: Memory=166.00MB F03:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 | Per-Host Resources: mem-estimate=0B mem-reservation=0B @@ -301,7 +301,7 @@ Per-Host Resources: mem-estimate=36.00MB mem-reservation=36.00MB runtime-filters | stored statistics: | table: rows=150000 size=12.31MB | columns: all -| extrapolated-rows=disabled +| extrapolated-rows=disabled max-scan-range-rows=150000 | mem-estimate=24.00MB mem-reservation=16.00MB | tuple-ids=1 row-size=238B cardinality=150000 | @@ -310,15 +310,15 @@ Per-Host Resources: mem-estimate=36.00MB mem-reservation=36.00MB runtime-filters | tuple-ids=0 row-size=191B cardinality=1500000 | F00:PLAN FRAGMENT [RANDOM] hosts=2 instances=4 -Per-Host Resources: mem-estimate=98.00MB mem-reservation=98.00MB runtime-filters-memory=1.00MB +Per-Host Resources: mem-estimate=82.00MB mem-reservation=50.00MB runtime-filters-memory=1.00MB 00:SCAN HDFS [tpch_parquet.orders, RANDOM] partitions=1/1 files=2 size=54.07MB runtime filters: RF000[bloom] -> o_custkey stored statistics: table: rows=1500000 size=54.07MB columns: all - extrapolated-rows=disabled - mem-estimate=48.00MB mem-reservation=48.00MB + extrapolated-rows=disabled max-scan-range-rows=1177135 + mem-estimate=40.00MB mem-reservation=24.00MB tuple-ids=0 row-size=191B cardinality=1500000 ==== # Broadcast join with mid-sized input - should use larger buffers than shuffle join. @@ -326,8 +326,8 @@ select straight_join * from tpch_parquet.orders join /*+broadcast*/ tpch_parquet.customer on o_custkey = c_custkey ---- DISTRIBUTEDPLAN -Max Per-Host Resource Reservation: Memory=99.00MB -Per-Host Resource Estimates: Memory=110.38MB +Max Per-Host Resource Reservation: Memory=75.00MB +Per-Host Resource Estimates: Memory=102.38MB F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 | Per-Host Resources: mem-estimate=0B mem-reservation=0B @@ -339,7 +339,7 @@ PLAN-ROOT SINK | tuple-ids=0,1 row-size=428B cardinality=1500000 | F00:PLAN FRAGMENT [RANDOM] hosts=2 instances=2 -Per-Host Resources: mem-estimate=86.38MB mem-reservation=83.00MB runtime-filters-memory=1.00MB +Per-Host Resources: mem-estimate=78.38MB mem-reservation=59.00MB runtime-filters-memory=1.00MB 02:HASH JOIN [INNER JOIN, BROADCAST] | hash predicates: o_custkey = c_custkey | fk/pk conjuncts: o_custkey = c_custkey @@ -358,7 +358,7 @@ Per-Host Resources: mem-estimate=86.38MB mem-reservation=83.00MB runtime-filters | stored statistics: | table: rows=150000 size=12.31MB | columns: all -| extrapolated-rows=disabled +| extrapolated-rows=disabled max-scan-range-rows=150000 | mem-estimate=24.00MB mem-reservation=16.00MB | tuple-ids=1 row-size=238B cardinality=150000 | @@ -368,12 +368,12 @@ Per-Host Resources: mem-estimate=86.38MB mem-reservation=83.00MB runtime-filters stored statistics: table: rows=1500000 size=54.07MB columns: all - extrapolated-rows=disabled - mem-estimate=48.00MB mem-reservation=48.00MB + extrapolated-rows=disabled max-scan-range-rows=1177135 + mem-estimate=40.00MB mem-reservation=24.00MB tuple-ids=0 row-size=191B cardinality=1500000 ---- PARALLELPLANS -Max Per-Host Resource Reservation: Memory=198.00MB -Per-Host Resource Estimates: Memory=220.76MB +Max Per-Host Resource Reservation: Memory=150.00MB +Per-Host Resource Estimates: Memory=204.76MB F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 | Per-Host Resources: mem-estimate=0B mem-reservation=0B @@ -385,7 +385,7 @@ PLAN-ROOT SINK | tuple-ids=0,1 row-size=428B cardinality=1500000 | F00:PLAN FRAGMENT [RANDOM] hosts=2 instances=4 -Per-Host Resources: mem-estimate=172.76MB mem-reservation=166.00MB runtime-filters-memory=1.00MB +Per-Host Resources: mem-estimate=156.76MB mem-reservation=118.00MB runtime-filters-memory=1.00MB 02:HASH JOIN [INNER JOIN, BROADCAST] | hash-table-id=00 | hash predicates: o_custkey = c_custkey @@ -412,7 +412,7 @@ Per-Host Resources: mem-estimate=172.76MB mem-reservation=166.00MB runtime-filte | stored statistics: | table: rows=150000 size=12.31MB | columns: all -| extrapolated-rows=disabled +| extrapolated-rows=disabled max-scan-range-rows=150000 | mem-estimate=24.00MB mem-reservation=16.00MB | tuple-ids=1 row-size=238B cardinality=150000 | @@ -422,8 +422,8 @@ Per-Host Resources: mem-estimate=172.76MB mem-reservation=166.00MB runtime-filte stored statistics: table: rows=1500000 size=54.07MB columns: all - extrapolated-rows=disabled - mem-estimate=48.00MB mem-reservation=48.00MB + extrapolated-rows=disabled max-scan-range-rows=1177135 + mem-estimate=40.00MB mem-reservation=24.00MB tuple-ids=0 row-size=191B cardinality=1500000 ==== # Join with no stats for right input - should use default buffers. @@ -460,22 +460,22 @@ Per-Host Resources: mem-estimate=2.02GB mem-reservation=34.09MB | F01:PLAN FRAGMENT [RANDOM] hosts=3 instances=3 | Per-Host Resources: mem-estimate=16.00MB mem-reservation=88.00KB | 01:SCAN HDFS [functional_parquet.alltypestiny, RANDOM] -| partitions=4/4 files=4 size=9.75KB +| partitions=4/4 files=4 size=9.79KB | stored statistics: | table: rows=unavailable size=unavailable | partitions: 0/4 rows=unavailable | columns: unavailable -| extrapolated-rows=disabled +| extrapolated-rows=disabled max-scan-range-rows=unavailable | mem-estimate=16.00MB mem-reservation=88.00KB | tuple-ids=1 row-size=88B cardinality=unavailable | 00:SCAN HDFS [functional_parquet.alltypes, RANDOM] - partitions=24/24 files=24 size=174.39KB + partitions=24/24 files=24 size=188.29KB stored statistics: table: rows=unavailable size=unavailable partitions: 0/24 rows=unavailable columns missing stats: id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=unavailable mem-estimate=16.00MB mem-reservation=88.00KB tuple-ids=0 row-size=88B cardinality=unavailable ---- PARALLELPLANS @@ -516,22 +516,22 @@ Per-Host Resources: mem-estimate=4.03GB mem-reservation=68.17MB | F01:PLAN FRAGMENT [RANDOM] hosts=3 instances=6 | Per-Host Resources: mem-estimate=32.00MB mem-reservation=176.00KB | 01:SCAN HDFS [functional_parquet.alltypestiny, RANDOM] -| partitions=4/4 files=4 size=9.75KB +| partitions=4/4 files=4 size=9.79KB | stored statistics: | table: rows=unavailable size=unavailable | partitions: 0/4 rows=unavailable | columns: unavailable -| extrapolated-rows=disabled +| extrapolated-rows=disabled max-scan-range-rows=unavailable | mem-estimate=16.00MB mem-reservation=88.00KB | tuple-ids=1 row-size=88B cardinality=unavailable | 00:SCAN HDFS [functional_parquet.alltypes, RANDOM] - partitions=24/24 files=24 size=174.39KB + partitions=24/24 files=24 size=188.29KB stored statistics: table: rows=unavailable size=unavailable partitions: 0/24 rows=unavailable columns missing stats: id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=unavailable mem-estimate=16.00MB mem-reservation=88.00KB tuple-ids=0 row-size=88B cardinality=unavailable ==== @@ -540,7 +540,7 @@ select c_nationkey, avg(c_acctbal) from tpch_parquet.customer group by c_nationkey ---- DISTRIBUTEDPLAN -Max Per-Host Resource Reservation: Memory=19.94MB +Max Per-Host Resource Reservation: Memory=5.94MB Per-Host Resource Estimates: Memory=44.00MB F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 @@ -565,7 +565,7 @@ Per-Host Resources: mem-estimate=10.00MB mem-reservation=1.94MB | tuple-ids=1 row-size=10B cardinality=25 | F00:PLAN FRAGMENT [RANDOM] hosts=1 instances=1 -Per-Host Resources: mem-estimate=34.00MB mem-reservation=18.00MB +Per-Host Resources: mem-estimate=34.00MB mem-reservation=4.00MB 01:AGGREGATE [STREAMING] | output: avg(c_acctbal) | group by: c_nationkey @@ -577,11 +577,11 @@ Per-Host Resources: mem-estimate=34.00MB mem-reservation=18.00MB stored statistics: table: rows=150000 size=12.31MB columns: all - extrapolated-rows=disabled - mem-estimate=24.00MB mem-reservation=16.00MB + extrapolated-rows=disabled max-scan-range-rows=150000 + mem-estimate=24.00MB mem-reservation=2.00MB tuple-ids=0 row-size=10B cardinality=150000 ---- PARALLELPLANS -Max Per-Host Resource Reservation: Memory=39.88MB +Max Per-Host Resource Reservation: Memory=11.88MB Per-Host Resource Estimates: Memory=88.00MB F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 @@ -606,7 +606,7 @@ Per-Host Resources: mem-estimate=20.00MB mem-reservation=3.88MB | tuple-ids=1 row-size=10B cardinality=25 | F00:PLAN FRAGMENT [RANDOM] hosts=1 instances=2 -Per-Host Resources: mem-estimate=68.00MB mem-reservation=36.00MB +Per-Host Resources: mem-estimate=68.00MB mem-reservation=8.00MB 01:AGGREGATE [STREAMING] | output: avg(c_acctbal) | group by: c_nationkey @@ -618,8 +618,8 @@ Per-Host Resources: mem-estimate=68.00MB mem-reservation=36.00MB stored statistics: table: rows=150000 size=12.31MB columns: all - extrapolated-rows=disabled - mem-estimate=24.00MB mem-reservation=16.00MB + extrapolated-rows=disabled max-scan-range-rows=150000 + mem-estimate=24.00MB mem-reservation=2.00MB tuple-ids=0 row-size=10B cardinality=150000 ==== # Mid NDV aggregation - should scale down buffers to intermediate size. @@ -629,7 +629,7 @@ from tpch_parquet.lineitem group by 1, 2 having count(*) = 1 ---- DISTRIBUTEDPLAN -Max Per-Host Resource Reservation: Memory=111.00MB +Max Per-Host Resource Reservation: Memory=99.00MB Per-Host Resource Estimates: Memory=227.12MB F04:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 @@ -674,14 +674,14 @@ Per-Host Resources: mem-estimate=72.12MB mem-reservation=52.00MB runtime-filters | | tuple-ids=1 row-size=25B cardinality=1500000 | | | F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=2 -| Per-Host Resources: mem-estimate=40.00MB mem-reservation=16.00MB +| Per-Host Resources: mem-estimate=40.00MB mem-reservation=8.00MB | 01:SCAN HDFS [tpch_parquet.orders, RANDOM] | partitions=1/1 files=2 size=54.07MB | stored statistics: | table: rows=1500000 size=54.07MB | columns: all -| extrapolated-rows=disabled -| mem-estimate=40.00MB mem-reservation=16.00MB +| extrapolated-rows=disabled max-scan-range-rows=1177135 +| mem-estimate=40.00MB mem-reservation=8.00MB | tuple-ids=1 row-size=25B cardinality=1500000 | 04:EXCHANGE [HASH(l_orderkey)] @@ -689,18 +689,18 @@ Per-Host Resources: mem-estimate=72.12MB mem-reservation=52.00MB runtime-filters | tuple-ids=0 row-size=8B cardinality=6001215 | F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3 -Per-Host Resources: mem-estimate=81.00MB mem-reservation=9.00MB runtime-filters-memory=1.00MB +Per-Host Resources: mem-estimate=81.00MB mem-reservation=5.00MB runtime-filters-memory=1.00MB 00:SCAN HDFS [tpch_parquet.lineitem, RANDOM] - partitions=1/1 files=3 size=193.72MB + partitions=1/1 files=3 size=193.71MB runtime filters: RF000[bloom] -> l_orderkey stored statistics: - table: rows=6001215 size=193.72MB + table: rows=6001215 size=193.71MB columns: all - extrapolated-rows=disabled - mem-estimate=80.00MB mem-reservation=8.00MB + extrapolated-rows=disabled max-scan-range-rows=2141802 + mem-estimate=80.00MB mem-reservation=4.00MB tuple-ids=0 row-size=8B cardinality=6001215 ---- PARALLELPLANS -Max Per-Host Resource Reservation: Memory=171.00MB +Max Per-Host Resource Reservation: Memory=147.00MB Per-Host Resource Estimates: Memory=363.00MB F04:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 @@ -753,14 +753,14 @@ Per-Host Resources: mem-estimate=87.00MB mem-reservation=87.00MB runtime-filters | | tuple-ids=1 row-size=25B cardinality=1500000 | | | F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=4 -| Per-Host Resources: mem-estimate=80.00MB mem-reservation=32.00MB +| Per-Host Resources: mem-estimate=80.00MB mem-reservation=16.00MB | 01:SCAN HDFS [tpch_parquet.orders, RANDOM] | partitions=1/1 files=2 size=54.07MB | stored statistics: | table: rows=1500000 size=54.07MB | columns: all -| extrapolated-rows=disabled -| mem-estimate=40.00MB mem-reservation=16.00MB +| extrapolated-rows=disabled max-scan-range-rows=1177135 +| mem-estimate=40.00MB mem-reservation=8.00MB | tuple-ids=1 row-size=25B cardinality=1500000 | 04:EXCHANGE [HASH(l_orderkey)] @@ -768,22 +768,22 @@ Per-Host Resources: mem-estimate=87.00MB mem-reservation=87.00MB runtime-filters | tuple-ids=0 row-size=8B cardinality=6001215 | F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6 -Per-Host Resources: mem-estimate=162.00MB mem-reservation=18.00MB runtime-filters-memory=1.00MB +Per-Host Resources: mem-estimate=162.00MB mem-reservation=10.00MB runtime-filters-memory=1.00MB 00:SCAN HDFS [tpch_parquet.lineitem, RANDOM] - partitions=1/1 files=3 size=193.72MB + partitions=1/1 files=3 size=193.71MB runtime filters: RF000[bloom] -> l_orderkey stored statistics: - table: rows=6001215 size=193.72MB + table: rows=6001215 size=193.71MB columns: all - extrapolated-rows=disabled - mem-estimate=80.00MB mem-reservation=8.00MB + extrapolated-rows=disabled max-scan-range-rows=2141802 + mem-estimate=80.00MB mem-reservation=4.00MB tuple-ids=0 row-size=8B cardinality=6001215 ==== # High NDV aggregation - should use default buffer size. select distinct * from tpch_parquet.lineitem ---- DISTRIBUTEDPLAN -Max Per-Host Resource Reservation: Memory=140.00MB +Max Per-Host Resource Reservation: Memory=108.00MB Per-Host Resource Estimates: Memory=3.31GB F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 @@ -807,22 +807,22 @@ Per-Host Resources: mem-estimate=1.62GB mem-reservation=34.00MB | tuple-ids=1 row-size=263B cardinality=6001215 | F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3 -Per-Host Resources: mem-estimate=1.69GB mem-reservation=106.00MB +Per-Host Resources: mem-estimate=1.69GB mem-reservation=74.00MB 01:AGGREGATE [STREAMING] | group by: tpch_parquet.lineitem.l_orderkey, tpch_parquet.lineitem.l_partkey, tpch_parquet.lineitem.l_suppkey, tpch_parquet.lineitem.l_linenumber, tpch_parquet.lineitem.l_quantity, tpch_parquet.lineitem.l_extendedprice, tpch_parquet.lineitem.l_discount, tpch_parquet.lineitem.l_tax, tpch_parquet.lineitem.l_returnflag, tpch_parquet.lineitem.l_linestatus, tpch_parquet.lineitem.l_shipdate, tpch_parquet.lineitem.l_commitdate, tpch_parquet.lineitem.l_receiptdate, tpch_parquet.lineitem.l_shipinstruct, tpch_parquet.lineitem.l_shipmode, tpch_parquet.lineitem.l_comment | mem-estimate=1.62GB mem-reservation=34.00MB spill-buffer=2.00MB | tuple-ids=1 row-size=263B cardinality=6001215 | 00:SCAN HDFS [tpch_parquet.lineitem, RANDOM] - partitions=1/1 files=3 size=193.72MB + partitions=1/1 files=3 size=193.71MB stored statistics: - table: rows=6001215 size=193.72MB + table: rows=6001215 size=193.71MB columns: all - extrapolated-rows=disabled - mem-estimate=80.00MB mem-reservation=72.00MB + extrapolated-rows=disabled max-scan-range-rows=2141802 + mem-estimate=80.00MB mem-reservation=40.00MB tuple-ids=0 row-size=263B cardinality=6001215 ---- PARALLELPLANS -Max Per-Host Resource Reservation: Memory=280.00MB +Max Per-Host Resource Reservation: Memory=216.00MB Per-Host Resource Estimates: Memory=6.62GB F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 @@ -846,19 +846,19 @@ Per-Host Resources: mem-estimate=3.23GB mem-reservation=68.00MB | tuple-ids=1 row-size=263B cardinality=6001215 | F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6 -Per-Host Resources: mem-estimate=3.39GB mem-reservation=212.00MB +Per-Host Resources: mem-estimate=3.39GB mem-reservation=148.00MB 01:AGGREGATE [STREAMING] | group by: tpch_parquet.lineitem.l_orderkey, tpch_parquet.lineitem.l_partkey, tpch_parquet.lineitem.l_suppkey, tpch_parquet.lineitem.l_linenumber, tpch_parquet.lineitem.l_quantity, tpch_parquet.lineitem.l_extendedprice, tpch_parquet.lineitem.l_discount, tpch_parquet.lineitem.l_tax, tpch_parquet.lineitem.l_returnflag, tpch_parquet.lineitem.l_linestatus, tpch_parquet.lineitem.l_shipdate, tpch_parquet.lineitem.l_commitdate, tpch_parquet.lineitem.l_receiptdate, tpch_parquet.lineitem.l_shipinstruct, tpch_parquet.lineitem.l_shipmode, tpch_parquet.lineitem.l_comment | mem-estimate=1.62GB mem-reservation=34.00MB spill-buffer=2.00MB | tuple-ids=1 row-size=263B cardinality=6001215 | 00:SCAN HDFS [tpch_parquet.lineitem, RANDOM] - partitions=1/1 files=3 size=193.72MB + partitions=1/1 files=3 size=193.71MB stored statistics: - table: rows=6001215 size=193.72MB + table: rows=6001215 size=193.71MB columns: all - extrapolated-rows=disabled - mem-estimate=80.00MB mem-reservation=72.00MB + extrapolated-rows=disabled max-scan-range-rows=2141802 + mem-estimate=80.00MB mem-reservation=40.00MB tuple-ids=0 row-size=263B cardinality=6001215 ==== # Aggregation with unknown input - should use default buffer size. @@ -901,12 +901,12 @@ Per-Host Resources: mem-estimate=144.00MB mem-reservation=34.01MB | tuple-ids=1 row-size=24B cardinality=unavailable | 00:SCAN HDFS [functional_parquet.alltypestiny, RANDOM] - partitions=4/4 files=4 size=9.75KB + partitions=4/4 files=4 size=9.79KB stored statistics: table: rows=unavailable size=unavailable partitions: 0/4 rows=unavailable columns: unavailable - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=unavailable mem-estimate=16.00MB mem-reservation=8.00KB tuple-ids=0 row-size=16B cardinality=unavailable ---- PARALLELPLANS @@ -945,12 +945,12 @@ Per-Host Resources: mem-estimate=288.00MB mem-reservation=68.02MB | tuple-ids=1 row-size=24B cardinality=unavailable | 00:SCAN HDFS [functional_parquet.alltypestiny, RANDOM] - partitions=4/4 files=4 size=9.75KB + partitions=4/4 files=4 size=9.79KB stored statistics: table: rows=unavailable size=unavailable partitions: 0/4 rows=unavailable columns: unavailable - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=unavailable mem-estimate=16.00MB mem-reservation=8.00KB tuple-ids=0 row-size=16B cardinality=unavailable ==== http://git-wip-us.apache.org/repos/asf/impala/blob/83a70a7a/testdata/workloads/functional-planner/queries/PlannerTest/tablesample.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/tablesample.test b/testdata/workloads/functional-planner/queries/PlannerTest/tablesample.test index 1844f11..86ef666 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/tablesample.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/tablesample.test @@ -12,7 +12,7 @@ PLAN-ROOT SINK table: rows=7300 size=478.45KB partitions: 24/24 rows=7300 columns: all - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=2449 mem-estimate=32.00MB mem-reservation=32.00KB tuple-ids=0 row-size=97B cardinality=730 ==== @@ -30,7 +30,7 @@ PLAN-ROOT SINK table: rows=7300 size=478.45KB partitions: 24/24 rows=7300 columns: all - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=621 mem-estimate=80.00MB mem-reservation=32.00KB tuple-ids=0 row-size=97B cardinality=3650 ==== @@ -51,7 +51,7 @@ PLAN-ROOT SINK table: rows=7300 size=478.45KB partitions: 24/24 rows=7300 columns: all - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=621 parquet dictionary predicates: id < 10 mem-estimate=80.00MB mem-reservation=32.00KB tuple-ids=0 row-size=97B cardinality=365 @@ -71,7 +71,7 @@ PLAN-ROOT SINK table: rows=7300 size=478.45KB partitions: 12/12 rows=3650 columns: all - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=1241 mem-estimate=48.00MB mem-reservation=32.00KB tuple-ids=0 row-size=97B cardinality=1825 ==== @@ -89,7 +89,7 @@ PLAN-ROOT SINK table: rows=7300 size=478.45KB partitions: 24/24 rows=7300 columns: all - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=0 mem-estimate=0B mem-reservation=0B tuple-ids=0 row-size=97B cardinality=0 ==== @@ -107,7 +107,7 @@ PLAN-ROOT SINK table: rows=7300 size=478.45KB partitions: 24/24 rows=7300 columns: all - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=7300 mem-estimate=32.00MB mem-reservation=32.00KB tuple-ids=0 row-size=97B cardinality=73 ==== @@ -126,7 +126,7 @@ PLAN-ROOT SINK table: rows=7300 size=478.45KB partitions: 12/12 rows=3650 columns: all - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=7300 mem-estimate=32.00MB mem-reservation=32.00KB tuple-ids=0 row-size=97B cardinality=37 ==== @@ -144,7 +144,7 @@ PLAN-ROOT SINK table: rows=7300 size=478.45KB partitions: 24/24 rows=7300 columns: all - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=310 mem-estimate=128.00MB mem-reservation=32.00KB tuple-ids=0 row-size=97B cardinality=7300 ==== @@ -162,7 +162,7 @@ PLAN-ROOT SINK table: rows=unavailable size=unavailable partitions: 0/24 rows=unavailable columns: unavailable - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=unavailable mem-estimate=16.00MB mem-reservation=8.00KB tuple-ids=0 row-size=4B cardinality=unavailable ==== @@ -188,7 +188,7 @@ PLAN-ROOT SINK | table: rows=100 size=6.32KB | partitions: 4/4 rows=100 | columns: all -| extrapolated-rows=disabled +| extrapolated-rows=disabled max-scan-range-rows=100 | mem-estimate=32.00MB mem-reservation=8.00KB | tuple-ids=1 row-size=4B cardinality=10 | @@ -199,7 +199,7 @@ PLAN-ROOT SINK table: rows=7300 size=478.45KB partitions: 24/24 rows=7300 columns: all - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=310 mem-estimate=128.00MB mem-reservation=32.00KB tuple-ids=0 row-size=4B cardinality=7300 ==== @@ -218,7 +218,7 @@ PLAN-ROOT SINK table: rows=7300 size=478.45KB partitions: 24/24 rows=7300 columns: all - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=2449 mem-estimate=32.00MB mem-reservation=32.00KB tuple-ids=0 row-size=4B cardinality=730 ==== http://git-wip-us.apache.org/repos/asf/impala/blob/83a70a7a/testdata/workloads/functional-planner/queries/PlannerTest/union.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/union.test b/testdata/workloads/functional-planner/queries/PlannerTest/union.test index 09ac11b..62bf2dd 100644 --- a/testdata/workloads/functional-planner/queries/PlannerTest/union.test +++ b/testdata/workloads/functional-planner/queries/PlannerTest/union.test @@ -3213,7 +3213,7 @@ PLAN-ROOT SINK | 04:UNNEST [c.c_orders] | 00:SCAN HDFS [tpch_nested_parquet.customer c] - partitions=1/1 files=4 size=292.36MB + partitions=1/1 files=4 size=288.98MB ==== # IMPALA-6388: Verify that the order of the union operands does not impact the # number of hosts computation @@ -3245,7 +3245,7 @@ Per-Host Resources: mem-estimate=128.00MB mem-reservation=32.00KB | table: rows=7300 size=478.45KB | partitions: 24/24 rows=7300 | columns: all -| extrapolated-rows=disabled +| extrapolated-rows=disabled max-scan-range-rows=310 | mem-estimate=128.00MB mem-reservation=32.00KB | tuple-ids=1 row-size=4B cardinality=7300 | @@ -3255,7 +3255,7 @@ Per-Host Resources: mem-estimate=128.00MB mem-reservation=32.00KB table: rows=unavailable size=unavailable partitions: 0/0 rows=unavailable columns: all - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=0 mem-estimate=0B mem-reservation=0B tuple-ids=0 row-size=4B cardinality=0 ==== @@ -3289,7 +3289,7 @@ Per-Host Resources: mem-estimate=128.00MB mem-reservation=32.00KB | table: rows=unavailable size=unavailable | partitions: 0/0 rows=unavailable | columns: all -| extrapolated-rows=disabled +| extrapolated-rows=disabled max-scan-range-rows=0 | mem-estimate=0B mem-reservation=0B | tuple-ids=1 row-size=4B cardinality=0 | @@ -3299,7 +3299,7 @@ Per-Host Resources: mem-estimate=128.00MB mem-reservation=32.00KB table: rows=7300 size=478.45KB partitions: 24/24 rows=7300 columns: all - extrapolated-rows=disabled + extrapolated-rows=disabled max-scan-range-rows=310 mem-estimate=128.00MB mem-reservation=32.00KB tuple-ids=0 row-size=4B cardinality=7300 ==== http://git-wip-us.apache.org/repos/asf/impala/blob/83a70a7a/testdata/workloads/functional-query/queries/QueryTest/admission-reject-mem-estimate.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/admission-reject-mem-estimate.test b/testdata/workloads/functional-query/queries/QueryTest/admission-reject-mem-estimate.test index 8868c9f..530b623 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/admission-reject-mem-estimate.test +++ b/testdata/workloads/functional-query/queries/QueryTest/admission-reject-mem-estimate.test @@ -2,52 +2,66 @@ ---- QUERY # Small query where estimate exceeds available memory but the query can execute with a # low mem_limit. -select min(n_name) from tpch_parquet.nation +select min(c_name), min(n_name) +from tpch_parquet.customer + join tpch_parquet.nation on c_nationkey = n_nationkey ---- CATCH -Rejected query from pool default-pool: request memory needed 26.00 MB is greater than pool max mem resources 10.00 MB +Rejected query from pool default-pool: request memory needed 82.94 MB is greater than pool max mem resources 40.00 MB ==== ---- QUERY # Overriding the memory estimate should allow the query to execute. -set max_mem_estimate_for_admission=10m; -select min(n_name) from tpch_parquet.nation +set max_mem_estimate_for_admission=30m; +select min(c_name), min(n_name) +from tpch_parquet.customer + join tpch_parquet.nation on c_nationkey = n_nationkey ---- RESULTS -'ALGERIA' +'Customer#000000001','ALGERIA' +---- TYPES +STRING,STRING ---- RUNTIME_PROFILE -row_regex: .*Query Options \(set by configuration\).*MAX_MEM_ESTIMATE_FOR_ADMISSION=10485760.* +row_regex: .*Query Options \(set by configuration\).*MAX_MEM_ESTIMATE_FOR_ADMISSION=31457280.* # Memory estimate sent to backend is overridden, but the explain plan shows the planner output. -row_regex: .*Estimated Per-Host Mem: 10485760.* -row_regex: .*Per-Host Resource Estimates: Memory=26.00MB.* +row_regex: .*Estimated Per-Host Mem: 31457280.* +row_regex: .*Per-Host Resource Estimates: Memory=82.94MB.* ==== ---- QUERY # If the estimate is set to a higher value that is still > the max mem resources for # the pool, the query still can't execute. -set max_mem_estimate_for_admission=20m; -select min(n_name) from tpch_parquet.nation +set max_mem_estimate_for_admission=45m; +select min(c_name), min(n_name) +from tpch_parquet.customer + join tpch_parquet.nation on c_nationkey = n_nationkey ---- CATCH -Rejected query from pool default-pool: request memory needed 20.00 MB is greater than pool max mem resources 10.00 MB +Rejected query from pool default-pool: request memory needed 45.00 MB is greater than pool max mem resources 40.00 MB ==== ---- QUERY # MEM_LIMIT takes precedence over MAX_MEM_ESTIMATE_FOR_ADMISSION, so the query is # rejected if MEM_LIMIT is higher. -set mem_limit=20m; -set max_mem_estimate_for_admission=10m; -select min(n_name) from tpch_parquet.nation +set mem_limit=45m; +set max_mem_estimate_for_admission=30m; +select min(c_name), min(n_name) +from tpch_parquet.customer + join tpch_parquet.nation on c_nationkey = n_nationkey ---- CATCH -Rejected query from pool default-pool: request memory needed 20.00 MB is greater than pool max mem resources 10.00 MB +Rejected query from pool default-pool: request memory needed 45.00 MB is greater than pool max mem resources 40.00 MB ==== ---- QUERY # MEM_LIMIT takes precedence over MAX_MEM_ESTIMATE_FOR_ADMISSION, so the query is # admitted if MEM_LIMIT is lower. -set mem_limit=10m; -set max_mem_estimate_for_admission=20m; -select min(n_name) from tpch_parquet.nation +set mem_limit=40m; +set max_mem_estimate_for_admission=45m; +select min(c_name), min(n_name) +from tpch_parquet.customer + join tpch_parquet.nation on c_nationkey = n_nationkey ---- RESULTS -'ALGERIA' +'Customer#000000001','ALGERIA' +---- TYPES +STRING,STRING ---- RUNTIME_PROFILE -row_regex: .*Query Options \(set by configuration\).*MEM_LIMIT=10485760.*MAX_MEM_ESTIMATE_FOR_ADMISSION=20971520.* +row_regex: .*Query Options \(set by configuration\).*MEM_LIMIT=41943040.*MAX_MEM_ESTIMATE_FOR_ADMISSION=47185920.* # Memory estimate sent to backend is overridden, but the explain plan shows the planner output. -row_regex: .*Estimated Per-Host Mem: 20971520.* -row_regex: .*Per-Host Resource Estimates: Memory=26.00MB.* +row_regex: .*Estimated Per-Host Mem: 47185920.* +row_regex: .*Per-Host Resource Estimates: Memory=82.94MB.* ==== ---- QUERY # Larger queries that use more memory than the estimate can still run because no mem_limit is set. @@ -55,6 +69,8 @@ set max_mem_estimate_for_admission=10m; select min(l_comment) from tpch_parquet.lineitem ---- RESULTS ' Tiresias ' +---- TYPES +STRING ---- RUNTIME_PROFILE row_regex: .*Query Options \(set by configuration\).*MAX_MEM_ESTIMATE_FOR_ADMISSION=10485760.* # Memory estimate sent to backend is overridden, but the explain plan shows the planner output. @@ -64,14 +80,15 @@ row_regex: .*Per-Host Resource Estimates: Memory=90.00MB.* ---- QUERY # If the memory estimate is less than max_mem_estimate_for_admission, then the estimate # is used for admission. This query's memory estimate is 10MB, and it would be rejected -# if the memory estimate was > 10MB. -set max_mem_estimate_for_admission=20m; +# if the memory estimate was > 40MB. +set max_mem_estimate_for_admission=50m; select 'foo' ---- RESULTS 'foo' +---- TYPES +STRING ---- RUNTIME_PROFILE -row_regex: .*Query Options \(set by configuration\).*MAX_MEM_ESTIMATE_FOR_ADMISSION=20971520.* -# Memory estimate sent to backend is overridden, but the explain plan shows the planner output. +row_regex: .*Query Options \(set by configuration\).*MAX_MEM_ESTIMATE_FOR_ADMISSION=52428800.* row_regex: .*Estimated Per-Host Mem: 10485760.* row_regex: .*Per-Host Resource Estimates: Memory=10.00MB.* ==== http://git-wip-us.apache.org/repos/asf/impala/blob/83a70a7a/testdata/workloads/functional-query/queries/QueryTest/admission-reject-min-reservation.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/admission-reject-min-reservation.test b/testdata/workloads/functional-query/queries/QueryTest/admission-reject-min-reservation.test index de12db1..2ddf999 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/admission-reject-min-reservation.test +++ b/testdata/workloads/functional-query/queries/QueryTest/admission-reject-min-reservation.test @@ -12,7 +12,7 @@ set mem_limit=150mb; select distinct * from functional_parquet.alltypesagg ---- CATCH minimum memory reservation needed is greater than pool max mem resources. - Pool max mem resources: 10.00 MB. + Pool max mem resources: 40.00 MB. Cluster-wide memory reservation needed: 204.26 MB ==== ---- QUERY @@ -29,7 +29,7 @@ from tpch_parquet.lineitem join tpch_parquet.orders on l_orderkey = o_orderkey ---- CATCH minimum memory reservation is greater than memory available to the query for buffer reservations. Memory reservation needed given the - current plan: 22.75 MB. Set mem_limit to at least 54.75 MB. + current plan: 14.75 MB. Set mem_limit to at least 46.75 MB. ==== ---- QUERY set mem_limit=50mb; @@ -38,5 +38,5 @@ from tpch_parquet.lineitem join tpch_parquet.orders on l_orderkey = o_orderkey ---- CATCH minimum memory reservation is greater than memory available to the query for buffer reservations. Memory reservation needed given the - current plan: 34.00 MB. Set mem_limit to at least 66.00 MB. + current plan: 26.00 MB. Set mem_limit to at least 58.00 MB. ==== http://git-wip-us.apache.org/repos/asf/impala/blob/83a70a7a/testdata/workloads/functional-query/queries/QueryTest/explain-level2.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/explain-level2.test b/testdata/workloads/functional-query/queries/QueryTest/explain-level2.test index 17fdd10..10ddf0a 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/explain-level2.test +++ b/testdata/workloads/functional-query/queries/QueryTest/explain-level2.test @@ -37,7 +37,7 @@ row_regex:.*partitions=1/1 files=1 size=.* '| stored statistics:' row_regex:.*table: rows=1500000 size=.* '| columns: all' -'| extrapolated-rows=disabled' +row_regex:.*| extrapolated-rows=disabled max-scan-range-rows=[0-9]*.* row_regex:.*| mem-estimate=[0-9.]*MB mem-reservation=[0-9.]*MB.* '| tuple-ids=1 row-size=191B cardinality=1500000' '|' @@ -47,7 +47,7 @@ row_regex:.*partitions=1/1 files=1 size=.* ' stored statistics:' row_regex:.*table: rows=6001215 size=.* ' columns: all' -' extrapolated-rows=disabled' +row_regex:.* extrapolated-rows=disabled max-scan-range-rows=[0-9]*.* row_regex:.* mem-estimate=[0-9.]*MB mem-reservation=[0-9.]*MB.* ' tuple-ids=0 row-size=263B cardinality=6001215' ==== http://git-wip-us.apache.org/repos/asf/impala/blob/83a70a7a/testdata/workloads/functional-query/queries/QueryTest/explain-level3.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/explain-level3.test b/testdata/workloads/functional-query/queries/QueryTest/explain-level3.test index 4ef3524..3b2058e 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/explain-level3.test +++ b/testdata/workloads/functional-query/queries/QueryTest/explain-level3.test @@ -38,7 +38,7 @@ row_regex:.*partitions=1/1 files=1 size=.* ' stored statistics:' row_regex:.*table: rows=6001215 size=.* ' columns: all' -' extrapolated-rows=disabled' +row_regex:.*| extrapolated-rows=disabled max-scan-range-rows=[0-9]*.* row_regex:.* mem-estimate=[0-9.]*MB mem-reservation=[0-9.]*MB.* ' tuple-ids=0 row-size=263B cardinality=6001215' '' @@ -51,7 +51,7 @@ row_regex:.*partitions=1/1 files=1 size=.* ' stored statistics:' row_regex:.*table: rows=1500000 size=.* ' columns: all' -' extrapolated-rows=disabled' +row_regex:.* extrapolated-rows=disabled max-scan-range-rows=[0-9]*.* row_regex:.* mem-estimate=[0-9.]*MB mem-reservation=[0-9.]*MB.* ' tuple-ids=1 row-size=191B cardinality=1500000' ==== http://git-wip-us.apache.org/repos/asf/impala/blob/83a70a7a/testdata/workloads/functional-query/queries/QueryTest/nested-types-tpch.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/nested-types-tpch.test b/testdata/workloads/functional-query/queries/QueryTest/nested-types-tpch.test index a4c6c62..9dae6e6 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/nested-types-tpch.test +++ b/testdata/workloads/functional-query/queries/QueryTest/nested-types-tpch.test @@ -163,8 +163,8 @@ BIGINT,BIGINT # IMPALA-2376: run scan that constructs large collection and set memory limit low enough # to get the below query to consistently fail when allocating a large collection. Set # num_nodes and mt_dop to 1 in order to make the query as deterministic as possible. -set buffer_pool_limit=40m; -set mem_limit=50m; +set buffer_pool_limit=24m; +set mem_limit=35m; set num_nodes=1; set mt_dop=1; select max(cnt1), max(cnt2), max(cnt3), max(cnt4), max(cnt5) http://git-wip-us.apache.org/repos/asf/impala/blob/83a70a7a/testdata/workloads/functional-query/queries/QueryTest/scanner-reservation.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/scanner-reservation.test b/testdata/workloads/functional-query/queries/QueryTest/scanner-reservation.test new file mode 100644 index 0000000..0ca55ff --- /dev/null +++ b/testdata/workloads/functional-query/queries/QueryTest/scanner-reservation.test @@ -0,0 +1,55 @@ +==== +---- QUERY +# Scan moderately large file - scanner should try to increase reservation and succeed. +select count(*) +from tpch.customer +---- TYPES +BIGINT +---- RESULTS +150000 +---- RUNTIME_PROFILE +row_regex:.*InitialRangeIdealReservation.*Avg: 24.00 MB.*Number of samples: 1.* +row_regex:.*InitialRangeActualReservation.*Avg: 24.00 MB.*Number of samples: 1.* +==== +---- QUERY +# Scan moderately large file - scanner should try to increase reservation and fail. +set debug_action="-1:OPEN:[email protected]"; +select count(*) +from tpch.customer +---- TYPES +BIGINT +---- RESULTS +150000 +---- RUNTIME_PROFILE +row_regex:.*InitialRangeIdealReservation.*Avg: 24.00 MB.*Number of samples: 1.* +row_regex:.*InitialRangeActualReservation.*Avg: 8.00 MB.*Number of samples: 1.* +==== +---- QUERY +# Scan large Parquet column - scanner should try to increase reservation and succeed. +select min(l_comment) +from tpch_parquet.lineitem +---- TYPES +STRING +---- RESULTS +' Tiresias ' +---- RUNTIME_PROFILE +row_regex:.*InitialRangeIdealReservation.*Avg: 128.00 KB.* +row_regex:.*InitialRangeActualReservation.*Avg: 4.00 MB.* +row_regex:.*ParquetRowGroupIdealReservation.*Avg: 24.00 MB.* +row_regex:.*ParquetRowGroupActualReservation.*Avg: 24.00 MB.* +==== +---- QUERY +# Scan moderately large file - scanner should try to increase reservation and fail. +set debug_action="-1:OPEN:[email protected]"; +select min(l_comment) +from tpch_parquet.lineitem +---- TYPES +STRING +---- RESULTS +' Tiresias ' +---- RUNTIME_PROFILE +row_regex:.*InitialRangeIdealReservation.*Avg: 128.00 KB.* +row_regex:.*InitialRangeActualReservation.*Avg: 4.00 MB.* +row_regex:.*ParquetRowGroupIdealReservation.*Avg: 24.00 MB.* +row_regex:.*ParquetRowGroupActualReservation.*Avg: 4.00 MB.* +==== http://git-wip-us.apache.org/repos/asf/impala/blob/83a70a7a/testdata/workloads/functional-query/queries/QueryTest/single-node-nlj-exhaustive.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/single-node-nlj-exhaustive.test b/testdata/workloads/functional-query/queries/QueryTest/single-node-nlj-exhaustive.test index e194465..637e898 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/single-node-nlj-exhaustive.test +++ b/testdata/workloads/functional-query/queries/QueryTest/single-node-nlj-exhaustive.test @@ -18,9 +18,8 @@ tinyint,bigint,bigint,bigint,int,decimal,decimal,decimal,decimal,string,string,s ---- QUERY # IMPALA-6595: same query as above, but tuned to trigger a different bug where we hit # "Memory Limit Exceeded" when adding a batch to NljBuilder. -# TODO: after IMPALA-4835 goes in, retune this query. -set buffer_pool_limit=50m; -set mem_limit=70m; +set buffer_pool_limit=20m; +set mem_limit=30m; select straight_join * from (values(1 id), (2), (3)) v1, (select *, count(*) over() from tpch.lineitem where l_orderkey < 100000) v2 order by id, l_orderkey, l_partkey, l_suppkey, l_linenumber http://git-wip-us.apache.org/repos/asf/impala/blob/83a70a7a/testdata/workloads/functional-query/queries/QueryTest/spilling-aggs.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/spilling-aggs.test b/testdata/workloads/functional-query/queries/QueryTest/spilling-aggs.test index 20813c7..20a03a1 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/spilling-aggs.test +++ b/testdata/workloads/functional-query/queries/QueryTest/spilling-aggs.test @@ -122,7 +122,7 @@ row_regex: .*RowsPassedThrough: .* \([1-9][0-9]*\) ==== ---- QUERY # Test aggregation spill with group_concat distinct -set buffer_pool_limit=78m; +set buffer_pool_limit=50m; select l_orderkey, count(*), group_concat(distinct l_linestatus, '|') from lineitem group by 1 @@ -166,9 +166,9 @@ row_regex: .*SpilledPartitions: .* \([1-9][0-9]*\) ---- QUERY # Test aggregation with minimum required reservation to exercise IMPALA-2708. # Merge aggregation requires 17 buffers and preaggregation requires 16 buffers -# plus 1mb of hash tables. The buffer size is 256k for this test. +# plus 1mb of hash tables. The buffer size is 256k for this test. The scan needs 24MB. set max_row_size=256k; -set buffer_pool_limit="57.25M"; +set buffer_pool_limit="33.25M"; select count(*) from (select distinct * from orders) t ---- TYPES @@ -182,7 +182,7 @@ row_regex: .*SpilledPartitions: .* \([1-9][0-9]*\) # IMPALA-3304: test that avg() can spill with a query mem limit. # This test only covers that use FIXED_UDA_INTERMEDIATE, not functions that allocate # strings for intermediate values. -set mem_limit=150m; +set mem_limit=100m; select l_orderkey, avg(l_tax), avg(l_quantity), avg(l_discount), avg(l_extendedprice) from tpch_parquet.lineitem group by 1 http://git-wip-us.apache.org/repos/asf/impala/blob/83a70a7a/testdata/workloads/functional-query/queries/QueryTest/spilling.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/spilling.test b/testdata/workloads/functional-query/queries/QueryTest/spilling.test index 23d9271..13f863b 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/spilling.test +++ b/testdata/workloads/functional-query/queries/QueryTest/spilling.test @@ -56,11 +56,11 @@ INT, BIGINT # This query is not meant to spill. row_regex: .*SpilledPartitions: 0 .* ==== ----- QUERY: TPCH-Q22 +---- QUERY: TPCH-Q21 # Adding TPCH-Q21 in the spilling test to check for IMPALA-1471 (spilling left anti # and left outer joins were returning wrong results). # Q21 - Suppliers Who Kept Orders Waiting Query -set buffer_pool_limit=200m; +set buffer_pool_limit=100m; select s_name, count(*) as numwait http://git-wip-us.apache.org/repos/asf/impala/blob/83a70a7a/tests/custom_cluster/test_admission_controller.py ---------------------------------------------------------------------- diff --git a/tests/custom_cluster/test_admission_controller.py b/tests/custom_cluster/test_admission_controller.py index e633d44..7f53df8 100644 --- a/tests/custom_cluster/test_admission_controller.py +++ b/tests/custom_cluster/test_admission_controller.py @@ -390,7 +390,7 @@ class TestAdmissionController(TestAdmissionControllerBase, HS2TestSuite): @pytest.mark.execute_serially @CustomClusterTestSuite.with_args( impalad_args=impalad_admission_ctrl_flags(max_requests=1, max_queued=1, - pool_max_mem=10 * 1024 * 1024, proc_mem_limit=1024 * 1024 * 1024), + pool_max_mem=40 * 1024 * 1024, proc_mem_limit=1024 * 1024 * 1024), statestored_args=_STATESTORED_ARGS) def test_memory_rejection(self, vector): """Test that rejection of queries based on reservation and estimates works as http://git-wip-us.apache.org/repos/asf/impala/blob/83a70a7a/tests/query_test/test_mem_usage_scaling.py ---------------------------------------------------------------------- diff --git a/tests/query_test/test_mem_usage_scaling.py b/tests/query_test/test_mem_usage_scaling.py index 2d6c17b..749f77f 100644 --- a/tests/query_test/test_mem_usage_scaling.py +++ b/tests/query_test/test_mem_usage_scaling.py @@ -123,16 +123,17 @@ class TestLowMemoryLimits(ImpalaTestSuite): @SkipIfLocal.mem_usage_different class TestTpchMemLimitError(TestLowMemoryLimits): # The mem limits that will be used. - MEM_IN_MB = [20, 140, 180, 220, 275, 450, 700] + MEM_IN_MB = [20, 50, 80, 130, 160, 200, 400] # Different values of mem limits and minimum mem limit (in MBs) each query is expected # to run without problem. These were determined using the query_runtime_info.json file - # produced by the stress test (i.e. concurrent_select.py). - MIN_MEM_FOR_TPCH = { 'Q1' : 125, 'Q2' : 125, 'Q3' : 112, 'Q4' : 137, 'Q5' : 137,\ - 'Q6' : 25, 'Q7' : 200, 'Q8' : 125, 'Q9' : 200, 'Q10' : 162,\ - 'Q11' : 112, 'Q12' : 150, 'Q13' : 125, 'Q14' : 125, 'Q15' : 125,\ - 'Q16' : 137, 'Q17' : 137, 'Q18' : 196, 'Q19' : 112, 'Q20' : 162,\ - 'Q21' : 300, 'Q22' : 125} + # produced by the stress test (i.e. concurrent_select.py) and extracted with + # tests/stress/extract_min_mem.py + MIN_MEM_FOR_TPCH = {'Q1': 55, 'Q2': 89, 'Q3': 80, 'Q4': 70, 'Q5': 99, + 'Q6': 48, 'Q7': 127, 'Q8': 111, 'Q9': 189, 'Q10': 108, + 'Q11': 76, 'Q12': 70, 'Q13': 71, 'Q14': 57, 'Q15': 83, + 'Q16': 71, 'Q17': 73, 'Q18': 153, 'Q19': 54, 'Q20': 128, + 'Q21': 147, 'Q22': 57} @classmethod def get_workload(self): http://git-wip-us.apache.org/repos/asf/impala/blob/83a70a7a/tests/query_test/test_queries.py ---------------------------------------------------------------------- diff --git a/tests/query_test/test_queries.py b/tests/query_test/test_queries.py index 2d6914b..fafa3a1 100644 --- a/tests/query_test/test_queries.py +++ b/tests/query_test/test_queries.py @@ -169,9 +169,6 @@ class TestQueriesTextTables(ImpalaTestSuite): vector.get_value('exec_option')['num_nodes'] = 1 self.run_test_case('QueryTest/random', vector) - def test_mixed_format(self, vector): - self.run_test_case('QueryTest/mixed-format', vector) - def test_values(self, vector): self.run_test_case('QueryTest/values', vector) http://git-wip-us.apache.org/repos/asf/impala/blob/83a70a7a/tests/query_test/test_scanners.py ---------------------------------------------------------------------- diff --git a/tests/query_test/test_scanners.py b/tests/query_test/test_scanners.py index 5085f96..7936b7e 100644 --- a/tests/query_test/test_scanners.py +++ b/tests/query_test/test_scanners.py @@ -39,7 +39,8 @@ from tests.common.skip import ( SkipIfLocal) from tests.common.test_dimensions import ( create_single_exec_option_dimension, - create_exec_option_dimension) + create_exec_option_dimension, + create_uncompressed_text_dimension) from tests.common.test_result_verifier import ( parse_column_types, parse_column_labels, @@ -122,6 +123,30 @@ class TestScannersAllTableFormatsWithLimit(ImpalaTestSuite): # IMPALA-3337: The error log should be empty. assert not result.log +class TestScannersMixedTableFormats(ImpalaTestSuite): + BATCH_SIZES = [0, 1, 16] + + @classmethod + def get_workload(cls): + return 'functional-query' + + @classmethod + def add_test_dimensions(cls): + super(TestScannersMixedTableFormats, cls).add_test_dimensions() + # Only run with a single dimension format, since the table includes mixed formats. + cls.ImpalaTestMatrix.add_dimension( + create_uncompressed_text_dimension(cls.get_workload())) + cls.ImpalaTestMatrix.add_dimension( + ImpalaTestDimension('batch_size', *TestScannersAllTableFormats.BATCH_SIZES)) + cls.ImpalaTestMatrix.add_dimension( + ImpalaTestDimension('debug_action', *DEBUG_ACTION_DIMS)) + + def test_mixed_format(self, vector): + new_vector = deepcopy(vector) + new_vector.get_value('exec_option')['batch_size'] = vector.get_value('batch_size') + new_vector.get_value('exec_option')['debug_action'] = vector.get_value('debug_action') + self.run_test_case('QueryTest/mixed-format', new_vector) + # Test case to verify the scanners work properly when the table metadata (specifically the # number of columns in the table) does not match the number of columns in the data file. class TestUnmatchedSchema(ImpalaTestSuite): @@ -1017,3 +1042,20 @@ class TestOrc(ImpalaTestSuite): self.client.execute("alter table %s.mismatch_decimals recover partitions" % unique_database) self.run_test_case('DataErrorsTest/orc-type-checks', vector, unique_database) + +class TestScannerReservation(ImpalaTestSuite): + @classmethod + def get_workload(self): + return 'functional-query' + + @classmethod + def add_test_dimensions(cls): + super(TestScannerReservation, cls).add_test_dimensions() + # Only run with a single dimension - all queries are format-specific and + # reference tpch or tpch_parquet directly. + cls.ImpalaTestMatrix.add_dimension( + create_uncompressed_text_dimension(cls.get_workload())) + + def test_scanners(self, vector): + self.run_test_case('QueryTest/scanner-reservation', vector) + http://git-wip-us.apache.org/repos/asf/impala/blob/83a70a7a/tests/query_test/test_sort.py ---------------------------------------------------------------------- diff --git a/tests/query_test/test_sort.py b/tests/query_test/test_sort.py index 9024cfb..379b758 100644 --- a/tests/query_test/test_sort.py +++ b/tests/query_test/test_sort.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -from copy import copy +from copy import copy, deepcopy from tests.common.impala_test_suite import ImpalaTestSuite from tests.common.skip import SkipIfLocal @@ -171,7 +171,10 @@ class TestQueryFullSort(ImpalaTestSuite): @SkipIfLocal.mem_usage_different def test_sort_reservation_usage(self, vector): """Tests for sorter reservation usage.""" - self.run_test_case('sort-reservation-usage', vector) + new_vector = deepcopy(vector) + # Run with num_nodes=1 to make execution more deterministic. + new_vector.get_value('exec_option')['num_nodes'] = 1 + self.run_test_case('sort-reservation-usage', new_vector) class TestRandomSort(ImpalaTestSuite): @classmethod http://git-wip-us.apache.org/repos/asf/impala/blob/83a70a7a/tests/stress/extract_min_mem.py ---------------------------------------------------------------------- diff --git a/tests/stress/extract_min_mem.py b/tests/stress/extract_min_mem.py new file mode 100755 index 0000000..e84148b --- /dev/null +++ b/tests/stress/extract_min_mem.py @@ -0,0 +1,49 @@ +#!/usr/bin/env impala-python +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Used to extract minimum memory data for test_mem_usage_scaling.py from stress test +# runtime info json. +# +# Usage +# ===== +# Run the stress test binary search on the 3 node minicluster: +# +# ./tests/stress/concurrent_select.py --tpch-db=tpch_parquet \ +# --runtime-info-path=mem_usage_scaling_runtime_info.json --samples 3 \ +# --mem_limit_eq_threshold_percent=0.01 --mem_limit_eq_threshold_mb=5 \ +# --common-query-options="default_spillable_buffer_size=256k" +# +# Then run this script to extract minimum memory: +# +# ./tests/stress/extract_min_mem.py mem_usage_scaling_runtime_info.json +# +import json +import sys + +results = [] +with open(sys.argv[1]) as f: + data = json.load(f) + for query_data in data['db_names']['tpch_parquet'].itervalues(): + runtime_info = query_data['[]'] + # Build up list of query numbers and minimum memory. + results.append((int(runtime_info['name'][1:]), + runtime_info['required_mem_mb_with_spilling'])) + +results.sort() +print ', '.join(["'Q{0}': {1}".format(num, mem) for num, mem in results])
