http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9a29dfc9/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test b/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test new file mode 100644 index 0000000..7ae30fa --- /dev/null +++ b/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test @@ -0,0 +1,1121 @@ +# Parquet scan +select * from tpch_parquet.lineitem +---- DISTRIBUTEDPLAN +Per-Host Resource Reservation: Memory=0B +Per-Host Resource Estimates: Memory=80.00MB + +F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 +PLAN-ROOT SINK +| mem-estimate=0B mem-reservation=0B +| +01:EXCHANGE [UNPARTITIONED] +| mem-estimate=0B mem-reservation=0B +| tuple-ids=0 row-size=263B cardinality=6001215 +| +F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3 +00:SCAN HDFS [tpch_parquet.lineitem, RANDOM] + partitions=1/1 files=3 size=193.61MB + table stats: 6001215 rows total + column stats: all + mem-estimate=80.00MB mem-reservation=0B + tuple-ids=0 row-size=263B cardinality=6001215 +---- PARALLELPLANS +Per-Host Resource Reservation: Memory=0B +Per-Host Resource Estimates: Memory=160.00MB + +F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 +PLAN-ROOT SINK +| mem-estimate=0B mem-reservation=0B +| +01:EXCHANGE [UNPARTITIONED] +| mem-estimate=0B mem-reservation=0B +| tuple-ids=0 row-size=263B cardinality=6001215 +| +F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6 +00:SCAN HDFS [tpch_parquet.lineitem, RANDOM] + partitions=1/1 files=3 size=193.61MB + table stats: 6001215 rows total + column stats: all + mem-estimate=80.00MB mem-reservation=0B + tuple-ids=0 row-size=263B cardinality=6001215 +==== +# Text scan +select * from tpch.lineitem; +---- DISTRIBUTEDPLAN +Per-Host Resource Reservation: Memory=0B +Per-Host Resource Estimates: Memory=88.00MB + +F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 +PLAN-ROOT SINK +| mem-estimate=0B mem-reservation=0B +| +01:EXCHANGE [UNPARTITIONED] +| mem-estimate=0B mem-reservation=0B +| tuple-ids=0 row-size=263B cardinality=6001215 +| +F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3 +00:SCAN HDFS [tpch.lineitem, RANDOM] + partitions=1/1 files=1 size=718.94MB + table stats: 6001215 rows total + column stats: all + mem-estimate=88.00MB mem-reservation=0B + tuple-ids=0 row-size=263B cardinality=6001215 +---- PARALLELPLANS +Per-Host Resource Reservation: Memory=0B +Per-Host Resource Estimates: Memory=176.00MB + +F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 +PLAN-ROOT SINK +| mem-estimate=0B mem-reservation=0B +| +01:EXCHANGE [UNPARTITIONED] +| mem-estimate=0B mem-reservation=0B +| tuple-ids=0 row-size=263B cardinality=6001215 +| +F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6 +00:SCAN HDFS [tpch.lineitem, RANDOM] + partitions=1/1 files=1 size=718.94MB + table stats: 6001215 rows total + column stats: all + mem-estimate=88.00MB mem-reservation=0B + tuple-ids=0 row-size=263B cardinality=6001215 +==== +# HBase scan +select * from functional_hbase.alltypes +---- DISTRIBUTEDPLAN +Per-Host Resource Reservation: Memory=0B +Per-Host Resource Estimates: Memory=1.00GB +WARNING: The following tables are missing relevant table and/or column statistics. +functional_hbase.alltypes + +F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 +PLAN-ROOT SINK +| mem-estimate=0B mem-reservation=0B +| +01:EXCHANGE [UNPARTITIONED] +| mem-estimate=0B mem-reservation=0B +| tuple-ids=0 row-size=88B cardinality=14298 +| +F00:PLAN FRAGMENT [RANDOM] hosts=100 instances=100 +00:SCAN HBASE [functional_hbase.alltypes] + table stats: unavailable + column stats: unavailable + mem-estimate=1.00GB mem-reservation=0B + tuple-ids=0 row-size=88B cardinality=14298 +---- PARALLELPLANS +Per-Host Resource Reservation: Memory=0B +Per-Host Resource Estimates: Memory=2.00GB +WARNING: The following tables are missing relevant table and/or column statistics. +functional_hbase.alltypes + +F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 +PLAN-ROOT SINK +| mem-estimate=0B mem-reservation=0B +| +01:EXCHANGE [UNPARTITIONED] +| mem-estimate=0B mem-reservation=0B +| tuple-ids=0 row-size=88B cardinality=14298 +| +F00:PLAN FRAGMENT [RANDOM] hosts=100 instances=200 +00:SCAN HBASE [functional_hbase.alltypes] + table stats: unavailable + column stats: unavailable + mem-estimate=1.00GB mem-reservation=0B + tuple-ids=0 row-size=88B cardinality=14298 +==== +# Data source scan +select * from functional.alltypes_datasource +---- DISTRIBUTEDPLAN +Per-Host Resource Reservation: Memory=0B +Per-Host Resource Estimates: Memory=1.00GB +WARNING: The following tables are missing relevant table and/or column statistics. +functional.alltypes_datasource + +F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 +PLAN-ROOT SINK +| mem-estimate=0B mem-reservation=0B +| +01:EXCHANGE [UNPARTITIONED] +| mem-estimate=0B mem-reservation=0B +| tuple-ids=0 row-size=116B cardinality=5000 +| +F00:PLAN FRAGMENT [RANDOM] hosts=1 instances=1 +00:SCAN DATA SOURCE [functional.alltypes_datasource] + mem-estimate=1.00GB mem-reservation=0B + tuple-ids=0 row-size=116B cardinality=5000 +---- PARALLELPLANS +Per-Host Resource Reservation: Memory=0B +Per-Host Resource Estimates: Memory=2.00GB +WARNING: The following tables are missing relevant table and/or column statistics. +functional.alltypes_datasource + +F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 +PLAN-ROOT SINK +| mem-estimate=0B mem-reservation=0B +| +01:EXCHANGE [UNPARTITIONED] +| mem-estimate=0B mem-reservation=0B +| tuple-ids=0 row-size=116B cardinality=5000 +| +F00:PLAN FRAGMENT [RANDOM] hosts=1 instances=2 +00:SCAN DATA SOURCE [functional.alltypes_datasource] + mem-estimate=1.00GB mem-reservation=0B + tuple-ids=0 row-size=116B cardinality=5000 +==== +# Union +select * from tpch.lineitem +union all +select * from tpch.lineitem +---- DISTRIBUTEDPLAN +Per-Host Resource Reservation: Memory=0B +Per-Host Resource Estimates: Memory=176.00MB + +F03:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 +PLAN-ROOT SINK +| mem-estimate=0B mem-reservation=0B +| +03:EXCHANGE [UNPARTITIONED] +| mem-estimate=0B mem-reservation=0B +| tuple-ids=2 row-size=263B cardinality=12002430 +| +F02:PLAN FRAGMENT [RANDOM] hosts=3 instances=3 +00:UNION +| pass-through-operands: all +| mem-estimate=0B mem-reservation=0B +| tuple-ids=2 row-size=263B cardinality=12002430 +| +|--02:SCAN HDFS [tpch.lineitem, RANDOM] +| partitions=1/1 files=1 size=718.94MB +| table stats: 6001215 rows total +| column stats: all +| mem-estimate=88.00MB mem-reservation=0B +| tuple-ids=1 row-size=263B cardinality=6001215 +| +01:SCAN HDFS [tpch.lineitem, RANDOM] + partitions=1/1 files=1 size=718.94MB + table stats: 6001215 rows total + column stats: all + mem-estimate=88.00MB mem-reservation=0B + tuple-ids=0 row-size=263B cardinality=6001215 +---- PARALLELPLANS +Per-Host Resource Reservation: Memory=0B +Per-Host Resource Estimates: Memory=352.00MB + +F03:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 +PLAN-ROOT SINK +| mem-estimate=0B mem-reservation=0B +| +03:EXCHANGE [UNPARTITIONED] +| mem-estimate=0B mem-reservation=0B +| tuple-ids=2 row-size=263B cardinality=12002430 +| +F02:PLAN FRAGMENT [RANDOM] hosts=3 instances=6 +00:UNION +| pass-through-operands: all +| mem-estimate=0B mem-reservation=0B +| tuple-ids=2 row-size=263B cardinality=12002430 +| +|--02:SCAN HDFS [tpch.lineitem, RANDOM] +| partitions=1/1 files=1 size=718.94MB +| table stats: 6001215 rows total +| column stats: all +| mem-estimate=88.00MB mem-reservation=0B +| tuple-ids=1 row-size=263B cardinality=6001215 +| +01:SCAN HDFS [tpch.lineitem, RANDOM] + partitions=1/1 files=1 size=718.94MB + table stats: 6001215 rows total + column stats: all + mem-estimate=88.00MB mem-reservation=0B + tuple-ids=0 row-size=263B cardinality=6001215 +==== +# Grouping aggregation +select l_orderkey, count(*) +from tpch_parquet.lineitem +group by l_orderkey +---- DISTRIBUTEDPLAN +Per-Host Resource Reservation: Memory=264.00MB +Per-Host Resource Estimates: Memory=116.24MB + +F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 +PLAN-ROOT SINK +| mem-estimate=0B mem-reservation=0B +| +04:EXCHANGE [UNPARTITIONED] +| mem-estimate=0B mem-reservation=0B +| tuple-ids=1 row-size=16B cardinality=1563438 +| +F01:PLAN FRAGMENT [HASH(l_orderkey)] hosts=3 instances=3 +03:AGGREGATE [FINALIZE] +| output: count:merge(*) +| group by: l_orderkey +| mem-estimate=10.00MB mem-reservation=264.00MB +| tuple-ids=1 row-size=16B cardinality=1563438 +| +02:EXCHANGE [HASH(l_orderkey)] +| mem-estimate=0B mem-reservation=0B +| tuple-ids=1 row-size=16B cardinality=1563438 +| +F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3 +01:AGGREGATE [STREAMING] +| output: count(*) +| group by: l_orderkey +| mem-estimate=26.24MB mem-reservation=0B +| tuple-ids=1 row-size=16B cardinality=1563438 +| +00:SCAN HDFS [tpch_parquet.lineitem, RANDOM] + partitions=1/1 files=3 size=193.61MB + table stats: 6001215 rows total + column stats: all + mem-estimate=80.00MB mem-reservation=0B + tuple-ids=0 row-size=8B cardinality=6001215 +---- PARALLELPLANS +Per-Host Resource Reservation: Memory=528.00MB +Per-Host Resource Estimates: Memory=232.48MB + +F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 +PLAN-ROOT SINK +| mem-estimate=0B mem-reservation=0B +| +04:EXCHANGE [UNPARTITIONED] +| mem-estimate=0B mem-reservation=0B +| tuple-ids=1 row-size=16B cardinality=1563438 +| +F01:PLAN FRAGMENT [HASH(l_orderkey)] hosts=3 instances=6 +03:AGGREGATE [FINALIZE] +| output: count:merge(*) +| group by: l_orderkey +| mem-estimate=10.00MB mem-reservation=264.00MB +| tuple-ids=1 row-size=16B cardinality=1563438 +| +02:EXCHANGE [HASH(l_orderkey)] +| mem-estimate=0B mem-reservation=0B +| tuple-ids=1 row-size=16B cardinality=1563438 +| +F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6 +01:AGGREGATE [STREAMING] +| output: count(*) +| group by: l_orderkey +| mem-estimate=26.24MB mem-reservation=0B +| tuple-ids=1 row-size=16B cardinality=1563438 +| +00:SCAN HDFS [tpch_parquet.lineitem, RANDOM] + partitions=1/1 files=3 size=193.61MB + table stats: 6001215 rows total + column stats: all + mem-estimate=80.00MB mem-reservation=0B + tuple-ids=0 row-size=8B cardinality=6001215 +==== +# Non-grouping aggregation with zero-slot parquet scan +select count(*) from tpch_parquet.lineitem +---- DISTRIBUTEDPLAN +Per-Host Resource Reservation: Memory=0B +Per-Host Resource Estimates: Memory=20.00MB + +F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 +PLAN-ROOT SINK +| mem-estimate=0B mem-reservation=0B +| +03:AGGREGATE [FINALIZE] +| output: count:merge(*) +| mem-estimate=10.00MB mem-reservation=0B +| tuple-ids=1 row-size=8B cardinality=1 +| +02:EXCHANGE [UNPARTITIONED] +| mem-estimate=0B mem-reservation=0B +| tuple-ids=1 row-size=8B cardinality=1 +| +F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3 +01:AGGREGATE +| output: count(*) +| mem-estimate=10.00MB mem-reservation=0B +| tuple-ids=1 row-size=8B cardinality=1 +| +00:SCAN HDFS [tpch_parquet.lineitem, RANDOM] + partitions=1/1 files=3 size=193.61MB + table stats: 6001215 rows total + column stats: all + mem-estimate=0B mem-reservation=0B + tuple-ids=0 row-size=0B cardinality=6001215 +---- PARALLELPLANS +Per-Host Resource Reservation: Memory=0B +Per-Host Resource Estimates: Memory=180.00MB + +F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 +PLAN-ROOT SINK +| mem-estimate=0B mem-reservation=0B +| +03:AGGREGATE [FINALIZE] +| output: count:merge(*) +| mem-estimate=10.00MB mem-reservation=0B +| tuple-ids=1 row-size=8B cardinality=1 +| +02:EXCHANGE [UNPARTITIONED] +| mem-estimate=0B mem-reservation=0B +| tuple-ids=1 row-size=8B cardinality=1 +| +F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6 +01:AGGREGATE +| output: count(*) +| mem-estimate=10.00MB mem-reservation=0B +| tuple-ids=1 row-size=8B cardinality=1 +| +00:SCAN HDFS [tpch_parquet.lineitem, RANDOM] + partitions=1/1 files=3 size=193.61MB + table stats: 6001215 rows total + column stats: all + mem-estimate=80.00MB mem-reservation=0B + tuple-ids=0 row-size=0B cardinality=6001215 +==== +# Sort +select * +from tpch_parquet.lineitem +order by l_comment +---- DISTRIBUTEDPLAN +Per-Host Resource Reservation: Memory=48.00MB +Per-Host Resource Estimates: Memory=240.00MB + +F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 +PLAN-ROOT SINK +| mem-estimate=0B mem-reservation=0B +| +02:MERGING-EXCHANGE [UNPARTITIONED] +| order by: l_comment ASC +| mem-estimate=0B mem-reservation=0B +| tuple-ids=1 row-size=263B cardinality=6001215 +| +F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3 +01:SORT +| order by: l_comment ASC +| mem-estimate=160.00MB mem-reservation=48.00MB +| tuple-ids=1 row-size=263B cardinality=6001215 +| +00:SCAN HDFS [tpch_parquet.lineitem, RANDOM] + partitions=1/1 files=3 size=193.61MB + table stats: 6001215 rows total + column stats: all + mem-estimate=80.00MB mem-reservation=0B + tuple-ids=0 row-size=263B cardinality=6001215 +---- PARALLELPLANS +Per-Host Resource Reservation: Memory=96.00MB +Per-Host Resource Estimates: Memory=480.00MB + +F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 +PLAN-ROOT SINK +| mem-estimate=0B mem-reservation=0B +| +02:MERGING-EXCHANGE [UNPARTITIONED] +| order by: l_comment ASC +| mem-estimate=0B mem-reservation=0B +| tuple-ids=1 row-size=263B cardinality=6001215 +| +F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6 +01:SORT +| order by: l_comment ASC +| mem-estimate=160.00MB mem-reservation=48.00MB +| tuple-ids=1 row-size=263B cardinality=6001215 +| +00:SCAN HDFS [tpch_parquet.lineitem, RANDOM] + partitions=1/1 files=3 size=193.61MB + table stats: 6001215 rows total + column stats: all + mem-estimate=80.00MB mem-reservation=0B + tuple-ids=0 row-size=263B cardinality=6001215 +==== +# TOP-N +select * +from tpch_parquet.lineitem +order by l_comment +limit 100 +---- DISTRIBUTEDPLAN +Per-Host Resource Reservation: Memory=0B +Per-Host Resource Estimates: Memory=80.03MB + +F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 +PLAN-ROOT SINK +| mem-estimate=0B mem-reservation=0B +| +02:MERGING-EXCHANGE [UNPARTITIONED] +| order by: l_comment ASC +| limit: 100 +| mem-estimate=0B mem-reservation=0B +| tuple-ids=1 row-size=263B cardinality=100 +| +F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3 +01:TOP-N [LIMIT=100] +| order by: l_comment ASC +| mem-estimate=25.66KB mem-reservation=0B +| tuple-ids=1 row-size=263B cardinality=100 +| +00:SCAN HDFS [tpch_parquet.lineitem, RANDOM] + partitions=1/1 files=3 size=193.61MB + table stats: 6001215 rows total + column stats: all + mem-estimate=80.00MB mem-reservation=0B + tuple-ids=0 row-size=263B cardinality=6001215 +---- PARALLELPLANS +Per-Host Resource Reservation: Memory=0B +Per-Host Resource Estimates: Memory=160.05MB + +F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 +PLAN-ROOT SINK +| mem-estimate=0B mem-reservation=0B +| +02:MERGING-EXCHANGE [UNPARTITIONED] +| order by: l_comment ASC +| limit: 100 +| mem-estimate=0B mem-reservation=0B +| tuple-ids=1 row-size=263B cardinality=100 +| +F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6 +01:TOP-N [LIMIT=100] +| order by: l_comment ASC +| mem-estimate=25.66KB mem-reservation=0B +| tuple-ids=1 row-size=263B cardinality=100 +| +00:SCAN HDFS [tpch_parquet.lineitem, RANDOM] + partitions=1/1 files=3 size=193.61MB + table stats: 6001215 rows total + column stats: all + mem-estimate=80.00MB mem-reservation=0B + tuple-ids=0 row-size=263B cardinality=6001215 +==== +# Hash Join +select * +from tpch.lineitem inner join tpch.orders on l_orderkey = o_orderkey +---- DISTRIBUTEDPLAN +Per-Host Resource Reservation: Memory=136.00MB +Per-Host Resource Estimates: Memory=388.41MB + +F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 +PLAN-ROOT SINK +| mem-estimate=0B mem-reservation=0B +| +04:EXCHANGE [UNPARTITIONED] +| mem-estimate=0B mem-reservation=0B +| tuple-ids=0,1 row-size=454B cardinality=5757710 +| +F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3 +02:HASH JOIN [INNER JOIN, BROADCAST] +| hash predicates: l_orderkey = o_orderkey +| runtime filters: RF000 <- o_orderkey +| mem-estimate=300.41MB mem-reservation=136.00MB +| tuple-ids=0,1 row-size=454B cardinality=5757710 +| +|--03:EXCHANGE [BROADCAST] +| | mem-estimate=0B mem-reservation=0B +| | tuple-ids=1 row-size=191B cardinality=1500000 +| | +| F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=2 +| 01:SCAN HDFS [tpch.orders, RANDOM] +| partitions=1/1 files=1 size=162.56MB +| table stats: 1500000 rows total +| column stats: all +| mem-estimate=88.00MB mem-reservation=0B +| tuple-ids=1 row-size=191B cardinality=1500000 +| +00:SCAN HDFS [tpch.lineitem, RANDOM] + partitions=1/1 files=1 size=718.94MB + runtime filters: RF000 -> l_orderkey + table stats: 6001215 rows total + column stats: all + mem-estimate=88.00MB mem-reservation=0B + tuple-ids=0 row-size=263B cardinality=6001215 +---- PARALLELPLANS +Per-Host Resource Reservation: Memory=272.00MB +Per-Host Resource Estimates: Memory=952.83MB + +F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 +PLAN-ROOT SINK +| mem-estimate=0B mem-reservation=0B +| +04:EXCHANGE [UNPARTITIONED] +| mem-estimate=0B mem-reservation=0B +| tuple-ids=0,1 row-size=454B cardinality=5757710 +| +F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6 +02:HASH JOIN [INNER JOIN, BROADCAST] +| hash-table-id=00 +| hash predicates: l_orderkey = o_orderkey +| runtime filters: RF000 <- o_orderkey +| mem-estimate=300.41MB mem-reservation=136.00MB +| tuple-ids=0,1 row-size=454B cardinality=5757710 +| +|--F03:PLAN FRAGMENT [RANDOM] hosts=2 instances=4 +| JOIN BUILD +| | join-table-id=00 plan-id=01 cohort-id=01 +| | build expressions: o_orderkey +| | mem-estimate=0B mem-reservation=0B +| | +| 03:EXCHANGE [BROADCAST] +| | mem-estimate=0B mem-reservation=0B +| | tuple-ids=1 row-size=191B cardinality=1500000 +| | +| F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=4 +| 01:SCAN HDFS [tpch.orders, RANDOM] +| partitions=1/1 files=1 size=162.56MB +| table stats: 1500000 rows total +| column stats: all +| mem-estimate=88.00MB mem-reservation=0B +| tuple-ids=1 row-size=191B cardinality=1500000 +| +00:SCAN HDFS [tpch.lineitem, RANDOM] + partitions=1/1 files=1 size=718.94MB + runtime filters: RF000 -> l_orderkey + table stats: 6001215 rows total + column stats: all + mem-estimate=88.00MB mem-reservation=0B + tuple-ids=0 row-size=263B cardinality=6001215 +==== +# Nested loop join +select * +from tpch.lineitem, tpch.orders +---- DISTRIBUTEDPLAN +Per-Host Resource Reservation: Memory=0B +Per-Host Resource Estimates: Memory=361.10MB + +F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 +PLAN-ROOT SINK +| mem-estimate=0B mem-reservation=0B +| +04:EXCHANGE [UNPARTITIONED] +| mem-estimate=0B mem-reservation=0B +| tuple-ids=0,1 row-size=454B cardinality=9001822500000 +| +F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3 +02:NESTED LOOP JOIN [CROSS JOIN, BROADCAST] +| mem-estimate=273.10MB mem-reservation=0B +| tuple-ids=0,1 row-size=454B cardinality=9001822500000 +| +|--03:EXCHANGE [BROADCAST] +| | mem-estimate=0B mem-reservation=0B +| | tuple-ids=1 row-size=191B cardinality=1500000 +| | +| F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=2 +| 01:SCAN HDFS [tpch.orders, RANDOM] +| partitions=1/1 files=1 size=162.56MB +| table stats: 1500000 rows total +| column stats: all +| mem-estimate=88.00MB mem-reservation=0B +| tuple-ids=1 row-size=191B cardinality=1500000 +| +00:SCAN HDFS [tpch.lineitem, RANDOM] + partitions=1/1 files=1 size=718.94MB + table stats: 6001215 rows total + column stats: all + mem-estimate=88.00MB mem-reservation=0B + tuple-ids=0 row-size=263B cardinality=6001215 +---- PARALLELPLANS +Per-Host Resource Reservation: Memory=0B +Per-Host Resource Estimates: Memory=898.21MB + +F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 +PLAN-ROOT SINK +| mem-estimate=0B mem-reservation=0B +| +04:EXCHANGE [UNPARTITIONED] +| mem-estimate=0B mem-reservation=0B +| tuple-ids=0,1 row-size=454B cardinality=9001822500000 +| +F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6 +02:NESTED LOOP JOIN [CROSS JOIN, BROADCAST] +| join table id: 00 +| mem-estimate=273.10MB mem-reservation=0B +| tuple-ids=0,1 row-size=454B cardinality=9001822500000 +| +|--F03:PLAN FRAGMENT [RANDOM] hosts=2 instances=4 +| JOIN BUILD +| | join-table-id=00 plan-id=01 cohort-id=01 +| | mem-estimate=0B mem-reservation=0B +| | +| 03:EXCHANGE [BROADCAST] +| | mem-estimate=0B mem-reservation=0B +| | tuple-ids=1 row-size=191B cardinality=1500000 +| | +| F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=4 +| 01:SCAN HDFS [tpch.orders, RANDOM] +| partitions=1/1 files=1 size=162.56MB +| table stats: 1500000 rows total +| column stats: all +| mem-estimate=88.00MB mem-reservation=0B +| tuple-ids=1 row-size=191B cardinality=1500000 +| +00:SCAN HDFS [tpch.lineitem, RANDOM] + partitions=1/1 files=1 size=718.94MB + table stats: 6001215 rows total + column stats: all + mem-estimate=88.00MB mem-reservation=0B + tuple-ids=0 row-size=263B cardinality=6001215 +==== +# Empty set node +select * from functional.alltypes where 1 = 2 +---- DISTRIBUTEDPLAN +Per-Host Resource Reservation: Memory=0B +Per-Host Resource Estimates: Memory=10.00MB + +F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 +PLAN-ROOT SINK +| mem-estimate=0B mem-reservation=0B +| +00:EMPTYSET + mem-estimate=0B mem-reservation=0B + tuple-ids=0 row-size=0B cardinality=0 +---- PARALLELPLANS +Per-Host Resource Reservation: Memory=0B +Per-Host Resource Estimates: Memory=10.00MB + +F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 +PLAN-ROOT SINK +| mem-estimate=0B mem-reservation=0B +| +00:EMPTYSET + mem-estimate=0B mem-reservation=0B + tuple-ids=0 row-size=0B cardinality=0 +==== +# Analytic function +select max(tinyint_col) over(partition by int_col) +from functional.alltypes +---- DISTRIBUTEDPLAN +Per-Host Resource Reservation: Memory=40.00MB +Per-Host Resource Estimates: Memory=24.00MB + +F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 +PLAN-ROOT SINK +| mem-estimate=0B mem-reservation=0B +| +04:EXCHANGE [UNPARTITIONED] +| mem-estimate=0B mem-reservation=0B +| tuple-ids=3,2 row-size=6B cardinality=7300 +| +F01:PLAN FRAGMENT [HASH(int_col)] hosts=3 instances=3 +02:ANALYTIC +| functions: max(tinyint_col) +| partition by: int_col +| mem-estimate=0B mem-reservation=16.00MB +| tuple-ids=3,2 row-size=6B cardinality=7300 +| +01:SORT +| order by: int_col ASC NULLS FIRST +| mem-estimate=8.00MB mem-reservation=24.00MB +| tuple-ids=3 row-size=5B cardinality=7300 +| +03:EXCHANGE [HASH(int_col)] +| mem-estimate=0B mem-reservation=0B +| tuple-ids=0 row-size=5B cardinality=7300 +| +F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3 +00:SCAN HDFS [functional.alltypes, RANDOM] + partitions=24/24 files=24 size=478.45KB + table stats: 7300 rows total + column stats: all + mem-estimate=16.00MB mem-reservation=0B + tuple-ids=0 row-size=5B cardinality=7300 +---- PARALLELPLANS +Per-Host Resource Reservation: Memory=80.00MB +Per-Host Resource Estimates: Memory=48.00MB + +F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 +PLAN-ROOT SINK +| mem-estimate=0B mem-reservation=0B +| +04:EXCHANGE [UNPARTITIONED] +| mem-estimate=0B mem-reservation=0B +| tuple-ids=3,2 row-size=6B cardinality=7300 +| +F01:PLAN FRAGMENT [HASH(int_col)] hosts=3 instances=6 +02:ANALYTIC +| functions: max(tinyint_col) +| partition by: int_col +| mem-estimate=0B mem-reservation=16.00MB +| tuple-ids=3,2 row-size=6B cardinality=7300 +| +01:SORT +| order by: int_col ASC NULLS FIRST +| mem-estimate=8.00MB mem-reservation=24.00MB +| tuple-ids=3 row-size=5B cardinality=7300 +| +03:EXCHANGE [HASH(int_col)] +| mem-estimate=0B mem-reservation=0B +| tuple-ids=0 row-size=5B cardinality=7300 +| +F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6 +00:SCAN HDFS [functional.alltypes, RANDOM] + partitions=24/24 files=24 size=478.45KB + table stats: 7300 rows total + column stats: all + mem-estimate=16.00MB mem-reservation=0B + tuple-ids=0 row-size=5B cardinality=7300 +==== +# TPC-H Q18: Pipeline of joins with some non-trivial subtrees on right side of joins + select + c_name, + c_custkey, + o_orderkey, + o_orderdate, + o_totalprice, + sum(l_quantity) +from + tpch.customer, + tpch.orders, + tpch.lineitem +where + o_orderkey in ( + select + l_orderkey + from + tpch.lineitem + group by + l_orderkey + having + sum(l_quantity) > 300 + ) + and c_custkey = o_custkey + and o_orderkey = l_orderkey +group by + c_name, + c_custkey, + o_orderkey, + o_orderdate, + o_totalprice +order by + o_totalprice desc, + o_orderdate +limit 100 +---- DISTRIBUTEDPLAN +Per-Host Resource Reservation: Memory=672.00MB +Per-Host Resource Estimates: Memory=242.95MB + +F07:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 +PLAN-ROOT SINK +| mem-estimate=0B mem-reservation=0B +| +17:MERGING-EXCHANGE [UNPARTITIONED] +| order by: o_totalprice DESC, o_orderdate ASC +| limit: 100 +| mem-estimate=0B mem-reservation=0B +| tuple-ids=7 row-size=100B cardinality=100 +| +F06:PLAN FRAGMENT [HASH(c_name,c_custkey,o_orderkey,o_orderdate,o_totalprice)] hosts=3 instances=3 +09:TOP-N [LIMIT=100] +| order by: o_totalprice DESC, o_orderdate ASC +| mem-estimate=9.77KB mem-reservation=0B +| tuple-ids=7 row-size=100B cardinality=100 +| +16:AGGREGATE [FINALIZE] +| output: sum:merge(l_quantity) +| group by: c_name, c_custkey, o_orderkey, o_orderdate, o_totalprice +| mem-estimate=60.40MB mem-reservation=264.00MB +| tuple-ids=6 row-size=100B cardinality=575772 +| +15:EXCHANGE [HASH(c_name,c_custkey,o_orderkey,o_orderdate,o_totalprice)] +| mem-estimate=0B mem-reservation=0B +| tuple-ids=6 row-size=100B cardinality=575772 +| +F02:PLAN FRAGMENT [HASH(l_orderkey)] hosts=3 instances=3 +08:AGGREGATE [STREAMING] +| output: sum(l_quantity) +| group by: c_name, c_custkey, o_orderkey, o_orderdate, o_totalprice +| mem-estimate=60.40MB mem-reservation=0B +| tuple-ids=6 row-size=100B cardinality=575772 +| +07:HASH JOIN [LEFT SEMI JOIN, PARTITIONED] +| hash predicates: o_orderkey = l_orderkey +| runtime filters: RF000 <- l_orderkey +| mem-estimate=1.31MB mem-reservation=136.00MB +| tuple-ids=2,1,0 row-size=108B cardinality=575772 +| +|--14:AGGREGATE [FINALIZE] +| | output: sum:merge(l_quantity) +| | group by: l_orderkey +| | having: sum(l_quantity) > 300 +| | mem-estimate=10.00MB mem-reservation=264.00MB +| | tuple-ids=4 row-size=24B cardinality=156344 +| | +| 13:EXCHANGE [HASH(l_orderkey)] +| | mem-estimate=0B mem-reservation=0B +| | tuple-ids=4 row-size=24B cardinality=1563438 +| | +| F04:PLAN FRAGMENT [RANDOM] hosts=3 instances=3 +| 04:AGGREGATE [STREAMING] +| | output: sum(l_quantity) +| | group by: l_orderkey +| | mem-estimate=39.36MB mem-reservation=0B +| | tuple-ids=4 row-size=24B cardinality=1563438 +| | +| 03:SCAN HDFS [tpch.lineitem, RANDOM] +| partitions=1/1 files=1 size=718.94MB +| table stats: 6001215 rows total +| column stats: all +| mem-estimate=88.00MB mem-reservation=0B +| tuple-ids=3 row-size=16B cardinality=6001215 +| +06:HASH JOIN [INNER JOIN, BROADCAST] +| hash predicates: o_custkey = c_custkey +| runtime filters: RF001 <- c_custkey +| mem-estimate=6.61MB mem-reservation=136.00MB +| tuple-ids=2,1,0 row-size=108B cardinality=5757710 +| +|--12:EXCHANGE [BROADCAST] +| | mem-estimate=0B mem-reservation=0B +| | tuple-ids=0 row-size=42B cardinality=150000 +| | +| F03:PLAN FRAGMENT [RANDOM] hosts=1 instances=1 +| 00:SCAN HDFS [tpch.customer, RANDOM] +| partitions=1/1 files=1 size=23.08MB +| table stats: 150000 rows total +| column stats: all +| mem-estimate=32.00MB mem-reservation=0B +| tuple-ids=0 row-size=42B cardinality=150000 +| +05:HASH JOIN [INNER JOIN, PARTITIONED] +| hash predicates: l_orderkey = o_orderkey +| runtime filters: RF002 <- o_orderkey +| mem-estimate=26.23MB mem-reservation=136.00MB +| tuple-ids=2,1 row-size=66B cardinality=5757710 +| +|--11:EXCHANGE [HASH(o_orderkey)] +| | mem-estimate=0B mem-reservation=0B +| | tuple-ids=1 row-size=50B cardinality=1500000 +| | +| F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=2 +| 01:SCAN HDFS [tpch.orders, RANDOM] +| partitions=1/1 files=1 size=162.56MB +| runtime filters: RF000 -> o_orderkey, RF001 -> o_custkey +| table stats: 1500000 rows total +| column stats: all +| mem-estimate=88.00MB mem-reservation=0B +| tuple-ids=1 row-size=50B cardinality=1500000 +| +10:EXCHANGE [HASH(l_orderkey)] +| mem-estimate=0B mem-reservation=0B +| tuple-ids=2 row-size=16B cardinality=6001215 +| +F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3 +02:SCAN HDFS [tpch.lineitem, RANDOM] + partitions=1/1 files=1 size=718.94MB + runtime filters: RF000 -> tpch.lineitem.l_orderkey, RF002 -> l_orderkey + table stats: 6001215 rows total + column stats: all + mem-estimate=88.00MB mem-reservation=0B + tuple-ids=2 row-size=16B cardinality=6001215 +---- PARALLELPLANS +Per-Host Resource Reservation: Memory=1.83GB +Per-Host Resource Estimates: Memory=973.08MB + +F07:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1 +PLAN-ROOT SINK +| mem-estimate=0B mem-reservation=0B +| +17:MERGING-EXCHANGE [UNPARTITIONED] +| order by: o_totalprice DESC, o_orderdate ASC +| limit: 100 +| mem-estimate=0B mem-reservation=0B +| tuple-ids=7 row-size=100B cardinality=100 +| +F06:PLAN FRAGMENT [HASH(c_name,c_custkey,o_orderkey,o_orderdate,o_totalprice)] hosts=3 instances=6 +09:TOP-N [LIMIT=100] +| order by: o_totalprice DESC, o_orderdate ASC +| mem-estimate=9.77KB mem-reservation=0B +| tuple-ids=7 row-size=100B cardinality=100 +| +16:AGGREGATE [FINALIZE] +| output: sum:merge(l_quantity) +| group by: c_name, c_custkey, o_orderkey, o_orderdate, o_totalprice +| mem-estimate=60.40MB mem-reservation=264.00MB +| tuple-ids=6 row-size=100B cardinality=575772 +| +15:EXCHANGE [HASH(c_name,c_custkey,o_orderkey,o_orderdate,o_totalprice)] +| mem-estimate=0B mem-reservation=0B +| tuple-ids=6 row-size=100B cardinality=575772 +| +F02:PLAN FRAGMENT [HASH(l_orderkey)] hosts=3 instances=6 +08:AGGREGATE [STREAMING] +| output: sum(l_quantity) +| group by: c_name, c_custkey, o_orderkey, o_orderdate, o_totalprice +| mem-estimate=60.40MB mem-reservation=0B +| tuple-ids=6 row-size=100B cardinality=575772 +| +07:HASH JOIN [LEFT SEMI JOIN, PARTITIONED] +| hash-table-id=00 +| hash predicates: o_orderkey = l_orderkey +| runtime filters: RF000 <- l_orderkey +| mem-estimate=671.79KB mem-reservation=136.00MB +| tuple-ids=2,1,0 row-size=108B cardinality=575772 +| +|--F08:PLAN FRAGMENT [HASH(l_orderkey)] hosts=3 instances=6 +| JOIN BUILD +| | join-table-id=00 plan-id=01 cohort-id=01 +| | build expressions: l_orderkey +| | mem-estimate=0B mem-reservation=0B +| | +| 14:AGGREGATE [FINALIZE] +| | output: sum:merge(l_quantity) +| | group by: l_orderkey +| | having: sum(l_quantity) > 300 +| | mem-estimate=10.00MB mem-reservation=264.00MB +| | tuple-ids=4 row-size=24B cardinality=156344 +| | +| 13:EXCHANGE [HASH(l_orderkey)] +| | mem-estimate=0B mem-reservation=0B +| | tuple-ids=4 row-size=24B cardinality=1563438 +| | +| F04:PLAN FRAGMENT [RANDOM] hosts=3 instances=6 +| 04:AGGREGATE [STREAMING] +| | output: sum(l_quantity) +| | group by: l_orderkey +| | mem-estimate=39.36MB mem-reservation=0B +| | tuple-ids=4 row-size=24B cardinality=1563438 +| | +| 03:SCAN HDFS [tpch.lineitem, RANDOM] +| partitions=1/1 files=1 size=718.94MB +| table stats: 6001215 rows total +| column stats: all +| mem-estimate=88.00MB mem-reservation=0B +| tuple-ids=3 row-size=16B cardinality=6001215 +| +06:HASH JOIN [INNER JOIN, BROADCAST] +| hash-table-id=01 +| hash predicates: o_custkey = c_custkey +| runtime filters: RF001 <- c_custkey +| mem-estimate=6.61MB mem-reservation=136.00MB +| tuple-ids=2,1,0 row-size=108B cardinality=5757710 +| +|--F09:PLAN FRAGMENT [HASH(l_orderkey)] hosts=1 instances=2 +| JOIN BUILD +| | join-table-id=01 plan-id=02 cohort-id=01 +| | build expressions: c_custkey +| | mem-estimate=0B mem-reservation=0B +| | +| 12:EXCHANGE [BROADCAST] +| | mem-estimate=0B mem-reservation=0B +| | tuple-ids=0 row-size=42B cardinality=150000 +| | +| F03:PLAN FRAGMENT [RANDOM] hosts=1 instances=2 +| 00:SCAN HDFS [tpch.customer, RANDOM] +| partitions=1/1 files=1 size=23.08MB +| table stats: 150000 rows total +| column stats: all +| mem-estimate=32.00MB mem-reservation=0B +| tuple-ids=0 row-size=42B cardinality=150000 +| +05:HASH JOIN [INNER JOIN, PARTITIONED] +| hash-table-id=02 +| hash predicates: l_orderkey = o_orderkey +| runtime filters: RF002 <- o_orderkey +| mem-estimate=13.11MB mem-reservation=136.00MB +| tuple-ids=2,1 row-size=66B cardinality=5757710 +| +|--F10:PLAN FRAGMENT [HASH(l_orderkey)] hosts=2 instances=4 +| JOIN BUILD +| | join-table-id=02 plan-id=03 cohort-id=01 +| | build expressions: o_orderkey +| | mem-estimate=0B mem-reservation=0B +| | +| 11:EXCHANGE [HASH(o_orderkey)] +| | mem-estimate=0B mem-reservation=0B +| | tuple-ids=1 row-size=50B cardinality=1500000 +| | +| F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=4 +| 01:SCAN HDFS [tpch.orders, RANDOM] +| partitions=1/1 files=1 size=162.56MB +| runtime filters: RF000 -> o_orderkey, RF001 -> o_custkey +| table stats: 1500000 rows total +| column stats: all +| mem-estimate=88.00MB mem-reservation=0B +| tuple-ids=1 row-size=50B cardinality=1500000 +| +10:EXCHANGE [HASH(l_orderkey)] +| mem-estimate=0B mem-reservation=0B +| tuple-ids=2 row-size=16B cardinality=6001215 +| +F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6 +02:SCAN HDFS [tpch.lineitem, RANDOM] + partitions=1/1 files=1 size=718.94MB + runtime filters: RF000 -> tpch.lineitem.l_orderkey, RF002 -> l_orderkey + table stats: 6001215 rows total + column stats: all + mem-estimate=88.00MB mem-reservation=0B + tuple-ids=2 row-size=16B cardinality=6001215 +==== +# Unpartitioned HDFS insert +insert into table functional.alltypesnopart +select id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, +float_col, double_col, date_string_col, string_col, timestamp_col +from functional.alltypes +where year=2009 and month=05 +---- DISTRIBUTEDPLAN +Per-Host Resource Reservation: Memory=0B +Per-Host Resource Estimates: Memory=16.03MB + +F00:PLAN FRAGMENT [RANDOM] hosts=1 instances=1 +WRITE TO HDFS [functional.alltypesnopart, OVERWRITE=false] +| partitions=1 +| mem-estimate=26.94KB mem-reservation=0B +| +00:SCAN HDFS [functional.alltypes, RANDOM] + partitions=1/24 files=1 size=20.36KB + table stats: 7300 rows total + column stats: all + mem-estimate=16.00MB mem-reservation=0B + tuple-ids=0 row-size=89B cardinality=310 +---- PARALLELPLANS +Per-Host Resource Reservation: Memory=0B +Per-Host Resource Estimates: Memory=32.03MB + +F00:PLAN FRAGMENT [RANDOM] hosts=1 instances=2 +WRITE TO HDFS [functional.alltypesnopart, OVERWRITE=false] +| partitions=1 +| mem-estimate=13.47KB mem-reservation=0B +| +00:SCAN HDFS [functional.alltypes, RANDOM] + partitions=1/24 files=1 size=20.36KB + table stats: 7300 rows total + column stats: all + mem-estimate=16.00MB mem-reservation=0B + tuple-ids=0 row-size=89B cardinality=310 +==== +# Dynamically partitioned HDFS insert +create table dummy_insert +partitioned by (l_partkey) as +select l_comment, l_partkey from tpch.lineitem +---- DISTRIBUTEDPLAN +Per-Host Resource Reservation: Memory=0B +Per-Host Resource Estimates: Memory=184.33MB + +F01:PLAN FRAGMENT [HASH(l_partkey)] hosts=3 instances=3 +WRITE TO HDFS [default.dummy_insert, OVERWRITE=false, PARTITION-KEYS=(l_partkey)] +| partitions=200516 +| mem-estimate=96.33MB mem-reservation=0B +| +01:EXCHANGE [HASH(l_partkey)] +| mem-estimate=0B mem-reservation=0B +| tuple-ids=0 row-size=50B cardinality=6001215 +| +F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3 +00:SCAN HDFS [tpch.lineitem, RANDOM] + partitions=1/1 files=1 size=718.94MB + table stats: 6001215 rows total + column stats: all + mem-estimate=88.00MB mem-reservation=0B + tuple-ids=0 row-size=50B cardinality=6001215 +---- PARALLELPLANS +Per-Host Resource Reservation: Memory=0B +Per-Host Resource Estimates: Memory=272.33MB + +F01:PLAN FRAGMENT [HASH(l_partkey)] hosts=3 instances=6 +WRITE TO HDFS [default.dummy_insert, OVERWRITE=false, PARTITION-KEYS=(l_partkey)] +| partitions=200516 +| mem-estimate=48.16MB mem-reservation=0B +| +01:EXCHANGE [HASH(l_partkey)] +| mem-estimate=0B mem-reservation=0B +| tuple-ids=0 row-size=50B cardinality=6001215 +| +F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6 +00:SCAN HDFS [tpch.lineitem, RANDOM] + partitions=1/1 files=1 size=718.94MB + table stats: 6001215 rows total + column stats: all + mem-estimate=88.00MB mem-reservation=0B + tuple-ids=0 row-size=50B cardinality=6001215 +====
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9a29dfc9/testdata/workloads/functional-query/queries/QueryTest/explain-level0.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/explain-level0.test b/testdata/workloads/functional-query/queries/QueryTest/explain-level0.test index 0cb0610..06d76c8 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/explain-level0.test +++ b/testdata/workloads/functional-query/queries/QueryTest/explain-level0.test @@ -1,117 +1,19 @@ ==== ---- QUERY -# Tests explaining a query (TPCDS-Q19) +# Explain a simple hash join query. explain -select - i_brand_id, - i_brand, - i_manufact_id, - i_manufact, - sum(ss_ext_sales_price) ext_price -from - tpcds.store_sales - join tpcds.item on (store_sales.ss_item_sk = item.i_item_sk) - join tpcds.customer on (store_sales.ss_customer_sk = customer.c_customer_sk) - join tpcds.customer_address on (customer.c_current_addr_sk = customer_address.ca_address_sk) - join tpcds.store on (store_sales.ss_store_sk = store.s_store_sk) -where - ss_date between '1999-11-01' and '1999-11-30' - and i_manager_id = 7 - and substr(ca_zip, 1, 5) <> substr(s_zip, 1, 5) -group by - i_brand, - i_brand_id, - i_manufact_id, - i_manufact -order by - ext_price desc, - i_brand, - i_brand_id, - i_manufact_id, - i_manufact -limit 100 ----- RESULTS -'Estimated Per-Host Requirements: Memory=72.66MB VCores=5' +select * +from tpch.lineitem join tpch.orders on l_orderkey = o_orderkey; +---- RESULTS: VERIFY_IS_EQUAL +'Per-Host Resource Reservation: Memory=136.00MB' +'Per-Host Resource Estimates: Memory=388.41MB' '' -'19:TOP-N [LIMIT=100]' -'18:EXCHANGE [UNPARTITIONED]' -'10:TOP-N [LIMIT=100]' -'17:AGGREGATE [MERGE FINALIZE]' -'16:EXCHANGE [HASH(i_brand,i_brand_id,i_manufact_id,i_manufact)]' -'09:AGGREGATE' -'08:HASH JOIN [INNER JOIN, BROADCAST]' -'|--15:EXCHANGE [BROADCAST]' -'| 04:SCAN HDFS [tpcds.store]' -'07:HASH JOIN [INNER JOIN, BROADCAST]' -'|--14:EXCHANGE [BROADCAST]' -'| 01:SCAN HDFS [tpcds.item]' -'06:HASH JOIN [INNER JOIN, BROADCAST]' -'|--13:EXCHANGE [BROADCAST]' -'| 00:SCAN HDFS [tpcds.store_sales]' -'05:HASH JOIN [INNER JOIN, PARTITIONED]' -'|--12:EXCHANGE [HASH(customer.c_current_addr_sk)]' -'| 02:SCAN HDFS [tpcds.customer]' -'11:EXCHANGE [HASH(customer_address.ca_address_sk)]' -'03:SCAN HDFS [tpcds.customer_address]' -==== ----- QUERY -# Tests explaining an insert query -explain insert overwrite functional.alltypessmall (id, string_col) -partition (year, month) -select a.id, a.string_col, a.year, a.month from functional.alltypes a - left semi join functional.alltypesagg b on (a.id = b.id) -where a.year > 2009 and a.month = 4 -union distinct -select id, string_col, year, month from functional.alltypes ----- RESULTS -'Estimated Per-Host Requirements: Memory=30.00MB VCores=3' -'' -'WRITE TO HDFS [functional.alltypessmall, OVERWRITE=true, PARTITION-KEYS=(year,month)]' -'14:EXCHANGE [HASH(year,month)]' -'13:AGGREGATE [MERGE FINALIZE]' -'12:EXCHANGE [HASH(id,string_col,year,month)]' -'|--11:AGGREGATE' -'| 10:MERGE' -'| 04:SCAN HDFS [functional.alltypes]' -'05:AGGREGATE' -'09:MERGE' -'03:HASH JOIN [LEFT SEMI JOIN, PARTITIONED]' -'|--07:EXCHANGE [HASH(b.id)]' -'| 02:SCAN HDFS [functional.alltypesagg b]' -'06:EXCHANGE [HASH(a.id)]' -'01:SCAN HDFS [functional.alltypes a]' -==== ----- QUERY -# Tests explaining an insert query to/from an HBase table -explain insert into functional_hbase.alltypes -select a.* from functional_hbase.alltypessmall a - cross join functional.alltypessmall b -where a.year > 2009 and a.month = 4 -union all -select * from functional_hbase.alltypessmall ----- RESULTS -'Estimated Per-Host Requirements: Memory=1.02GB VCores=3' -'' -'WRITE TO HBASE table=functional_hbase.alltypes' -'06:EXCHANGE [UNPARTITIONED]' -'|--08:MERGE' -'| 04:SCAN HBASE [functional_hbase.alltypessmall]' -'07:MERGE' -'03:CROSS JOIN [BROADCAST]' -'|--05:EXCHANGE [BROADCAST]' -'| 02:SCAN HDFS [functional.alltypessmall b]' -'01:SCAN HBASE [functional_hbase.alltypessmall a]' -==== ----- QUERY -# Tests explaining an CTAS statement. -explain create table t as -select * from functional.alltypes -where month = 2 ----- RESULTS -'Estimated Per-Host Requirements: Memory=16.02MB VCores=1' -'' -'WRITE TO HDFS [functional.t, OVERWRITE=false]' -'00:SCAN HDFS [functional.alltypes]' +'PLAN-ROOT SINK' +'04:EXCHANGE [UNPARTITIONED]' +'02:HASH JOIN [INNER JOIN, BROADCAST]' +'|--03:EXCHANGE [BROADCAST]' +'| 01:SCAN HDFS [tpch.orders]' +'00:SCAN HDFS [tpch.lineitem]' ==== ---- QUERY # Tests the warning about missing table stats in the explain header. @@ -120,19 +22,7 @@ from functional_avro.alltypes t1 inner join functional_parquet.alltypessmall t2 on (t1.id = t2.id) left outer join functional_avro.alltypes t3 on (t2.id = t3.id) where t1.month = 1 and t2.year = 2009 and t3.bool_col = false ----- RESULTS -'Estimated Per-Host Requirements: Memory=4.03GB VCores=3' +---- RESULTS: VERIFY_IS_SUBSET 'WARNING: The following tables are missing relevant table and/or column statistics.' 'functional_avro.alltypes, functional_parquet.alltypessmall' -'' -'09:AGGREGATE [MERGE FINALIZE]' -'08:EXCHANGE [UNPARTITIONED]' -'05:AGGREGATE' -'04:HASH JOIN [LEFT OUTER JOIN, BROADCAST]' -'|--07:EXCHANGE [BROADCAST]' -'| 02:SCAN HDFS [functional_avro.alltypes t3]' -'03:HASH JOIN [INNER JOIN, BROADCAST]' -'|--06:EXCHANGE [BROADCAST]' -'| 01:SCAN HDFS [functional_parquet.alltypessmall t2]' -'00:SCAN HDFS [functional_avro.alltypes t1]' ==== http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9a29dfc9/testdata/workloads/functional-query/queries/QueryTest/explain-level1.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/explain-level1.test b/testdata/workloads/functional-query/queries/QueryTest/explain-level1.test index a1a172d..a082603 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/explain-level1.test +++ b/testdata/workloads/functional-query/queries/QueryTest/explain-level1.test @@ -1,187 +1,29 @@ ==== ---- QUERY -# Tests explaining a query (TPCDS-Q19) +# Explain a simple hash join query. explain -select - i_brand_id, - i_brand, - i_manufact_id, - i_manufact, - sum(ss_ext_sales_price) ext_price -from - tpcds.store_sales - join tpcds.item on (store_sales.ss_item_sk = item.i_item_sk) - join tpcds.customer on (store_sales.ss_customer_sk = customer.c_customer_sk) - join tpcds.customer_address on (customer.c_current_addr_sk = customer_address.ca_address_sk) - join tpcds.store on (store_sales.ss_store_sk = store.s_store_sk) -where - ss_date between '1999-11-01' and '1999-11-30' - and i_manager_id = 7 - and substr(ca_zip, 1, 5) <> substr(s_zip, 1, 5) -group by - i_brand, - i_brand_id, - i_manufact_id, - i_manufact -order by - ext_price desc, - i_brand, - i_brand_id, - i_manufact_id, - i_manufact -limit 100 ----- RESULTS -'Estimated Per-Host Requirements: Memory=72.66MB VCores=5' +select * +from tpch.lineitem join tpch.orders on l_orderkey = o_orderkey; +---- RESULTS: VERIFY_IS_EQUAL +'Per-Host Resource Reservation: Memory=136.00MB' +'Per-Host Resource Estimates: Memory=388.41MB' '' -'19:TOP-N [LIMIT=100]' -'| order by: sum(ss_ext_sales_price) DESC, i_brand ASC, i_brand_id ASC, i_manufact_id ASC, i_manufact ASC' +'PLAN-ROOT SINK' '|' -'18:EXCHANGE [UNPARTITIONED]' +'04:EXCHANGE [UNPARTITIONED]' '|' -'10:TOP-N [LIMIT=100]' -'| order by: sum(ss_ext_sales_price) DESC, i_brand ASC, i_brand_id ASC, i_manufact_id ASC, i_manufact ASC' +'02:HASH JOIN [INNER JOIN, BROADCAST]' +'| hash predicates: l_orderkey = o_orderkey' +'| runtime filters: RF000 <- o_orderkey' '|' -'17:AGGREGATE [MERGE FINALIZE]' -'| output: sum(sum(ss_ext_sales_price))' -'| group by: i_brand, i_brand_id, i_manufact_id, i_manufact' -'|' -'16:EXCHANGE [HASH(i_brand,i_brand_id,i_manufact_id,i_manufact)]' -'|' -'09:AGGREGATE' -'| output: sum(ss_ext_sales_price)' -'| group by: i_brand, i_brand_id, i_manufact_id, i_manufact' -'|' -'08:HASH JOIN [INNER JOIN, BROADCAST]' -'| hash predicates: store_sales.ss_store_sk = store.s_store_sk' -'| other predicates: substr(ca_zip, 1, 5) != substr(s_zip, 1, 5)' -'|' -'|--15:EXCHANGE [BROADCAST]' -'| |' -'| 04:SCAN HDFS [tpcds.store]' -'| partitions=1/1 size=3.08KB' -'|' -'07:HASH JOIN [INNER JOIN, BROADCAST]' -'| hash predicates: store_sales.ss_item_sk = item.i_item_sk' -'|' -'|--14:EXCHANGE [BROADCAST]' +'|--03:EXCHANGE [BROADCAST]' '| |' -'| 01:SCAN HDFS [tpcds.item]' -'| partitions=1/1 size=4.82MB' -'| predicates: i_manager_id = 7' -'|' -'06:HASH JOIN [INNER JOIN, BROADCAST]' -'| hash predicates: customer.c_customer_sk = store_sales.ss_customer_sk' -'|' -'|--13:EXCHANGE [BROADCAST]' -'| |' -'| 00:SCAN HDFS [tpcds.store_sales]' -'| partitions=2/120 size=663.52KB' -'|' -'05:HASH JOIN [INNER JOIN, PARTITIONED]' -'| hash predicates: customer_address.ca_address_sk = customer.c_current_addr_sk' -'|' -'|--12:EXCHANGE [HASH(customer.c_current_addr_sk)]' -'| |' -'| 02:SCAN HDFS [tpcds.customer]' -'| partitions=1/1 size=12.60MB' -'|' -'11:EXCHANGE [HASH(customer_address.ca_address_sk)]' +'| 01:SCAN HDFS [tpch.orders]' +row_regex:.*partitions=1/1 files=1 size=.* '|' -'03:SCAN HDFS [tpcds.customer_address]' -' partitions=1/1 size=5.25MB' -==== ----- QUERY -# Tests explaining an insert query -explain insert overwrite functional.alltypessmall (id, string_col) -partition (year, month) -select a.id, a.string_col, a.year, a.month from functional.alltypes a - left semi join functional.alltypesagg b on (a.id = b.id) -where a.year > 2009 and a.month = 4 -union distinct -select id, string_col, year, month from functional.alltypes ----- RESULTS -'Estimated Per-Host Requirements: Memory=30.00MB VCores=3' -'' -'WRITE TO HDFS [functional.alltypessmall, OVERWRITE=true, PARTITION-KEYS=(year,month)]' -'| partitions=96' -'|' -'14:EXCHANGE [HASH(year,month)]' -'|' -'13:AGGREGATE [MERGE FINALIZE]' -'| group by: id, string_col, year, month' -'|' -'12:EXCHANGE [HASH(id,string_col,year,month)]' -'|' -'|--11:AGGREGATE' -'| | group by: id, string_col, year, month' -'| |' -'| 10:MERGE' -'| |' -'| 04:SCAN HDFS [functional.alltypes]' -'| partitions=24/24 size=478.45KB' -'|' -'05:AGGREGATE' -'| group by: id, string_col, year, month' -'|' -'09:MERGE' -'|' -'03:HASH JOIN [LEFT SEMI JOIN, PARTITIONED]' -'| hash predicates: a.id = b.id' -'|' -'|--07:EXCHANGE [HASH(b.id)]' -'| |' -'| 02:SCAN HDFS [functional.alltypesagg b]' -'| partitions=10/10 size=743.67KB' -'|' -'06:EXCHANGE [HASH(a.id)]' -'|' -'01:SCAN HDFS [functional.alltypes a]' -' partitions=1/24 size=19.71KB' -==== ----- QUERY -# Tests explaining an insert query to/from an HBase table -explain insert into functional_hbase.alltypes -select a.* from functional_hbase.alltypessmall a - cross join functional.alltypessmall b -where a.year > 2009 and a.month = 4 -union all -select * from functional_hbase.alltypessmall ----- RESULTS -'Estimated Per-Host Requirements: Memory=1.02GB VCores=3' -'' -'WRITE TO HBASE table=functional_hbase.alltypes' -'|' -'06:EXCHANGE [UNPARTITIONED]' -'|' -'|--08:MERGE' -'| |' -'| 04:SCAN HBASE [functional_hbase.alltypessmall]' -'|' -'07:MERGE' -'|' -'03:CROSS JOIN [BROADCAST]' -'|' -'|--05:EXCHANGE [BROADCAST]' -'| |' -'| 02:SCAN HDFS [functional.alltypessmall b]' -'| partitions=4/4 size=6.32KB' -'|' -'01:SCAN HBASE [functional_hbase.alltypessmall a]' -' predicates: a.year > 2009, a.month = 4' -==== ----- QUERY -# Tests explaining an CTAS statement. -explain create table t as -select * from functional.alltypes -where month = 2 ----- RESULTS -'Estimated Per-Host Requirements: Memory=16.02MB VCores=1' -'' -'WRITE TO HDFS [functional.t, OVERWRITE=false]' -'| partitions=1' -'|' -'00:SCAN HDFS [functional.alltypes]' -' partitions=2/24 size=36.51KB' +'00:SCAN HDFS [tpch.lineitem]' +row_regex:.*partitions=1/1 files=1 size=.* +' runtime filters: RF000 -> l_orderkey' ==== ---- QUERY # Tests the warning about missing table stats in the explain header. @@ -190,37 +32,7 @@ from functional_avro.alltypes t1 inner join functional_parquet.alltypessmall t2 on (t1.id = t2.id) left outer join functional_avro.alltypes t3 on (t2.id = t3.id) where t1.month = 1 and t2.year = 2009 and t3.bool_col = false ----- RESULTS -'Estimated Per-Host Requirements: Memory=4.03GB VCores=3' +---- RESULTS: VERIFY_IS_SUBSET 'WARNING: The following tables are missing relevant table and/or column statistics.' 'functional_avro.alltypes, functional_parquet.alltypessmall' -'' -'09:AGGREGATE [MERGE FINALIZE]' -'| output: sum(count(t1.int_col)), sum(sum(t2.float_col)), sum(count(t2.float_col)), sum(sum(t3.bigint_col))' -'|' -'08:EXCHANGE [UNPARTITIONED]' -'|' -'05:AGGREGATE' -'| output: count(t1.int_col), sum(t2.float_col), count(t2.float_col), sum(t3.bigint_col)' -'|' -'04:HASH JOIN [LEFT OUTER JOIN, BROADCAST]' -'| hash predicates: t2.id = t3.id' -'| other predicates: t3.bool_col = FALSE' -'|' -'|--07:EXCHANGE [BROADCAST]' -'| |' -'| 02:SCAN HDFS [functional_avro.alltypes t3]' -'| partitions=24/24 size=470.35KB' -'| predicates: t3.bool_col = FALSE' -'|' -'03:HASH JOIN [INNER JOIN, BROADCAST]' -'| hash predicates: t1.id = t2.id' -'|' -'|--06:EXCHANGE [BROADCAST]' -'| |' -'| 01:SCAN HDFS [functional_parquet.alltypessmall t2]' -'| partitions=4/4 size=9.63KB' -'|' -'00:SCAN HDFS [functional_avro.alltypes t1]' -' partitions=2/24 size=39.87KB' ==== http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9a29dfc9/testdata/workloads/functional-query/queries/QueryTest/explain-level2.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/explain-level2.test b/testdata/workloads/functional-query/queries/QueryTest/explain-level2.test index 2206ae8..32171fb 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/explain-level2.test +++ b/testdata/workloads/functional-query/queries/QueryTest/explain-level2.test @@ -1,298 +1,47 @@ ==== ---- QUERY -# Tests explaining a query (TPCDS-Q19) +# Explain a simple hash join query. explain -select - i_brand_id, - i_brand, - i_manufact_id, - i_manufact, - sum(ss_ext_sales_price) ext_price -from - tpcds.store_sales - join tpcds.item on (store_sales.ss_item_sk = item.i_item_sk) - join tpcds.customer on (store_sales.ss_customer_sk = customer.c_customer_sk) - join tpcds.customer_address on (customer.c_current_addr_sk = customer_address.ca_address_sk) - join tpcds.store on (store_sales.ss_store_sk = store.s_store_sk) -where - ss_date between '1999-11-01' and '1999-11-30' - and i_manager_id = 7 - and substr(ca_zip, 1, 5) <> substr(s_zip, 1, 5) -group by - i_brand, - i_brand_id, - i_manufact_id, - i_manufact -order by - ext_price desc, - i_brand, - i_brand_id, - i_manufact_id, - i_manufact -limit 100 ----- RESULTS -'Estimated Per-Host Requirements: Memory=72.66MB VCores=5' +select * +from tpch.lineitem join tpch.orders on l_orderkey = o_orderkey; +---- RESULTS: VERIFY_IS_EQUAL +'Per-Host Resource Reservation: Memory=136.00MB' +'Per-Host Resource Estimates: Memory=388.41MB' '' -'19:TOP-N [LIMIT=100]' -'| order by: sum(ss_ext_sales_price) DESC, i_brand ASC, i_brand_id ASC, i_manufact_id ASC, i_manufact ASC' -'| hosts=3 per-host-mem=unavailable' -'| tuple-ids=5 row-size=48B cardinality=100' -'|' -'18:EXCHANGE [UNPARTITIONED]' -'| hosts=3 per-host-mem=unavailable' -'| tuple-ids=5 row-size=48B cardinality=100' -'|' -'10:TOP-N [LIMIT=100]' -'| order by: sum(ss_ext_sales_price) DESC, i_brand ASC, i_brand_id ASC, i_manufact_id ASC, i_manufact ASC' -'| hosts=3 per-host-mem=4.69KB' -'| tuple-ids=5 row-size=48B cardinality=100' -'|' -'17:AGGREGATE [MERGE FINALIZE]' -'| output: sum(sum(ss_ext_sales_price))' -'| group by: i_brand, i_brand_id, i_manufact_id, i_manufact' -'| hosts=3 per-host-mem=10.00MB' -'| tuple-ids=5 row-size=48B cardinality=87208' -'|' -'16:EXCHANGE [HASH(i_brand,i_brand_id,i_manufact_id,i_manufact)]' -'| hosts=3 per-host-mem=0B' -'| tuple-ids=5 row-size=48B cardinality=87208' -'|' -'09:AGGREGATE' -'| output: sum(ss_ext_sales_price)' -'| group by: i_brand, i_brand_id, i_manufact_id, i_manufact' -'| hosts=3 per-host-mem=10.00MB' -'| tuple-ids=5 row-size=48B cardinality=87208' -'|' -'08:HASH JOIN [INNER JOIN, BROADCAST]' -'| hash predicates: store_sales.ss_store_sk = store.s_store_sk' -'| other predicates: substr(ca_zip, 1, 5) != substr(s_zip, 1, 5)' -'| hosts=3 per-host-mem=383B' -'| tuple-ids=3,2,0,1,4 row-size=192B cardinality=87208' -'|' -'|--15:EXCHANGE [BROADCAST]' -'| | hosts=3 per-host-mem=0B' -'| | tuple-ids=4 row-size=29B cardinality=12' -'| |' -'| 04:SCAN HDFS [tpcds.store, RANDOM]' -'| partitions=1/1 size=3.08KB' -'| table stats: 12 rows total' -'| column stats: all' -'| hosts=3 per-host-mem=16.00MB' -'| tuple-ids=4 row-size=29B cardinality=12' -'|' -'07:HASH JOIN [INNER JOIN, BROADCAST]' -'| hash predicates: store_sales.ss_item_sk = item.i_item_sk' -'| hosts=3 per-host-mem=14.60KB' -'| tuple-ids=3,2,0,1 row-size=163B cardinality=87208' -'|' -'|--14:EXCHANGE [BROADCAST]' -'| | hosts=3 per-host-mem=0B' -'| | tuple-ids=1 row-size=79B cardinality=171' -'| |' -'| 01:SCAN HDFS [tpcds.item, RANDOM]' -'| partitions=1/1 size=4.82MB' -'| predicates: i_manager_id = 7' -'| table stats: 18000 rows total' -'| column stats: all' -'| hosts=3 per-host-mem=16.00MB' -'| tuple-ids=1 row-size=79B cardinality=171' -'|' -'06:HASH JOIN [INNER JOIN, BROADCAST]' -'| hash predicates: customer.c_customer_sk = store_sales.ss_customer_sk' -'| hosts=3 per-host-mem=234.75KB' -'| tuple-ids=3,2,0 row-size=84B cardinality=87208' -'|' -'|--13:EXCHANGE [BROADCAST]' -'| | hosts=3 per-host-mem=0B' -'| | tuple-ids=0 row-size=43B cardinality=5082' -'| |' -'| 00:SCAN HDFS [tpcds.store_sales, RANDOM]' -'| partitions=2/120 size=663.52KB' -'| table stats: 183592 rows total' -'| column stats: all' -'| hosts=3 per-host-mem=16.00MB' -'| tuple-ids=0 row-size=43B cardinality=5082' -'|' -'05:HASH JOIN [INNER JOIN, PARTITIONED]' -'| hash predicates: customer_address.ca_address_sk = customer.c_current_addr_sk' -'| hosts=3 per-host-mem=429.69KB' -'| tuple-ids=3,2 row-size=41B cardinality=87208' -'|' -'|--12:EXCHANGE [HASH(customer.c_current_addr_sk)]' -'| | hosts=3 per-host-mem=0B' -'| | tuple-ids=2 row-size=12B cardinality=100000' -'| |' -'| 02:SCAN HDFS [tpcds.customer, RANDOM]' -'| partitions=1/1 size=12.60MB' -'| table stats: 100000 rows total' -'| column stats: all' -'| hosts=3 per-host-mem=24.00MB' -'| tuple-ids=2 row-size=12B cardinality=100000' -'|' -'11:EXCHANGE [HASH(customer_address.ca_address_sk)]' -'| hosts=3 per-host-mem=0B' -'| tuple-ids=3 row-size=29B cardinality=50000' -'|' -'03:SCAN HDFS [tpcds.customer_address, RANDOM]' -' partitions=1/1 size=5.25MB' -' table stats: 50000 rows total' +'F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1' +'PLAN-ROOT SINK' +'| mem-estimate=0B mem-reservation=0B' +'|' +'04:EXCHANGE [UNPARTITIONED]' +'| mem-estimate=0B mem-reservation=0B' +'| tuple-ids=0,1 row-size=454B cardinality=5757710' +'|' +'F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3' +'02:HASH JOIN [INNER JOIN, BROADCAST]' +'| hash predicates: l_orderkey = o_orderkey' +'| runtime filters: RF000 <- o_orderkey' +'| mem-estimate=300.41MB mem-reservation=136.00MB' +'| tuple-ids=0,1 row-size=454B cardinality=5757710' +'|' +'|--03:EXCHANGE [BROADCAST]' +'| | mem-estimate=0B mem-reservation=0B' +'| | tuple-ids=1 row-size=191B cardinality=1500000' +'| |' +'| F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=2' +'| 01:SCAN HDFS [tpch.orders, RANDOM]' +row_regex:.*partitions=1/1 files=1 size=.* +'| table stats: 1500000 rows total' +'| column stats: all' +'| mem-estimate=88.00MB mem-reservation=0B' +'| tuple-ids=1 row-size=191B cardinality=1500000' +'|' +'00:SCAN HDFS [tpch.lineitem, RANDOM]' +row_regex:.*partitions=1/1 files=1 size=.* +' runtime filters: RF000 -> l_orderkey' +' table stats: 6001215 rows total' ' column stats: all' -' hosts=3 per-host-mem=16.00MB' -' tuple-ids=3 row-size=29B cardinality=50000' -==== ----- QUERY -# Tests explaining an insert query -explain insert overwrite functional.alltypessmall (id, string_col) -partition (year, month) -select a.id, a.string_col, a.year, a.month from functional.alltypes a - left semi join functional.alltypesagg b on (a.id = b.id) -where a.year > 2009 and a.month = 4 -union distinct -select id, string_col, year, month from functional.alltypes ----- RESULTS -'Estimated Per-Host Requirements: Memory=30.00MB VCores=3' -'' -'WRITE TO HDFS [functional.alltypessmall, OVERWRITE=true, PARTITION-KEYS=(year,month)]' -'| partitions=96' -'| hosts=3 per-host-mem=69.26KB' -'|' -'14:EXCHANGE [HASH(year,month)]' -'| hosts=3 per-host-mem=0B' -'| tuple-ids=3 row-size=28B cardinality=7600' -'|' -'13:AGGREGATE [MERGE FINALIZE]' -'| group by: id, string_col, year, month' -'| hosts=3 per-host-mem=10.00MB' -'| tuple-ids=3 row-size=28B cardinality=7600' -'|' -'12:EXCHANGE [HASH(id,string_col,year,month)]' -'| hosts=3 per-host-mem=0B' -'| tuple-ids=3 row-size=28B cardinality=7600' -'|' -'|--11:AGGREGATE' -'| | group by: id, string_col, year, month' -'| | hosts=3 per-host-mem=10.00MB' -'| | tuple-ids=3 row-size=28B cardinality=7300' -'| |' -'| 10:MERGE' -'| | hosts=3 per-host-mem=0B' -'| | tuple-ids=3 row-size=28B cardinality=7300' -'| |' -'| 04:SCAN HDFS [functional.alltypes, RANDOM]' -'| partitions=24/24 size=478.45KB' -'| table stats: 7300 rows total' -'| column stats: all' -'| hosts=3 per-host-mem=16.00MB' -'| tuple-ids=2 row-size=29B cardinality=7300' -'|' -'05:AGGREGATE' -'| group by: id, string_col, year, month' -'| hosts=3 per-host-mem=10.00MB' -'| tuple-ids=3 row-size=28B cardinality=300' -'|' -'09:MERGE' -'| hosts=3 per-host-mem=0B' -'| tuple-ids=3 row-size=28B cardinality=300' -'|' -'03:HASH JOIN [LEFT SEMI JOIN, PARTITIONED]' -'| hash predicates: a.id = b.id' -'| hosts=3 per-host-mem=14.32KB' -'| tuple-ids=0,1 row-size=33B cardinality=300' -'|' -'|--07:EXCHANGE [HASH(b.id)]' -'| | hosts=3 per-host-mem=0B' -'| | tuple-ids=1 row-size=4B cardinality=10000' -'| |' -'| 02:SCAN HDFS [functional.alltypesagg b, RANDOM]' -'| partitions=10/10 size=743.67KB' -'| table stats: 10000 rows total' -'| column stats: all' -'| hosts=3 per-host-mem=16.00MB' -'| tuple-ids=1 row-size=4B cardinality=10000' -'|' -'06:EXCHANGE [HASH(a.id)]' -'| hosts=3 per-host-mem=0B' -'| tuple-ids=0 row-size=29B cardinality=300' -'|' -'01:SCAN HDFS [functional.alltypes a, RANDOM]' -' partitions=1/24 size=19.71KB' -' table stats: 7300 rows total' -' column stats: all' -' hosts=3 per-host-mem=16.00MB' -' tuple-ids=0 row-size=29B cardinality=300' -==== ----- QUERY -# Tests explaining an insert query to/from an HBase table -explain insert into functional_hbase.alltypes -select a.* from functional_hbase.alltypessmall a - cross join functional.alltypessmall b -where a.year > 2009 and a.month = 4 -union all -select * from functional_hbase.alltypessmall ----- RESULTS -'Estimated Per-Host Requirements: Memory=1.02GB VCores=3' -'' -'WRITE TO HBASE table=functional_hbase.alltypes' -'| hosts=1 per-host-mem=unavailable' -'|' -'06:EXCHANGE [UNPARTITIONED]' -'| hosts=100 per-host-mem=unavailable' -'| tuple-ids=3 row-size=88B cardinality=596' -'|' -'|--08:MERGE' -'| | hosts=100 per-host-mem=0B' -'| | tuple-ids=3 row-size=88B cardinality=196' -'| |' -'| 04:SCAN HBASE [functional_hbase.alltypessmall]' -'| table stats: 100 rows total' -'| column stats: all' -'| hosts=100 per-host-mem=1.00GB' -'| tuple-ids=2 row-size=97B cardinality=196' -'|' -'07:MERGE' -'| hosts=100 per-host-mem=0B' -'| tuple-ids=3 row-size=88B cardinality=400' -'|' -'03:CROSS JOIN [BROADCAST]' -'| hosts=100 per-host-mem=0B' -'| tuple-ids=0,1 row-size=97B cardinality=400' -'|' -'|--05:EXCHANGE [BROADCAST]' -'| | hosts=3 per-host-mem=0B' -'| | tuple-ids=1 row-size=0B cardinality=100' -'| |' -'| 02:SCAN HDFS [functional.alltypessmall b, RANDOM]' -'| partitions=4/4 size=6.32KB' -'| table stats: 100 rows total' -'| column stats: all' -'| hosts=3 per-host-mem=16.00MB' -'| tuple-ids=1 row-size=0B cardinality=100' -'|' -'01:SCAN HBASE [functional_hbase.alltypessmall a]' -' predicates: a.year > 2009, a.month = 4' -' table stats: 100 rows total' -' column stats: all' -' hosts=100 per-host-mem=1.00GB' -' tuple-ids=0 row-size=97B cardinality=4' -==== ----- QUERY -# Tests explaining an CTAS statement. -explain create table t as -select * from functional.alltypes -where month = 2 ----- RESULTS -'Estimated Per-Host Requirements: Memory=16.02MB VCores=1' -'' -'WRITE TO HDFS [functional.t, OVERWRITE=false]' -'| partitions=1' -'| hosts=3 per-host-mem=17.62KB' -'|' -'00:SCAN HDFS [functional.alltypes, RANDOM]' -' partitions=2/24 size=36.51KB' -' table stats: 7300 rows total' -' column stats: all' -' hosts=3 per-host-mem=16.00MB' -' tuple-ids=1 row-size=97B cardinality=560' +' mem-estimate=88.00MB mem-reservation=0B' +' tuple-ids=0 row-size=263B cardinality=6001215' ==== ---- QUERY # Tests the warning about missing table stats in the explain header. @@ -301,63 +50,8 @@ from functional_avro.alltypes t1 inner join functional_parquet.alltypessmall t2 on (t1.id = t2.id) left outer join functional_avro.alltypes t3 on (t2.id = t3.id) where t1.month = 1 and t2.year = 2009 and t3.bool_col = false ----- RESULTS -'Estimated Per-Host Requirements: Memory=4.03GB VCores=3' +---- RESULTS: VERIFY_IS_SUBSET +'Per-Host Resource Estimates: Memory=4.03GB' 'WARNING: The following tables are missing relevant table and/or column statistics.' 'functional_avro.alltypes, functional_parquet.alltypessmall' -'' -'09:AGGREGATE [MERGE FINALIZE]' -'| output: sum(count(t1.int_col)), sum(sum(t2.float_col)), sum(count(t2.float_col)), sum(sum(t3.bigint_col))' -'| hosts=3 per-host-mem=unavailable' -'| tuple-ids=3 row-size=32B cardinality=1' -'|' -'08:EXCHANGE [UNPARTITIONED]' -'| hosts=3 per-host-mem=unavailable' -'| tuple-ids=3 row-size=32B cardinality=1' -'|' -'05:AGGREGATE' -'| output: count(t1.int_col), sum(t2.float_col), count(t2.float_col), sum(t3.bigint_col)' -'| hosts=3 per-host-mem=10.00MB' -'| tuple-ids=3 row-size=32B cardinality=1' -'|' -'04:HASH JOIN [LEFT OUTER JOIN, BROADCAST]' -'| hash predicates: t2.id = t3.id' -'| other predicates: t3.bool_col = FALSE' -'| hosts=3 per-host-mem=2.00GB' -'| tuple-ids=0,1,2N row-size=37B cardinality=unavailable' -'|' -'|--07:EXCHANGE [BROADCAST]' -'| | hosts=3 per-host-mem=0B' -'| | tuple-ids=2 row-size=13B cardinality=unavailable' -'| |' -'| 02:SCAN HDFS [functional_avro.alltypes t3, RANDOM]' -'| partitions=24/24 size=470.35KB' -'| predicates: t3.bool_col = FALSE' -'| table stats: unavailable' -'| column stats: unavailable' -'| hosts=3 per-host-mem=16.00MB' -'| tuple-ids=2 row-size=13B cardinality=unavailable' -'|' -'03:HASH JOIN [INNER JOIN, BROADCAST]' -'| hash predicates: t1.id = t2.id' -'| hosts=3 per-host-mem=2.00GB' -'| tuple-ids=0,1 row-size=24B cardinality=unavailable' -'|' -'|--06:EXCHANGE [BROADCAST]' -'| | hosts=3 per-host-mem=0B' -'| | tuple-ids=1 row-size=12B cardinality=unavailable' -'| |' -'| 01:SCAN HDFS [functional_parquet.alltypessmall t2, RANDOM]' -'| partitions=4/4 size=9.63KB' -'| table stats: unavailable' -'| columns missing stats: id, float_col' -'| hosts=3 per-host-mem=16.00MB' -'| tuple-ids=1 row-size=12B cardinality=unavailable' -'|' -'00:SCAN HDFS [functional_avro.alltypes t1, RANDOM]' -' partitions=2/24 size=39.87KB' -' table stats: unavailable' -' columns missing stats: id, int_col' -' hosts=3 per-host-mem=16.00MB' -' tuple-ids=0 row-size=12B cardinality=unavailable' ==== http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9a29dfc9/testdata/workloads/functional-query/queries/QueryTest/explain-level3.test ---------------------------------------------------------------------- diff --git a/testdata/workloads/functional-query/queries/QueryTest/explain-level3.test b/testdata/workloads/functional-query/queries/QueryTest/explain-level3.test index 7da9b04..f1ff4a8 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/explain-level3.test +++ b/testdata/workloads/functional-query/queries/QueryTest/explain-level3.test @@ -1,332 +1,51 @@ ==== ---- QUERY -# Tests explaining a query (TPCDS-Q19) +# Explain a simple hash join query. explain -select - i_brand_id, - i_brand, - i_manufact_id, - i_manufact, - sum(ss_ext_sales_price) ext_price -from - tpcds.store_sales - join tpcds.item on (store_sales.ss_item_sk = item.i_item_sk) - join tpcds.customer on (store_sales.ss_customer_sk = customer.c_customer_sk) - join tpcds.customer_address on (customer.c_current_addr_sk = customer_address.ca_address_sk) - join tpcds.store on (store_sales.ss_store_sk = store.s_store_sk) -where - ss_date between '1999-11-01' and '1999-11-30' - and i_manager_id = 7 - and substr(ca_zip, 1, 5) <> substr(s_zip, 1, 5) -group by - i_brand, - i_brand_id, - i_manufact_id, - i_manufact -order by - ext_price desc, - i_brand, - i_brand_id, - i_manufact_id, - i_manufact -limit 100 ----- RESULTS -'Estimated Per-Host Requirements: Memory=72.66MB VCores=5' -'' -'F07:PLAN FRAGMENT [UNPARTITIONED]' -' 19:TOP-N [LIMIT=100]' -' | order by: sum(ss_ext_sales_price) DESC, i_brand ASC, i_brand_id ASC, i_manufact_id ASC, i_manufact ASC' -' | hosts=3 per-host-mem=unavailable' -' | tuple-ids=5 row-size=48B cardinality=100' -' |' -' 18:EXCHANGE [UNPARTITIONED]' -' hosts=3 per-host-mem=unavailable' -' tuple-ids=5 row-size=48B cardinality=100' -'' -'F06:PLAN FRAGMENT [HASH(i_brand,i_brand_id,i_manufact_id,i_manufact)]' -' DATASTREAM SINK [FRAGMENT=F07, EXCHANGE=18, UNPARTITIONED]' -' 10:TOP-N [LIMIT=100]' -' | order by: sum(ss_ext_sales_price) DESC, i_brand ASC, i_brand_id ASC, i_manufact_id ASC, i_manufact ASC' -' | hosts=3 per-host-mem=4.69KB' -' | tuple-ids=5 row-size=48B cardinality=100' -' |' -' 17:AGGREGATE [MERGE FINALIZE]' -' | output: sum(sum(ss_ext_sales_price))' -' | group by: i_brand, i_brand_id, i_manufact_id, i_manufact' -' | hosts=3 per-host-mem=10.00MB' -' | tuple-ids=5 row-size=48B cardinality=87208' -' |' -' 16:EXCHANGE [HASH(i_brand,i_brand_id,i_manufact_id,i_manufact)]' -' hosts=3 per-host-mem=0B' -' tuple-ids=5 row-size=48B cardinality=87208' -'' -'F02:PLAN FRAGMENT [HASH(customer_address.ca_address_sk)]' -' DATASTREAM SINK [FRAGMENT=F06, EXCHANGE=16, HASH(i_brand,i_brand_id,i_manufact_id,i_manufact)]' -' 09:AGGREGATE' -' | output: sum(ss_ext_sales_price)' -' | group by: i_brand, i_brand_id, i_manufact_id, i_manufact' -' | hosts=3 per-host-mem=10.00MB' -' | tuple-ids=5 row-size=48B cardinality=87208' -' |' -' 08:HASH JOIN [INNER JOIN, BROADCAST]' -' | hash predicates: store_sales.ss_store_sk = store.s_store_sk' -' | other predicates: substr(ca_zip, 1, 5) != substr(s_zip, 1, 5)' -' | hosts=3 per-host-mem=383B' -' | tuple-ids=3,2,0,1,4 row-size=192B cardinality=87208' -' |' -' |--15:EXCHANGE [BROADCAST]' -' | hosts=3 per-host-mem=0B' -' | tuple-ids=4 row-size=29B cardinality=12' -' |' -' 07:HASH JOIN [INNER JOIN, BROADCAST]' -' | hash predicates: store_sales.ss_item_sk = item.i_item_sk' -' | hosts=3 per-host-mem=14.60KB' -' | tuple-ids=3,2,0,1 row-size=163B cardinality=87208' -' |' -' |--14:EXCHANGE [BROADCAST]' -' | hosts=3 per-host-mem=0B' -' | tuple-ids=1 row-size=79B cardinality=171' -' |' -' 06:HASH JOIN [INNER JOIN, BROADCAST]' -' | hash predicates: customer.c_customer_sk = store_sales.ss_customer_sk' -' | hosts=3 per-host-mem=234.75KB' -' | tuple-ids=3,2,0 row-size=84B cardinality=87208' -' |' -' |--13:EXCHANGE [BROADCAST]' -' | hosts=3 per-host-mem=0B' -' | tuple-ids=0 row-size=43B cardinality=5082' -' |' -' 05:HASH JOIN [INNER JOIN, PARTITIONED]' -' | hash predicates: customer_address.ca_address_sk = customer.c_current_addr_sk' -' | hosts=3 per-host-mem=429.69KB' -' | tuple-ids=3,2 row-size=41B cardinality=87208' -' |' -' |--12:EXCHANGE [HASH(customer.c_current_addr_sk)]' -' | hosts=3 per-host-mem=0B' -' | tuple-ids=2 row-size=12B cardinality=100000' -' |' -' 11:EXCHANGE [HASH(customer_address.ca_address_sk)]' -' hosts=3 per-host-mem=0B' -' tuple-ids=3 row-size=29B cardinality=50000' -'' -'F05:PLAN FRAGMENT [RANDOM]' -' DATASTREAM SINK [FRAGMENT=F02, EXCHANGE=15, BROADCAST]' -' 04:SCAN HDFS [tpcds.store, RANDOM]' -' partitions=1/1 size=3.08KB' -' table stats: 12 rows total' -' column stats: all' -' hosts=3 per-host-mem=16.00MB' -' tuple-ids=4 row-size=29B cardinality=12' -'' -'F04:PLAN FRAGMENT [RANDOM]' -' DATASTREAM SINK [FRAGMENT=F02, EXCHANGE=14, BROADCAST]' -' 01:SCAN HDFS [tpcds.item, RANDOM]' -' partitions=1/1 size=4.82MB' -' predicates: i_manager_id = 7' -' table stats: 18000 rows total' -' column stats: all' -' hosts=3 per-host-mem=16.00MB' -' tuple-ids=1 row-size=79B cardinality=171' -'' -'F03:PLAN FRAGMENT [RANDOM]' -' DATASTREAM SINK [FRAGMENT=F02, EXCHANGE=13, BROADCAST]' -' 00:SCAN HDFS [tpcds.store_sales, RANDOM]' -' partitions=2/120 size=663.52KB' -' table stats: 183592 rows total' -' column stats: all' -' hosts=3 per-host-mem=16.00MB' -' tuple-ids=0 row-size=43B cardinality=5082' -'' -'F01:PLAN FRAGMENT [RANDOM]' -' DATASTREAM SINK [FRAGMENT=F02, EXCHANGE=12, HASH(customer.c_current_addr_sk)]' -' 02:SCAN HDFS [tpcds.customer, RANDOM]' -' partitions=1/1 size=12.60MB' -' table stats: 100000 rows total' -' column stats: all' -' hosts=3 per-host-mem=24.00MB' -' tuple-ids=2 row-size=12B cardinality=100000' -'' -'F00:PLAN FRAGMENT [RANDOM]' -' DATASTREAM SINK [FRAGMENT=F02, EXCHANGE=11, HASH(customer_address.ca_address_sk)]' -' 03:SCAN HDFS [tpcds.customer_address, RANDOM]' -' partitions=1/1 size=5.25MB' -' table stats: 50000 rows total' -' column stats: all' -' hosts=3 per-host-mem=16.00MB' -' tuple-ids=3 row-size=29B cardinality=50000' -==== ----- QUERY -# Tests explaining an insert query -explain insert overwrite functional.alltypessmall (id, string_col) -partition (year, month) -select a.id, a.string_col, a.year, a.month from functional.alltypes a - left semi join functional.alltypesagg b on (a.id = b.id) -where a.year > 2009 and a.month = 4 -union distinct -select id, string_col, year, month from functional.alltypes ----- RESULTS -'Estimated Per-Host Requirements: Memory=30.00MB VCores=3' -'' -'F05:PLAN FRAGMENT [HASH(year,month)]' -' WRITE TO HDFS [functional.alltypessmall, OVERWRITE=true, PARTITION-KEYS=(year,month)]' -' | partitions=96' -' | hosts=3 per-host-mem=69.26KB' -' |' -' 14:EXCHANGE [HASH(year,month)]' -' hosts=3 per-host-mem=0B' -' tuple-ids=3 row-size=28B cardinality=7600' -'' -'F04:PLAN FRAGMENT [HASH(id,string_col,year,month)]' -' DATASTREAM SINK [FRAGMENT=F05, EXCHANGE=14, HASH(year,month)]' -' 13:AGGREGATE [MERGE FINALIZE]' -' | group by: id, string_col, year, month' -' | hosts=3 per-host-mem=10.00MB' -' | tuple-ids=3 row-size=28B cardinality=7600' -' |' -' 12:EXCHANGE [HASH(id,string_col,year,month)]' -' hosts=3 per-host-mem=0B' -' tuple-ids=3 row-size=28B cardinality=7600' -'' -'F03:PLAN FRAGMENT [RANDOM]' -' DATASTREAM SINK [FRAGMENT=F04, EXCHANGE=12, HASH(id,string_col,year,month)]' -' 11:AGGREGATE' -' | group by: id, string_col, year, month' -' | hosts=3 per-host-mem=10.00MB' -' | tuple-ids=3 row-size=28B cardinality=7300' -' |' -' 10:MERGE' -' | hosts=3 per-host-mem=0B' -' | tuple-ids=3 row-size=28B cardinality=7300' -' |' -' 04:SCAN HDFS [functional.alltypes, RANDOM]' -' partitions=24/24 size=478.45KB' -' table stats: 7300 rows total' -' column stats: all' -' hosts=3 per-host-mem=16.00MB' -' tuple-ids=2 row-size=29B cardinality=7300' -'' -'F02:PLAN FRAGMENT [HASH(a.id)]' -' DATASTREAM SINK [FRAGMENT=F04, EXCHANGE=12, HASH(id,string_col,year,month)]' -' 05:AGGREGATE' -' | group by: id, string_col, year, month' -' | hosts=3 per-host-mem=10.00MB' -' | tuple-ids=3 row-size=28B cardinality=300' -' |' -' 09:MERGE' -' | hosts=3 per-host-mem=0B' -' | tuple-ids=3 row-size=28B cardinality=300' -' |' -' 03:HASH JOIN [LEFT SEMI JOIN, PARTITIONED]' -' | hash predicates: a.id = b.id' -' | hosts=3 per-host-mem=14.32KB' -' | tuple-ids=0,1 row-size=33B cardinality=300' -' |' -' |--07:EXCHANGE [HASH(b.id)]' -' | hosts=3 per-host-mem=0B' -' | tuple-ids=1 row-size=4B cardinality=10000' -' |' -' 06:EXCHANGE [HASH(a.id)]' -' hosts=3 per-host-mem=0B' -' tuple-ids=0 row-size=29B cardinality=300' -'' -'F01:PLAN FRAGMENT [RANDOM]' -' DATASTREAM SINK [FRAGMENT=F02, EXCHANGE=07, HASH(b.id)]' -' 02:SCAN HDFS [functional.alltypesagg b, RANDOM]' -' partitions=10/10 size=743.67KB' -' table stats: 10000 rows total' -' column stats: all' -' hosts=3 per-host-mem=16.00MB' -' tuple-ids=1 row-size=4B cardinality=10000' -'' -'F00:PLAN FRAGMENT [RANDOM]' -' DATASTREAM SINK [FRAGMENT=F02, EXCHANGE=06, HASH(a.id)]' -' 01:SCAN HDFS [functional.alltypes a, RANDOM]' -' partitions=1/24 size=19.71KB' -' table stats: 7300 rows total' -' column stats: all' -' hosts=3 per-host-mem=16.00MB' -' tuple-ids=0 row-size=29B cardinality=300' -==== ----- QUERY -# Tests explaining an insert query to/from an HBase table -explain insert into functional_hbase.alltypes -select a.* from functional_hbase.alltypessmall a - cross join functional.alltypessmall b -where a.year > 2009 and a.month = 4 -union all -select * from functional_hbase.alltypessmall ----- RESULTS -'Estimated Per-Host Requirements: Memory=1.02GB VCores=3' -'' -'F03:PLAN FRAGMENT [UNPARTITIONED]' -' WRITE TO HBASE table=functional_hbase.alltypes' -' | hosts=1 per-host-mem=unavailable' -' |' -' 06:EXCHANGE [UNPARTITIONED]' -' hosts=100 per-host-mem=unavailable' -' tuple-ids=3 row-size=88B cardinality=596' -'' -'F02:PLAN FRAGMENT [RANDOM]' -' DATASTREAM SINK [FRAGMENT=F03, EXCHANGE=06, UNPARTITIONED]' -' 08:MERGE' -' | hosts=100 per-host-mem=0B' -' | tuple-ids=3 row-size=88B cardinality=196' -' |' -' 04:SCAN HBASE [functional_hbase.alltypessmall]' -' table stats: 100 rows total' -' column stats: all' -' hosts=100 per-host-mem=1.00GB' -' tuple-ids=2 row-size=97B cardinality=196' -'' -'F00:PLAN FRAGMENT [RANDOM]' -' DATASTREAM SINK [FRAGMENT=F03, EXCHANGE=06, UNPARTITIONED]' -' 07:MERGE' -' | hosts=100 per-host-mem=0B' -' | tuple-ids=3 row-size=88B cardinality=400' -' |' -' 03:CROSS JOIN [BROADCAST]' -' | hosts=100 per-host-mem=0B' -' | tuple-ids=0,1 row-size=97B cardinality=400' -' |' -' |--05:EXCHANGE [BROADCAST]' -' | hosts=3 per-host-mem=0B' -' | tuple-ids=1 row-size=0B cardinality=100' -' |' -' 01:SCAN HBASE [functional_hbase.alltypessmall a]' -' predicates: a.year > 2009, a.month = 4' -' table stats: 100 rows total' -' column stats: all' -' hosts=100 per-host-mem=1.00GB' -' tuple-ids=0 row-size=97B cardinality=4' -'' -'F01:PLAN FRAGMENT [RANDOM]' -' DATASTREAM SINK [FRAGMENT=F00, EXCHANGE=05, BROADCAST]' -' 02:SCAN HDFS [functional.alltypessmall b, RANDOM]' -' partitions=4/4 size=6.32KB' -' table stats: 100 rows total' -' column stats: all' -' hosts=3 per-host-mem=16.00MB' -' tuple-ids=1 row-size=0B cardinality=100' -==== ----- QUERY -# Tests explaining an CTAS statement. -explain create table t as -select * from functional.alltypes -where month = 2 ----- RESULTS -'Estimated Per-Host Requirements: Memory=16.02MB VCores=1' -'' -'F00:PLAN FRAGMENT [RANDOM]' -' WRITE TO HDFS [functional.t, OVERWRITE=false]' -' | partitions=1' -' | hosts=3 per-host-mem=17.62KB' -' |' -' 00:SCAN HDFS [functional.alltypes, RANDOM]' -' partitions=2/24 size=36.51KB' -' table stats: 7300 rows total' -' column stats: all' -' hosts=3 per-host-mem=16.00MB' -' tuple-ids=1 row-size=97B cardinality=560' +select * +from tpch.lineitem join tpch.orders on l_orderkey = o_orderkey; +---- RESULTS: VERIFY_IS_EQUAL +'Per-Host Resource Reservation: Memory=136.00MB' +'Per-Host Resource Estimates: Memory=388.41MB' +'' +'F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1' +' PLAN-ROOT SINK' +' | mem-estimate=0B mem-reservation=0B' +' |' +' 04:EXCHANGE [UNPARTITIONED]' +' mem-estimate=0B mem-reservation=0B' +' tuple-ids=0,1 row-size=454B cardinality=5757710' +'' +'F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3' +' DATASTREAM SINK [FRAGMENT=F02, EXCHANGE=04, UNPARTITIONED]' +' | mem-estimate=0B mem-reservation=0B' +' 02:HASH JOIN [INNER JOIN, BROADCAST]' +' | hash predicates: l_orderkey = o_orderkey' +' | runtime filters: RF000 <- o_orderkey' +' | mem-estimate=300.41MB mem-reservation=136.00MB' +' | tuple-ids=0,1 row-size=454B cardinality=5757710' +' |' +' |--03:EXCHANGE [BROADCAST]' +' | mem-estimate=0B mem-reservation=0B' +' | tuple-ids=1 row-size=191B cardinality=1500000' +' |' +' 00:SCAN HDFS [tpch.lineitem, RANDOM]' +row_regex:.*partitions=1/1 files=1 size=.* +' runtime filters: RF000 -> l_orderkey' +' table stats: 6001215 rows total' +' column stats: all' +' mem-estimate=88.00MB mem-reservation=0B' +' tuple-ids=0 row-size=263B cardinality=6001215' +'' +'F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=2' +' DATASTREAM SINK [FRAGMENT=F00, EXCHANGE=03, BROADCAST]' +' | mem-estimate=0B mem-reservation=0B' +' 01:SCAN HDFS [tpch.orders, RANDOM]' +row_regex:.*partitions=1/1 files=1 size=.* +' table stats: 1500000 rows total' +' column stats: all' +' mem-estimate=88.00MB mem-reservation=0B' +' tuple-ids=1 row-size=191B cardinality=1500000' ==== ---- QUERY # Tests the warning about missing table stats in the explain header. @@ -335,70 +54,7 @@ from functional_avro.alltypes t1 inner join functional_parquet.alltypessmall t2 on (t1.id = t2.id) left outer join functional_avro.alltypes t3 on (t2.id = t3.id) where t1.month = 1 and t2.year = 2009 and t3.bool_col = false ----- RESULTS -'Estimated Per-Host Requirements: Memory=4.03GB VCores=3' +---- RESULTS: VERIFY_IS_SUBSET 'WARNING: The following tables are missing relevant table and/or column statistics.' 'functional_avro.alltypes, functional_parquet.alltypessmall' -'' -'F03:PLAN FRAGMENT [UNPARTITIONED]' -' 09:AGGREGATE [MERGE FINALIZE]' -' | output: sum(count(t1.int_col)), sum(sum(t2.float_col)), sum(count(t2.float_col)), sum(sum(t3.bigint_col))' -' | hosts=3 per-host-mem=unavailable' -' | tuple-ids=3 row-size=32B cardinality=1' -' |' -' 08:EXCHANGE [UNPARTITIONED]' -' hosts=3 per-host-mem=unavailable' -' tuple-ids=3 row-size=32B cardinality=1' -'' -'F00:PLAN FRAGMENT [RANDOM]' -' DATASTREAM SINK [FRAGMENT=F03, EXCHANGE=08, UNPARTITIONED]' -' 05:AGGREGATE' -' | output: count(t1.int_col), sum(t2.float_col), count(t2.float_col), sum(t3.bigint_col)' -' | hosts=3 per-host-mem=10.00MB' -' | tuple-ids=3 row-size=32B cardinality=1' -' |' -' 04:HASH JOIN [LEFT OUTER JOIN, BROADCAST]' -' | hash predicates: t2.id = t3.id' -' | other predicates: t3.bool_col = FALSE' -' | hosts=3 per-host-mem=2.00GB' -' | tuple-ids=0,1,2N row-size=37B cardinality=unavailable' -' |' -' |--07:EXCHANGE [BROADCAST]' -' | hosts=3 per-host-mem=0B' -' | tuple-ids=2 row-size=13B cardinality=unavailable' -' |' -' 03:HASH JOIN [INNER JOIN, BROADCAST]' -' | hash predicates: t1.id = t2.id' -' | hosts=3 per-host-mem=2.00GB' -' | tuple-ids=0,1 row-size=24B cardinality=unavailable' -' |' -' |--06:EXCHANGE [BROADCAST]' -' | hosts=3 per-host-mem=0B' -' | tuple-ids=1 row-size=12B cardinality=unavailable' -' |' -' 00:SCAN HDFS [functional_avro.alltypes t1, RANDOM]' -' partitions=2/24 size=39.87KB' -' table stats: unavailable' -' columns missing stats: id, int_col' -' hosts=3 per-host-mem=16.00MB' -' tuple-ids=0 row-size=12B cardinality=unavailable' -'' -'F02:PLAN FRAGMENT [RANDOM]' -' DATASTREAM SINK [FRAGMENT=F00, EXCHANGE=07, BROADCAST]' -' 02:SCAN HDFS [functional_avro.alltypes t3, RANDOM]' -' partitions=24/24 size=470.35KB' -' predicates: t3.bool_col = FALSE' -' table stats: unavailable' -' column stats: unavailable' -' hosts=3 per-host-mem=16.00MB' -' tuple-ids=2 row-size=13B cardinality=unavailable' -'' -'F01:PLAN FRAGMENT [RANDOM]' -' DATASTREAM SINK [FRAGMENT=F00, EXCHANGE=06, BROADCAST]' -' 01:SCAN HDFS [functional_parquet.alltypessmall t2, RANDOM]' -' partitions=4/4 size=9.63KB' -' table stats: unavailable' -' columns missing stats: id, float_col' -' hosts=3 per-host-mem=16.00MB' -' tuple-ids=1 row-size=12B cardinality=unavailable' ==== http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9a29dfc9/tests/custom_cluster/test_admission_controller.py ---------------------------------------------------------------------- diff --git a/tests/custom_cluster/test_admission_controller.py b/tests/custom_cluster/test_admission_controller.py index bf677d9..5201c75 100644 --- a/tests/custom_cluster/test_admission_controller.py +++ b/tests/custom_cluster/test_admission_controller.py @@ -296,7 +296,8 @@ class TestAdmissionController(TestAdmissionControllerBase, HS2TestSuite): @pytest.mark.execute_serially @CustomClusterTestSuite.with_args( - impalad_args=impalad_admission_ctrl_flags(1, 1, 1234, 1024 * 1024 * 1024), + impalad_args=impalad_admission_ctrl_flags(1, 1, 10 * 1024 * 1024, + 1024 * 1024 * 1024), statestored_args=_STATESTORED_ARGS) def test_trivial_coord_query_limits(self): """Tests that trivial coordinator only queries have negligible resource requirements. @@ -311,7 +312,8 @@ class TestAdmissionController(TestAdmissionControllerBase, HS2TestSuite): "select * from functional.alltypestiny"] for query in non_trivial_queries: ex = self.execute_query_expect_failure(self.client, query) - assert "memory needed 4.00 GB is greater than pool max mem resources" in str(ex) + assert re.search("Rejected query from pool default-pool : request memory needed " + ".* is greater than pool max mem resources 10.00 MB", str(ex)) class TestAdmissionControllerStress(TestAdmissionControllerBase): """Submits a number of queries (parameterized) with some delay between submissions http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/9a29dfc9/tests/metadata/test_explain.py ---------------------------------------------------------------------- diff --git a/tests/metadata/test_explain.py b/tests/metadata/test_explain.py index fd5b09a..3975e38 100644 --- a/tests/metadata/test_explain.py +++ b/tests/metadata/test_explain.py @@ -46,25 +46,29 @@ class TestExplain(ImpalaTestSuite): v.get_value('exec_option')['disable_codegen'] == False and\ v.get_value('exec_option')['num_nodes'] != 1) - @pytest.mark.xfail(run=False, reason="Expected per-host mem requirements inconsistent") + @pytest.mark.skip_if(pytest.config.option.testing_remote_cluster, + reason='Resource profile depends on number of nodes') def test_explain_level0(self, vector): vector.get_value('exec_option')['num_scanner_threads'] = self.NUM_SCANNER_THREADS vector.get_value('exec_option')['explain_level'] = 0 self.run_test_case('QueryTest/explain-level0', vector) - @pytest.mark.xfail(run=False, reason="Expected per-host mem requirements inconsistent") + @pytest.mark.skip_if(pytest.config.option.testing_remote_cluster, + reason='Resource profile depends on number of nodes') def test_explain_level1(self, vector): vector.get_value('exec_option')['num_scanner_threads'] = self.NUM_SCANNER_THREADS vector.get_value('exec_option')['explain_level'] = 1 self.run_test_case('QueryTest/explain-level1', vector) - @pytest.mark.xfail(run=False, reason="The test for missing table stats fails for avro") + @pytest.mark.skip_if(pytest.config.option.testing_remote_cluster, + reason='Resource profile depends on number of nodes') def test_explain_level2(self, vector): vector.get_value('exec_option')['num_scanner_threads'] = self.NUM_SCANNER_THREADS vector.get_value('exec_option')['explain_level'] = 2 self.run_test_case('QueryTest/explain-level2', vector) - @pytest.mark.xfail(run=False, reason="The test for missing table stats fails for avro") + @pytest.mark.skip_if(pytest.config.option.testing_remote_cluster, + reason='Resource profile depends on number of nodes') def test_explain_level3(self, vector): vector.get_value('exec_option')['num_scanner_threads'] = self.NUM_SCANNER_THREADS vector.get_value('exec_option')['explain_level'] = 3
