Michael Smith created IMPALA-15156:
--------------------------------------
Summary: Investigate averageColumnSizes impl
Key: IMPALA-15156
URL: https://issues.apache.org/jira/browse/IMPALA-15156
Project: IMPALA
Issue Type: Sub-task
Components: Frontend
Reporter: Michael Smith
IMPALA-13531 implements ImpalaRelMdSize#averageColumnSizes for RelSubset, and
handles non-CalciteTable instances. Without these, CTE rewrites fail with
{code:java}
E impala.error.HiveServer2Error: Query 11489c7bf994e9d8:e09e4e0200000000
failed:
E RuntimeException:
org.apache.calcite.plan.RelOptPlanner$CannotPlanException: There are not enough
rules to produce a node with desired properties: convention=NONE. All the
inputs have relevant nodes, however the cost is still infinite.
E Root: rel#305:RelSubset#6.NONE
E Original rel:
E LogicalProject(N_NAME=[$0], N_NAME0=[$2]): rowcount = 1.118033988749895,
cumulative cost = {2.2360679774997895E-6 cpu, 0.5004581690432799 io}, id = 274
E LogicalJoin(condition=[=($1, $3)], joinType=[inner]): rowcount =
1.118033988749895, cumulative cost = {2.2360679774997895E-6 cpu,
0.5004581690432799 io}, id = 273
E LogicalProject(N_NAME=[$1], $f2=[+(CAST($0):INTEGER, 1)]): rowcount =
1.118033988749895, cumulative cost = {0.0 cpu, 0.24929999828338623 io}, id = 269
E LogicalFilter(condition=[AND(=($2, 0), IS NOT NULL($0))]): rowcount =
1.118033988749895, cumulative cost = {0.0 cpu, 0.24929999828338623 io}, id = 268
E LogicalProject(n_nationkey=[$0], n_name=[$1], n_regionkey=[$2]):
rowcount = 25.0, cumulative cost = {0.0 cpu, 0.24929999828338623 io}, id = 267
E LogicalTableScan(table=[[tpch, nation]]): rowcount = 25.0,
cumulative cost = {0.0 cpu, 0.24929999828338623 io}, id = 1
E LogicalProject(N_NAME=[$1], N_NATIONKEY0=[CAST($0):INTEGER]): rowcount
= 1.118033988749895, cumulative cost = {0.0 cpu, 0.24929999828338623 io}, id =
272
E LogicalFilter(condition=[AND(=($2, 0), IS NOT NULL($0))]): rowcount =
1.118033988749895, cumulative cost = {0.0 cpu, 0.24929999828338623 io}, id = 271
E LogicalProject(n_nationkey=[$0], n_name=[$1], n_regionkey=[$2]):
rowcount = 25.0, cumulative cost = {0.0 cpu, 0.24929999828338623 io}, id = 270
E LogicalTableScan(table=[[tpch, nation]]): rowcount = 25.0,
cumulative cost = {0.0 cpu, 0.24929999828338623 io}, id = 1
E
E Sets:
E Set#0, type: RecordType(SMALLINT n_nationkey, VARCHAR(2147483647) n_name,
SMALLINT n_regionkey, VARCHAR(2147483647) n_comment)
E rel#291:RelSubset#0.NONE, best=rel#1
E rel#1:LogicalTableScan.NONE(table=[tpch, nation]),
rowcount=25.0, cumulative cost={0.0 cpu, 1.9205999708175658 io}
E Set#1, type: RecordType(SMALLINT n_nationkey, VARCHAR(2147483647) n_name,
SMALLINT n_regionkey)
E rel#293:RelSubset#1.NONE, best=rel#292
E rel#292:LogicalProject.NONE(input=RelSubset#291,inputs=0..2),
rowcount=25.0, cumulative cost={1.0 cpu, 1.9205999708175658 io}
E Set#2, type: RecordType(SMALLINT n_nationkey, VARCHAR(2147483647) n_name,
SMALLINT n_regionkey)
E rel#295:RelSubset#2.NONE, best=rel#290
E
rel#294:LogicalFilter.NONE(input=RelSubset#293,condition=AND(=($2, 0), IS NOT
NULL($0))), rowcount=1.118033988749895, cumulative cost={2.0 cpu,
1.9205999708175658 io}
E rel#290:ImpalaCTEConsumer.NONE(cteName=cte_suggestion_0),
rowcount=1.118033988749895, cumulative cost={tiny}
E Set#3, type: RecordType(VARCHAR(2147483647) N_NAME, INTEGER $f2)
E rel#297:RelSubset#3.NONE, best=rel#296
E rel#296:LogicalProject.NONE(input=RelSubset#295,exprs=[$1,
+(CAST($0):INTEGER, 1)]), rowcount=1.118033988749895, cumulative cost={2.0 cpu,
0.0 io}
E Set#4, type: RecordType(VARCHAR(2147483647) N_NAME, INTEGER N_NATIONKEY0)
E rel#301:RelSubset#4.NONE, best=rel#300
E rel#300:LogicalProject.NONE(input=RelSubset#295,exprs=[$1,
CAST($0):INTEGER]), rowcount=1.118033988749895, cumulative cost={2.0 cpu, 0.0
io}
E Set#5, type: RecordType(VARCHAR(2147483647) N_NAME, INTEGER $f2,
VARCHAR(2147483647) N_NAME0, INTEGER N_NATIONKEY0)
E rel#303:RelSubset#5.NONE, best=null
E
rel#302:LogicalJoin.NONE(left=RelSubset#297,right=RelSubset#301,condition==($1,
$3),joinType=inner), rowcount=1.118033988749895, cumulative cost=null
E Set#6, type: RecordType(VARCHAR(2147483647) N_NAME, VARCHAR(2147483647)
N_NAME0)
E rel#305:RelSubset#6.NONE, best=null
E
rel#304:LogicalProject.NONE(input=RelSubset#303,inputs=0,exprs=[$2]),
rowcount=1.118033988749895, cumulative cost={inf}
E
E Graphviz:
E digraph G {
E root [style=filled,label="Root"];
E subgraph cluster0{
E label="Set 0 RecordType(SMALLINT n_nationkey,
VARCHAR(2147483647) n_name, SMALLINT n_regionkey, VARCHAR(2147483647)
n_comment)";
E rel1 [label="rel#1:LogicalTableScan\ntable=[tpch,
nation]\nrows=25.0, cost={0.0 cpu, 1.9205999708175658 io}",color=blue,shape=box]
E subset291 [label="rel#291:RelSubset#0.NONE"]
E }
E subgraph cluster1{
E label="Set 1 RecordType(SMALLINT n_nationkey,
VARCHAR(2147483647) n_name, SMALLINT n_regionkey)";
E rel292
[label="rel#292:LogicalProject\ninput=RelSubset#291,inputs=0..2\nrows=25.0,
cost={1.0 cpu, 1.9205999708175658 io}",color=blue,shape=box]
E subset293 [label="rel#293:RelSubset#1.NONE"]
E }
E subgraph cluster2{
E label="Set 2 RecordType(SMALLINT n_nationkey,
VARCHAR(2147483647) n_name, SMALLINT n_regionkey)";
E rel294
[label="rel#294:LogicalFilter\ninput=RelSubset#293,condition=AND(=($2, 0), IS
NOT NULL($0))\nrows=1.118033988749895, cost={2.0 cpu, 1.9205999708175658
io}",shape=box]
E rel290
[label="rel#290:ImpalaCTEConsumer\ncteName=cte_suggestion_0\nrows=1.118033988749895,
cost={tiny}",color=blue,shape=box]
E subset295 [label="rel#295:RelSubset#2.NONE"]
E }
E subgraph cluster3{
E label="Set 3 RecordType(VARCHAR(2147483647) N_NAME, INTEGER
$f2)";
E rel296
[label="rel#296:LogicalProject\ninput=RelSubset#295,exprs=[$1,
+(CAST($0):INTEGER, 1)]\nrows=1.118033988749895, cost={2.0 cpu, 0.0
io}",color=blue,shape=box]
E subset297 [label="rel#297:RelSubset#3.NONE"]
E }
E subgraph cluster4{
E label="Set 4 RecordType(VARCHAR(2147483647) N_NAME, INTEGER
N_NATIONKEY0)";
E rel300
[label="rel#300:LogicalProject\ninput=RelSubset#295,exprs=[$1,
CAST($0):INTEGER]\nrows=1.118033988749895, cost={2.0 cpu, 0.0
io}",color=blue,shape=box]
E subset301 [label="rel#301:RelSubset#4.NONE"]
E }
E subgraph cluster5{
E label="Set 5 RecordType(VARCHAR(2147483647) N_NAME, INTEGER
$f2, VARCHAR(2147483647) N_NAME0, INTEGER N_NATIONKEY0)";
E rel302
[label="rel#302:LogicalJoin\nleft=RelSubset#297,right=RelSubset#301,condition==($1,
$3),joinType=inner\nrows=1.118033988749895, cost=null",shape=box]
E subset303 [label="rel#303:RelSubset#5.NONE"]
E }
E subgraph cluster6{
E label="Set 6 RecordType(VARCHAR(2147483647) N_NAME,
VARCHAR(2147483647) N_NAME0)";
E rel304
[label="rel#304:LogicalProject\ninput=RelSubset#303,inputs=0,exprs=[$2]\nrows=1.118033988749895,
cost={inf}",shape=box]
E subset305 [label="rel#305:RelSubset#6.NONE"]
E }
E root -> subset305;
E subset291 -> rel1[color=blue];
E subset293 -> rel292[color=blue]; rel292 -> subset291[color=blue];
E subset295 -> rel294; rel294 -> subset293;
E subset295 -> rel290[color=blue];
E subset297 -> rel296[color=blue]; rel296 -> subset295[color=blue];
E subset301 -> rel300[color=blue]; rel300 -> subset295[color=blue];
E subset303 -> rel302; rel302 -> subset297[label="0"]; rel302 ->
subset301[label="1"];
E subset305 -> rel304; rel304 -> subset303;
E }
E CAUSED BY: CannotPlanException: There are not enough rules to produce a
node with desired properties: convention=NONE. All the inputs have relevant
nodes, however the cost is still infinite.
E Root: rel#305:RelSubset#6.NONE
E Original rel:
E LogicalProject(N_NAME=[$0], N_NAME0=[$2]): rowcount = 1.118033988749895,
cumulative cost = {2.2360679774997895E-6 cpu, 0.5004581690432799 io}, id = 274
E LogicalJoin(condition=[=($1, $3)], joinType=[inner]): rowcount =
1.118033988749895, cumulative cost = {2.2360679774997895E-6 cpu,
0.5004581690432799 io}, id = 273
E LogicalProject(N_NAME=[$1], $f2=[+(CAST($0):INTEGER, 1)]): rowcount =
1.118033988749895, cumulative cost = {0.0 cpu, 0.24929999828338623 io}, id = 269
E LogicalFilter(condition=[AND(=($2, 0), IS NOT NULL($0))]): rowcount =
1.118033988749895, cumulative cost = {0.0 cpu, 0.24929999828338623 io}, id = 268
E LogicalProject(n_nationkey=[$0], n_name=[$1], n_regionkey=[$2]):
rowcount = 25.0, cumulative cost = {0.0 cpu, 0.24929999828338623 io}, id = 267
E LogicalTableScan(table=[[tpch, nation]]): rowcount = 25.0,
cumulative cost = {0.0 cpu, 0.24929999828338623 io}, id = 1
E LogicalProject(N_NAME=[$1], N_NATIONKEY0=[CAST($0):INTEGER]): rowcount
= 1.118033988749895, cumulative cost = {0.0 cpu, 0.24929999828338623 io}, id =
272
E LogicalFilter(condition=[AND(=($2, 0), IS NOT NULL($0))]): rowcount =
1.118033988749895, cumulative cost = {0.0 cpu, 0.24929999828338623 io}, id = 271
E LogicalProject(n_nationkey=[$0], n_name=[$1], n_regionkey=[$2]):
rowcount = 25.0, cumulative cost = {0.0 cpu, 0.24929999828338623 io}, id = 270
E LogicalTableScan(table=[[tpch, nation]]): rowcount = 25.0,
cumulative cost = {0.0 cpu, 0.24929999828338623 io}, id = 1
E
E Sets:
E Set#0, type: RecordType(SMALLINT n_nationkey, VARCHAR(2147483647) n_name,
SMALLINT n_regionkey, VARCHAR(2147483647) n_comment)
E rel#291:RelSubset#0.NONE, best=rel#1
E rel#1:LogicalTableScan.NONE(table=[tpch, nation]),
rowcount=25.0, cumulative cost={0.0 cpu, 1.9205999708175658 io}
E Set#1, type: RecordType(SMALLINT n_nationkey, VARCHAR(2147483647) n_name,
SMALLINT n_regionkey)
E rel#293:RelSubset#1.NONE, best=rel#292
E rel#292:LogicalProject.NONE(input=RelSubset#291,inputs=0..2),
rowcount=25.0, cumulative cost={1.0 cpu, 1.9205999708175658 io}
E Set#2, type: RecordType(SMALLINT n_nationkey, VARCHAR(2147483647) n_name,
SMALLINT n_regionkey)
E rel#295:RelSubset#2.NONE, best=rel#290
E
rel#294:LogicalFilter.NONE(input=RelSubset#293,condition=AND(=($2, 0), IS NOT
NULL($0))), rowcount=1.118033988749895, cumulative cost={2.0 cpu,
1.9205999708175658 io}
E rel#290:ImpalaCTEConsumer.NONE(cteName=cte_suggestion_0),
rowcount=1.118033988749895, cumulative cost={tiny}
E Set#3, type: RecordType(VARCHAR(2147483647) N_NAME, INTEGER $f2)
E rel#297:RelSubset#3.NONE, best=rel#296
E rel#296:LogicalProject.NONE(input=RelSubset#295,exprs=[$1,
+(CAST($0):INTEGER, 1)]), rowcount=1.118033988749895, cumulative cost={2.0 cpu,
0.0 io}
E Set#4, type: RecordType(VARCHAR(2147483647) N_NAME, INTEGER N_NATIONKEY0)
E rel#301:RelSubset#4.NONE, best=rel#300
E rel#300:LogicalProject.NONE(input=RelSubset#295,exprs=[$1,
CAST($0):INTEGER]), rowcount=1.118033988749895, cumulative cost={2.0 cpu, 0.0
io}
E Set#5, type: RecordType(VARCHAR(2147483647) N_NAME, INTEGER $f2,
VARCHAR(2147483647) N_NAME0, INTEGER N_NATIONKEY0)
E rel#303:RelSubset#5.NONE, best=null
E
rel#302:LogicalJoin.NONE(left=RelSubset#297,right=RelSubset#301,condition==($1,
$3),joinType=inner), rowcount=1.118033988749895, cumulative cost=null
E Set#6, type: RecordType(VARCHAR(2147483647) N_NAME, VARCHAR(2147483647)
N_NAME0)
E rel#305:RelSubset#6.NONE, best=null
E
rel#304:LogicalProject.NONE(input=RelSubset#303,inputs=0,exprs=[$2]),
rowcount=1.118033988749895, cumulative cost={inf}
E
E Graphviz:
E digraph G {
E root [style=filled,label="Root"];
E subgraph cluster0{
E label="Set 0 RecordType(SMALLINT n_nationkey,
VARCHAR(2147483647) n_name, SMALLINT n_regionkey, VARCHAR(2147483647)
n_comment)";
E rel1 [label="rel#1:LogicalTableScan\ntable=[tpch,
nation]\nrows=25.0, cost={0.0 cpu, 1.9205999708175658 io}",color=blue,shape=box]
E subset291 [label="rel#291:RelSubset#0.NONE"]
E }
E subgraph cluster1{
E label="Set 1 RecordType(SMALLINT n_nationkey,
VARCHAR(2147483647) n_name, SMALLINT n_regionkey)";
E rel292
[label="rel#292:LogicalProject\ninput=RelSubset#291,inputs=0..2\nrows=25.0,
cost={1.0 cpu, 1.9205999708175658 io}",color=blue,shape=box]
E subset293 [label="rel#293:RelSubset#1.NONE"]
E }
E subgraph cluster2{
E label="Set 2 RecordType(SMALLINT n_nationkey,
VARCHAR(2147483647) n_name, SMALLINT n_regionkey)";
E rel294
[label="rel#294:LogicalFilter\ninput=RelSubset#293,condition=AND(=($2, 0), IS
NOT NULL($0))\nrows=1.118033988749895, cost={2.0 cpu, 1.9205999708175658
io}",shape=box]
E rel290
[label="rel#290:ImpalaCTEConsumer\ncteName=cte_suggestion_0\nrows=1.118033988749895,
cost={tiny}",color=blue,shape=box]
E subset295 [label="rel#295:RelSubset#2.NONE"]
E }
E subgraph cluster3{
E label="Set 3 RecordType(VARCHAR(2147483647) N_NAME, INTEGER
$f2)";
E rel296
[label="rel#296:LogicalProject\ninput=RelSubset#295,exprs=[$1,
+(CAST($0):INTEGER, 1)]\nrows=1.118033988749895, cost={2.0 cpu, 0.0
io}",color=blue,shape=box]
E subset297 [label="rel#297:RelSubset#3.NONE"]
E }
E subgraph cluster4{
E label="Set 4 RecordType(VARCHAR(2147483647) N_NAME, INTEGER
N_NATIONKEY0)";
E rel300
[label="rel#300:LogicalProject\ninput=RelSubset#295,exprs=[$1,
CAST($0):INTEGER]\nrows=1.118033988749895, cost={2.0 cpu, 0.0
io}",color=blue,shape=box]
E subset301 [label="rel#301:RelSubset#4.NONE"]
E }
E subgraph cluster5{
E label="Set 5 RecordType(VARCHAR(2147483647) N_NAME, INTEGER
$f2, VARCHAR(2147483647) N_NAME0, INTEGER N_NATIONKEY0)";
E rel302
[label="rel#302:LogicalJoin\nleft=RelSubset#297,right=RelSubset#301,condition==($1,
$3),joinType=inner\nrows=1.118033988749895, cost=null",shape=box]
E subset303 [label="rel#303:RelSubset#5.NONE"]
E }
E subgraph cluster6{
E label="Set 6 RecordType(VARCHAR(2147483647) N_NAME,
VARCHAR(2147483647) N_NAME0)";
E rel304
[label="rel#304:LogicalProject\ninput=RelSubset#303,inputs=0,exprs=[$2]\nrows=1.118033988749895,
cost={inf}",shape=box]
E subset305 [label="rel#305:RelSubset#6.NONE"]
E }
E root -> subset305;
E subset291 -> rel1[color=blue];
E subset293 -> rel292[color=blue]; rel292 -> subset291[color=blue];
E subset295 -> rel294; rel294 -> subset293;
E subset295 -> rel290[color=blue];
E subset297 -> rel296[color=blue]; rel296 -> subset295[color=blue];
E subset301 -> rel300[color=blue]; rel300 -> subset295[color=blue];
E subset303 -> rel302; rel302 -> subset297[label="0"]; rel302 ->
subset301[label="1"];
E subset305 -> rel304; rel304 -> subset303;
E }
{code}
The cause is not entirely clear to me, but it seems to happen while producing
costs for different possible materializations.
Also review other costing related to CTE materialization.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
