soumyakanti3578 commented on PR #5798:
URL: https://github.com/apache/hive/pull/5798#issuecomment-2846111554
This is indeed a wider issue. All ICEBERG tables were classified as NATIVE
tables in `CalcitePlanner::genTableLogicalPlan`, however, the side effect was
only visible because the lateral view references all virtual columns too, as
shown in the Jira. This is also visible when we do an `explain formatted select
* from <table>` as shown below by running:
```
create external table test (a int, b int, c int, d struct<e:string, f:int>)
stored by iceberg;
explain formatted select * from test;
```
We see NATIVE and NON NATIVE virtual columns in the output:
```
{"CBOPlan":"{
\"rels\": [
{
\"id\": \"0\",
\"relOp\":
\"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",
\"table\": [
\"default\",
\"test\"
],
\"table:alias\": \"test\",
\"inputs\": [],
\"rowCount\": 1.0,
\"avgRowSize\": 248.0,
\"rowType\": {
\"fields\": [
{
\"type\": \"INTEGER\",
\"nullable\": true,
\"name\": \"a\"
},
{
\"type\": \"INTEGER\",
\"nullable\": true,
\"name\": \"b\"
},
{
\"type\": \"INTEGER\",
\"nullable\": true,
\"name\": \"c\"
},
{
\"fields\": {
\"fields\": [
{
\"type\": \"VARCHAR\",
\"nullable\": true,
\"precision\": 2147483647,
\"name\": \"e\"
},
{
\"type\": \"INTEGER\",
\"nullable\": true,
\"name\": \"f\"
}
],
\"nullable\": true
},
\"nullable\": true,
\"name\": \"d\"
},
{
\"type\": \"BIGINT\",
\"nullable\": true,
\"name\": \"BLOCK__OFFSET__INSIDE__FILE\"
},
{
\"type\": \"VARCHAR\",
\"nullable\": true,
\"precision\": 2147483647,
\"name\": \"INPUT__FILE__NAME\"
},
{
\"fields\": {
\"fields\": [
{
\"type\": \"BIGINT\",
\"nullable\": true,
\"name\": \"writeid\"
},
{
\"type\": \"INTEGER\",
\"nullable\": true,
\"name\": \"bucketid\"
},
{
\"type\": \"BIGINT\",
\"nullable\": true,
\"name\": \"rowid\"
}
],
\"nullable\": true
},
\"nullable\": true,
\"name\": \"ROW__ID\"
},
{
\"type\": \"BOOLEAN\",
\"nullable\": true,
\"name\": \"ROW__IS__DELETED\"
},
{
\"type\": \"INTEGER\",
\"nullable\": true,
\"name\": \"PARTITION__SPEC__ID\"
},
{
\"type\": \"BIGINT\",
\"nullable\": true,
\"name\": \"PARTITION__HASH\"
},
{
\"type\": \"VARCHAR\",
\"nullable\": true,
\"precision\": 2147483647,
\"name\": \"FILE__PATH\"
},
{
\"type\": \"BIGINT\",
\"nullable\": true,
\"name\": \"ROW__POSITION\"
},
{
\"type\": \"VARCHAR\",
\"nullable\": true,
\"precision\": 2147483647,
\"name\": \"PARTITION__PROJECTION\"
},
{
\"type\": \"BIGINT\",
\"nullable\": true,
\"name\": \"SNAPSHOT__ID\"
}
],
\"nullable\": false
},
\"colStats\": [
{
\"name\": \"a\",
\"ndv\": 1,
\"minValue\": -9223372036854775808,
\"maxValue\": 9223372036854775807
},
{
\"name\": \"b\",
\"ndv\": 1,
\"minValue\": -9223372036854775808,
\"maxValue\": 9223372036854775807
},
{
\"name\": \"c\",
\"ndv\": 1,
\"minValue\": -9223372036854775808,
\"maxValue\": 9223372036854775807
},
{
\"name\": \"d\",
\"ndv\": 1
}
]
},
{
\"id\": \"1\",
\"relOp\":
\"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",
\"fields\": [
\"a\",
\"b\",
\"c\",
\"d\"
],
\"exprs\": [
{
\"input\": 0,
\"name\": \"$0\"
},
{
\"input\": 1,
\"name\": \"$1\"
},
{
\"input\": 2,
\"name\": \"$2\"
},
{
\"input\": 3,
\"name\": \"$3\"
}
],
\"rowCount\": 1.0
}
]
}","optimizedSQL":"SELECT `a`, `b`, `c`, `d`
FROM `default`.`test`","cboInfo":"Plan optimized by CBO.","STAGE
DEPENDENCIES":{"Stage-0":{"ROOT STAGE":"TRUE"}},"STAGE
PLANS":{"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor
Tree:":{"TableScan":{"alias:":"test","columns:":["a","b","c","d"],"database:":"default","table:":"test","isTempTable:":"false","OperatorId:":"TS_0","children":{"Select
Operator":{"expressions:":"a (type: int), b (type: int), c (type: int), d
(type:
struct<e:string,f:int>)","columnExprMap:":{"_col0":"a","_col1":"b","_col2":"c","_col3":"d"},"outputColumnNames:":["_col0","_col1","_col2","_col3"],"OperatorId:":"SEL_1","children":{"ListSink":{"OperatorId:":"LIST_SINK_3"}}}}}}}}}}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
