This is an automated email from the ASF dual-hosted git repository.
gsaihemanth pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new e39f67a958f HIVE-28511: add column access information to
HivePrivilegeObject (#5440) (Jintong Jiang, Reviewed by Dmitriy Fingerman,
Soumyakanti Das, Sai Hemanth Gantasala)
e39f67a958f is described below
commit e39f67a958f882a6d4e5e2bac165d6b83bc1640f
Author: Hazel Jiang <107082535+jjiang...@users.noreply.github.com>
AuthorDate: Wed Sep 18 10:51:01 2024 -0700
HIVE-28511: add column access information to HivePrivilegeObject (#5440)
(Jintong Jiang, Reviewed by Dmitriy Fingerman, Soumyakanti Das, Sai Hemanth
Gantasala)
---
.../negative/write_iceberg_branch_negative.q.out | 2 +-
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 20 +-
.../clientpositive/authorization_insert_column.q | 67 +++
.../llap/authorization_insert_column.q.out | 633 +++++++++++++++++++++
4 files changed, 718 insertions(+), 4 deletions(-)
diff --git
a/iceberg/iceberg-handler/src/test/results/negative/write_iceberg_branch_negative.q.out
b/iceberg/iceberg-handler/src/test/results/negative/write_iceberg_branch_negative.q.out
index f24ab56e08c..07a8e5baa83 100644
---
a/iceberg/iceberg-handler/src/test/results/negative/write_iceberg_branch_negative.q.out
+++
b/iceberg/iceberg-handler/src/test/results/negative/write_iceberg_branch_negative.q.out
@@ -6,4 +6,4 @@ POSTHOOK: query: create external table ice01(a int, b string, c
int) stored by i
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@ice01
-FAILED: SemanticException Cannot use snapshotRef (does not exist): test1
+FAILED: SemanticException org.apache.hadoop.hive.ql.parse.SemanticException:
Cannot use snapshotRef (does not exist): test1
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index a966ec8d0d9..4f0decbcf54 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -2143,12 +2143,15 @@ public class SemanticAnalyzer extends
BaseSemanticAnalyzer {
*/
private void handleInsertStatementSpecPhase1(ASTNode ast, QBParseInfo qbp,
Phase1Ctx ctx_1) throws SemanticException {
ASTNode tabColName = (ASTNode)ast.getChild(1);
- if(ast.getType() == HiveParser.TOK_INSERT_INTO && tabColName != null &&
tabColName.getType() == HiveParser.TOK_TABCOLNAME) {
+ boolean hasSpecificColumns = tabColName != null && tabColName.getType() ==
HiveParser.TOK_TABCOLNAME;
+ if(ast.getType() == HiveParser.TOK_INSERT_INTO) {
//we have "insert into foo(a,b)..."; parser will enforce that 1+ columns
are listed if TOK_TABCOLNAME is present
String fullTableName = getUnescapedName((ASTNode)
ast.getChild(0).getChild(0),
SessionState.get().getCurrentDatabase());
- List<String> targetColumnNames = processTableColumnNames(tabColName,
fullTableName);
- qbp.setDestSchemaForClause(ctx_1.dest, targetColumnNames);
+ List<String> targetColumnNames = hasSpecificColumns ?
processTableColumnNames(tabColName, fullTableName) : new ArrayList<>();
+ if (hasSpecificColumns) {
+ qbp.setDestSchemaForClause(ctx_1.dest, targetColumnNames);
+ }
Table targetTable;
try {
targetTable = getTableObjectByName(fullTableName);
@@ -2160,6 +2163,17 @@ public class SemanticAnalyzer extends
BaseSemanticAnalyzer {
throw new SemanticException(generateErrorMessage(ast,
"Unable to access metadata for table " + fullTableName));
}
+ ColumnAccessInfo cai = new ColumnAccessInfo();
+ if (!hasSpecificColumns) {
+ for (FieldSchema col : targetTable.getCols()) {
+ targetColumnNames.add(col.getName());
+ }
+ }
+ String completeTableName = targetTable.getCompleteName();
+ for (String colName : targetColumnNames) {
+ cai.add(completeTableName, colName);
+ }
+ setUpdateColumnAccessInfo(cai);
Set<String> targetColumns = new HashSet<>(targetColumnNames);
for(FieldSchema f : targetTable.getCols()) {
//parser only allows foo(a,b), not foo(foo.a, foo.b)
diff --git a/ql/src/test/queries/clientpositive/authorization_insert_column.q
b/ql/src/test/queries/clientpositive/authorization_insert_column.q
new file mode 100644
index 00000000000..590487345f3
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/authorization_insert_column.q
@@ -0,0 +1,67 @@
+set
hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest;
+set test.hive.authz.sstd.validator.outputPrivObjs=true;
+set test.hive.authz.sstd.validator.bypassObjTypes=DATABASE;
+set hive.test.authz.sstd.hs2.mode=true;
+set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
+set hive.support.concurrency=true;
+set user.name=testuser;
+
+
+CREATE DATABASE IF NOT EXISTS test_insert_col_db;
+USE test_insert_col_db;
+
+-- Negative Case 1: Basic Table with Restricted Column
+CREATE TABLE test_basic_table (
+ id INT,
+ name STRING,
+ age INT
+);
+
+SHOW DATABASES LIKE 'test_insert_col_db';
+SHOW TABLES IN test_insert_col_db;
+EXPLAIN INSERT INTO test_insert_col_db.test_basic_table (id, name, age) VALUES
(1, 'Dummy', 24);
+DROP TABLE test_basic_table;
+
+-- Negative Case 2: Partitioned Table with Restricted Column
+CREATE TABLE test_partitioned_table (
+ id INT,
+ name STRING,
+ age INT
+)
+PARTITIONED BY (year INT);
+
+ALTER TABLE test_partitioned_table ADD PARTITION (year=2024);
+
+SHOW DATABASES LIKE 'test_insert_col_db';
+SHOW TABLES IN test_insert_col_db;
+EXPLAIN INSERT INTO test_insert_col_db.test_partitioned_table (id, name, age)
VALUES (2, 'Dummy2', 25);
+DROP TABLE test_partitioned_table;
+
+-- Negative Case 3: ACID Table with Restricted Column
+CREATE TABLE test_acid_table (
+ id INT,
+ name STRING,
+ age INT
+)
+STORED AS ORC
+TBLPROPERTIES ('transactional'='true');
+
+
+SHOW DATABASES LIKE 'test_insert_col_db';
+SHOW TABLES IN test_insert_col_db;
+EXPLAIN INSERT INTO test_insert_col_db.test_acid_table (id, name, age) VALUES
(3, 'Dummy3', 26);
+DROP TABLE test_acid_table;
+
+-- Negative Case 4: Insert Without Specifying Column Names (Restricted Column)
+CREATE TABLE test_restricted_column_table (
+ id INT,
+ name STRING,
+ age INT
+);
+
+SHOW DATABASES LIKE 'test_insert_col_db';
+SHOW TABLES IN test_insert_col_db;
+EXPLAIN INSERT INTO test_insert_col_db.test_restricted_column_table VALUES (8,
'Dummy4', 55);
+DROP TABLE test_restricted_column_table;
+
+DROP DATABASE test_insert_col_db;
diff --git
a/ql/src/test/results/clientpositive/llap/authorization_insert_column.q.out
b/ql/src/test/results/clientpositive/llap/authorization_insert_column.q.out
new file mode 100644
index 00000000000..d5f6175205d
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/authorization_insert_column.q.out
@@ -0,0 +1,633 @@
+PREHOOK: query: CREATE DATABASE IF NOT EXISTS test_insert_col_db
+PREHOOK: type: CREATEDATABASE
+PREHOOK: Output: database:test_insert_col_db
+POSTHOOK: query: CREATE DATABASE IF NOT EXISTS test_insert_col_db
+POSTHOOK: type: CREATEDATABASE
+POSTHOOK: Output: database:test_insert_col_db
+PREHOOK: query: USE test_insert_col_db
+PREHOOK: type: SWITCHDATABASE
+PREHOOK: Input: database:test_insert_col_db
+POSTHOOK: query: USE test_insert_col_db
+POSTHOOK: type: SWITCHDATABASE
+POSTHOOK: Input: database:test_insert_col_db
+PREHOOK: query: CREATE TABLE test_basic_table (
+ id INT,
+ name STRING,
+ age INT
+)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:test_insert_col_db
+PREHOOK: Output: test_insert_col_db@test_basic_table
+POSTHOOK: query: CREATE TABLE test_basic_table (
+ id INT,
+ name STRING,
+ age INT
+)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:test_insert_col_db
+POSTHOOK: Output: test_insert_col_db@test_basic_table
+PREHOOK: query: SHOW DATABASES LIKE 'test_insert_col_db'
+PREHOOK: type: SHOWDATABASES
+filterListCmdObjects
+HIVE PRIVILEGE OBJECT { objectName: default type: DATABASE actionType: OTHER
dbName: default}
+HIVE PRIVILEGE OBJECT { objectName: test_insert_col_db type: DATABASE
actionType: OTHER dbName: test_insert_col_db}
+POSTHOOK: query: SHOW DATABASES LIKE 'test_insert_col_db'
+POSTHOOK: type: SHOWDATABASES
+test_insert_col_db
+PREHOOK: query: SHOW TABLES IN test_insert_col_db
+PREHOOK: type: SHOWTABLES
+PREHOOK: Input: database:test_insert_col_db
+filterListCmdObjects
+HIVE PRIVILEGE OBJECT { objectName: test_basic_table type: TABLE_OR_VIEW
actionType: OTHER dbName: test_insert_col_db}
+POSTHOOK: query: SHOW TABLES IN test_insert_col_db
+POSTHOOK: type: SHOWTABLES
+POSTHOOK: Input: database:test_insert_col_db
+test_basic_table
+PREHOOK: query: EXPLAIN INSERT INTO test_insert_col_db.test_basic_table (id,
name, age) VALUES (1, 'Dummy', 24)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: test_insert_col_db@test_basic_table
+POSTHOOK: query: EXPLAIN INSERT INTO test_insert_col_db.test_basic_table (id,
name, age) VALUES (1, 'Dummy', 24)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: test_insert_col_db@test_basic_table
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: _dummy_table
+ Row Limit Per Split: 1
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE
Column stats: COMPLETE
+ Select Operator
+ expressions: array(const struct(1,'Dummy',24)) (type:
array<struct<col1:int,col2:string,col3:int>>)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 48 Basic stats:
COMPLETE Column stats: COMPLETE
+ UDTF Operator
+ Statistics: Num rows: 1 Data size: 48 Basic stats:
COMPLETE Column stats: COMPLETE
+ function name: inline
+ Select Operator
+ expressions: col1 (type: int), col2 (type: string),
col3 (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 8 Basic stats:
COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats:
COMPLETE Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.mapred.TextInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: test_insert_col_db.test_basic_table
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type:
string), _col2 (type: int)
+ outputColumnNames: id, name, age
+ Statistics: Num rows: 1 Data size: 8 Basic stats:
COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: min(id), max(id), count(1),
count(id), compute_bit_vector_hll(id), max(length(name)),
avg(COALESCE(length(name),0)), count(name), compute_bit_vector_hll(name),
min(age), max(age), count(age), compute_bit_vector_hll(age)
+ minReductionHashAggr: 0.4
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2, _col3,
_col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12
+ Statistics: Num rows: 1 Data size: 560 Basic
stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ null sort order:
+ sort order:
+ Statistics: Num rows: 1 Data size: 560 Basic
stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: int), _col1
(type: int), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary),
_col5 (type: int), _col6 (type: struct<count:bigint,sum:double,input:int>),
_col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type:
int), _col11 (type: bigint), _col12 (type: binary)
+ Execution mode: llap
+ LLAP IO: no inputs
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: min(VALUE._col0), max(VALUE._col1),
count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4),
max(VALUE._col5), avg(VALUE._col6), count(VALUE._col7),
compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10),
count(VALUE._col11), compute_bit_vector_hll(VALUE._col12)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5,
_col6, _col7, _col8, _col9, _col10, _col11, _col12
+ Statistics: Num rows: 1 Data size: 492 Basic stats: COMPLETE
Column stats: COMPLETE
+ Select Operator
+ expressions: 'LONG' (type: string), UDFToLong(_col0) (type:
bigint), UDFToLong(_col1) (type: bigint), (_col2 - _col3) (type: bigint),
COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary),
'STRING' (type: string), UDFToLong(COALESCE(_col5,0)) (type: bigint),
COALESCE(_col6,0) (type: double), (_col2 - _col7) (type: bigint),
COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary),
'LONG' (type: string), UDFToLong(_col9) (typ [...]
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5,
_col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15,
_col16, _col17
+ Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE
Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 794 Basic stats:
COMPLETE Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-2
+ Dependency Collection
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: test_insert_col_db.test_basic_table
+
+ Stage: Stage-3
+ Stats Work
+ Basic Stats Work:
+ Column Stats Desc:
+ Columns: id, name, age
+ Column Types: int, string, int
+ Table: test_insert_col_db.test_basic_table
+
+PREHOOK: query: DROP TABLE test_basic_table
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: test_insert_col_db@test_basic_table
+PREHOOK: Output: database:test_insert_col_db
+PREHOOK: Output: test_insert_col_db@test_basic_table
+POSTHOOK: query: DROP TABLE test_basic_table
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: test_insert_col_db@test_basic_table
+POSTHOOK: Output: database:test_insert_col_db
+POSTHOOK: Output: test_insert_col_db@test_basic_table
+PREHOOK: query: CREATE TABLE test_partitioned_table (
+ id INT,
+ name STRING,
+ age INT
+)
+PARTITIONED BY (year INT)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:test_insert_col_db
+PREHOOK: Output: test_insert_col_db@test_partitioned_table
+POSTHOOK: query: CREATE TABLE test_partitioned_table (
+ id INT,
+ name STRING,
+ age INT
+)
+PARTITIONED BY (year INT)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:test_insert_col_db
+POSTHOOK: Output: test_insert_col_db@test_partitioned_table
+PREHOOK: query: ALTER TABLE test_partitioned_table ADD PARTITION (year=2024)
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Output: test_insert_col_db@test_partitioned_table
+POSTHOOK: query: ALTER TABLE test_partitioned_table ADD PARTITION (year=2024)
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Output: test_insert_col_db@test_partitioned_table
+POSTHOOK: Output: test_insert_col_db@test_partitioned_table@year=2024
+PREHOOK: query: SHOW DATABASES LIKE 'test_insert_col_db'
+PREHOOK: type: SHOWDATABASES
+filterListCmdObjects
+HIVE PRIVILEGE OBJECT { objectName: default type: DATABASE actionType: OTHER
dbName: default}
+HIVE PRIVILEGE OBJECT { objectName: test_insert_col_db type: DATABASE
actionType: OTHER dbName: test_insert_col_db}
+POSTHOOK: query: SHOW DATABASES LIKE 'test_insert_col_db'
+POSTHOOK: type: SHOWDATABASES
+test_insert_col_db
+PREHOOK: query: SHOW TABLES IN test_insert_col_db
+PREHOOK: type: SHOWTABLES
+PREHOOK: Input: database:test_insert_col_db
+filterListCmdObjects
+HIVE PRIVILEGE OBJECT { objectName: test_partitioned_table type: TABLE_OR_VIEW
actionType: OTHER dbName: test_insert_col_db}
+POSTHOOK: query: SHOW TABLES IN test_insert_col_db
+POSTHOOK: type: SHOWTABLES
+POSTHOOK: Input: database:test_insert_col_db
+test_partitioned_table
+PREHOOK: query: EXPLAIN INSERT INTO test_insert_col_db.test_partitioned_table
(id, name, age) VALUES (2, 'Dummy2', 25)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: test_insert_col_db@test_partitioned_table
+POSTHOOK: query: EXPLAIN INSERT INTO test_insert_col_db.test_partitioned_table
(id, name, age) VALUES (2, 'Dummy2', 25)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: test_insert_col_db@test_partitioned_table
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: _dummy_table
+ Row Limit Per Split: 1
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE
Column stats: COMPLETE
+ Select Operator
+ expressions: array(const struct(2,'Dummy2',25)) (type:
array<struct<col1:int,col2:string,col3:int>>)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 48 Basic stats:
COMPLETE Column stats: COMPLETE
+ UDTF Operator
+ Statistics: Num rows: 1 Data size: 48 Basic stats:
COMPLETE Column stats: COMPLETE
+ function name: inline
+ Select Operator
+ expressions: col1 (type: int), col2 (type: string),
col3 (type: int), null (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 1 Data size: 4 Basic stats:
COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 4 Basic stats:
COMPLETE Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.mapred.TextInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: test_insert_col_db.test_partitioned_table
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type:
string), _col2 (type: int), _col3 (type: int)
+ outputColumnNames: id, name, age, year
+ Statistics: Num rows: 1 Data size: 4 Basic stats:
COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: min(id), max(id), count(1),
count(id), compute_bit_vector_hll(id), max(length(name)),
avg(COALESCE(length(name),0)), count(name), compute_bit_vector_hll(name),
min(age), max(age), count(age), compute_bit_vector_hll(age)
+ keys: year (type: int)
+ minReductionHashAggr: 0.4
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2, _col3,
_col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13
+ Statistics: Num rows: 1 Data size: 564 Basic
stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 1 Data size: 564 Basic
stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col1 (type: int), _col2
(type: int), _col3 (type: bigint), _col4 (type: bigint), _col5 (type: binary),
_col6 (type: int), _col7 (type: struct<count:bigint,sum:double,input:int>),
_col8 (type: bigint), _col9 (type: binary), _col10 (type: int), _col11 (type:
int), _col12 (type: bigint), _col13 (type: binary)
+ Execution mode: llap
+ LLAP IO: no inputs
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: min(VALUE._col0), max(VALUE._col1),
count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4),
max(VALUE._col5), avg(VALUE._col6), count(VALUE._col7),
compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10),
count(VALUE._col11), compute_bit_vector_hll(VALUE._col12)
+ keys: KEY._col0 (type: int)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5,
_col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13
+ Statistics: Num rows: 1 Data size: 496 Basic stats: COMPLETE
Column stats: COMPLETE
+ Select Operator
+ expressions: 'LONG' (type: string), UDFToLong(_col1) (type:
bigint), UDFToLong(_col2) (type: bigint), (_col3 - _col4) (type: bigint),
COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary),
'STRING' (type: string), UDFToLong(COALESCE(_col6,0)) (type: bigint),
COALESCE(_col7,0) (type: double), (_col3 - _col8) (type: bigint),
COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary),
'LONG' (type: string), UDFToLong(_col10) (ty [...]
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5,
_col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15,
_col16, _col17, _col18
+ Statistics: Num rows: 1 Data size: 798 Basic stats: COMPLETE
Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 798 Basic stats:
COMPLETE Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-2
+ Dependency Collection
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ partition:
+ year
+ replace: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: test_insert_col_db.test_partitioned_table
+
+ Stage: Stage-3
+ Stats Work
+ Basic Stats Work:
+ Column Stats Desc:
+ Columns: id, name, age
+ Column Types: int, string, int
+ Table: test_insert_col_db.test_partitioned_table
+
+PREHOOK: query: DROP TABLE test_partitioned_table
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: test_insert_col_db@test_partitioned_table
+PREHOOK: Output: database:test_insert_col_db
+PREHOOK: Output: test_insert_col_db@test_partitioned_table
+POSTHOOK: query: DROP TABLE test_partitioned_table
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: test_insert_col_db@test_partitioned_table
+POSTHOOK: Output: database:test_insert_col_db
+POSTHOOK: Output: test_insert_col_db@test_partitioned_table
+PREHOOK: query: CREATE TABLE test_acid_table (
+ id INT,
+ name STRING,
+ age INT
+)
+STORED AS ORC
+TBLPROPERTIES ('transactional'='true')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:test_insert_col_db
+PREHOOK: Output: test_insert_col_db@test_acid_table
+POSTHOOK: query: CREATE TABLE test_acid_table (
+ id INT,
+ name STRING,
+ age INT
+)
+STORED AS ORC
+TBLPROPERTIES ('transactional'='true')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:test_insert_col_db
+POSTHOOK: Output: test_insert_col_db@test_acid_table
+PREHOOK: query: SHOW DATABASES LIKE 'test_insert_col_db'
+PREHOOK: type: SHOWDATABASES
+filterListCmdObjects
+HIVE PRIVILEGE OBJECT { objectName: default type: DATABASE actionType: OTHER
dbName: default}
+HIVE PRIVILEGE OBJECT { objectName: test_insert_col_db type: DATABASE
actionType: OTHER dbName: test_insert_col_db}
+POSTHOOK: query: SHOW DATABASES LIKE 'test_insert_col_db'
+POSTHOOK: type: SHOWDATABASES
+test_insert_col_db
+PREHOOK: query: SHOW TABLES IN test_insert_col_db
+PREHOOK: type: SHOWTABLES
+PREHOOK: Input: database:test_insert_col_db
+filterListCmdObjects
+HIVE PRIVILEGE OBJECT { objectName: test_acid_table type: TABLE_OR_VIEW
actionType: OTHER dbName: test_insert_col_db}
+POSTHOOK: query: SHOW TABLES IN test_insert_col_db
+POSTHOOK: type: SHOWTABLES
+POSTHOOK: Input: database:test_insert_col_db
+test_acid_table
+PREHOOK: query: EXPLAIN INSERT INTO test_insert_col_db.test_acid_table (id,
name, age) VALUES (3, 'Dummy3', 26)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: test_insert_col_db@test_acid_table
+POSTHOOK: query: EXPLAIN INSERT INTO test_insert_col_db.test_acid_table (id,
name, age) VALUES (3, 'Dummy3', 26)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: test_insert_col_db@test_acid_table
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: _dummy_table
+ Row Limit Per Split: 1
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE
Column stats: COMPLETE
+ Select Operator
+ expressions: array(const struct(3,'Dummy3',26)) (type:
array<struct<col1:int,col2:string,col3:int>>)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 48 Basic stats:
COMPLETE Column stats: COMPLETE
+ UDTF Operator
+ Statistics: Num rows: 1 Data size: 48 Basic stats:
COMPLETE Column stats: COMPLETE
+ function name: inline
+ Select Operator
+ expressions: col1 (type: int), col2 (type: string),
col3 (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 8 Basic stats:
COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats:
COMPLETE Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: test_insert_col_db.test_acid_table
+ Write Type: INSERT
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type:
string), _col2 (type: int)
+ outputColumnNames: id, name, age
+ Statistics: Num rows: 1 Data size: 8 Basic stats:
COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: min(id), max(id), count(1),
count(id), compute_bit_vector_hll(id), max(length(name)),
avg(COALESCE(length(name),0)), count(name), compute_bit_vector_hll(name),
min(age), max(age), count(age), compute_bit_vector_hll(age)
+ minReductionHashAggr: 0.4
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2, _col3,
_col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12
+ Statistics: Num rows: 1 Data size: 560 Basic
stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ null sort order:
+ sort order:
+ Statistics: Num rows: 1 Data size: 560 Basic
stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: int), _col1
(type: int), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary),
_col5 (type: int), _col6 (type: struct<count:bigint,sum:double,input:int>),
_col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type:
int), _col11 (type: bigint), _col12 (type: binary)
+ Execution mode: llap
+ LLAP IO: no inputs
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: min(VALUE._col0), max(VALUE._col1),
count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4),
max(VALUE._col5), avg(VALUE._col6), count(VALUE._col7),
compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10),
count(VALUE._col11), compute_bit_vector_hll(VALUE._col12)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5,
_col6, _col7, _col8, _col9, _col10, _col11, _col12
+ Statistics: Num rows: 1 Data size: 492 Basic stats: COMPLETE
Column stats: COMPLETE
+ Select Operator
+ expressions: 'LONG' (type: string), UDFToLong(_col0) (type:
bigint), UDFToLong(_col1) (type: bigint), (_col2 - _col3) (type: bigint),
COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary),
'STRING' (type: string), UDFToLong(COALESCE(_col5,0)) (type: bigint),
COALESCE(_col6,0) (type: double), (_col2 - _col7) (type: bigint),
COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary),
'LONG' (type: string), UDFToLong(_col9) (typ [...]
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5,
_col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15,
_col16, _col17
+ Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE
Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 794 Basic stats:
COMPLETE Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-2
+ Dependency Collection
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: false
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: test_insert_col_db.test_acid_table
+ Write Type: INSERT
+
+ Stage: Stage-3
+ Stats Work
+ Basic Stats Work:
+ Column Stats Desc:
+ Columns: id, name, age
+ Column Types: int, string, int
+ Table: test_insert_col_db.test_acid_table
+
+PREHOOK: query: DROP TABLE test_acid_table
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: test_insert_col_db@test_acid_table
+PREHOOK: Output: database:test_insert_col_db
+PREHOOK: Output: test_insert_col_db@test_acid_table
+POSTHOOK: query: DROP TABLE test_acid_table
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: test_insert_col_db@test_acid_table
+POSTHOOK: Output: database:test_insert_col_db
+POSTHOOK: Output: test_insert_col_db@test_acid_table
+PREHOOK: query: CREATE TABLE test_restricted_column_table (
+ id INT,
+ name STRING,
+ age INT
+)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:test_insert_col_db
+PREHOOK: Output: test_insert_col_db@test_restricted_column_table
+POSTHOOK: query: CREATE TABLE test_restricted_column_table (
+ id INT,
+ name STRING,
+ age INT
+)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:test_insert_col_db
+POSTHOOK: Output: test_insert_col_db@test_restricted_column_table
+PREHOOK: query: SHOW DATABASES LIKE 'test_insert_col_db'
+PREHOOK: type: SHOWDATABASES
+filterListCmdObjects
+HIVE PRIVILEGE OBJECT { objectName: default type: DATABASE actionType: OTHER
dbName: default}
+HIVE PRIVILEGE OBJECT { objectName: test_insert_col_db type: DATABASE
actionType: OTHER dbName: test_insert_col_db}
+POSTHOOK: query: SHOW DATABASES LIKE 'test_insert_col_db'
+POSTHOOK: type: SHOWDATABASES
+test_insert_col_db
+PREHOOK: query: SHOW TABLES IN test_insert_col_db
+PREHOOK: type: SHOWTABLES
+PREHOOK: Input: database:test_insert_col_db
+filterListCmdObjects
+HIVE PRIVILEGE OBJECT { objectName: test_restricted_column_table type:
TABLE_OR_VIEW actionType: OTHER dbName: test_insert_col_db}
+POSTHOOK: query: SHOW TABLES IN test_insert_col_db
+POSTHOOK: type: SHOWTABLES
+POSTHOOK: Input: database:test_insert_col_db
+test_restricted_column_table
+PREHOOK: query: EXPLAIN INSERT INTO
test_insert_col_db.test_restricted_column_table VALUES (8, 'Dummy4', 55)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: test_insert_col_db@test_restricted_column_table
+POSTHOOK: query: EXPLAIN INSERT INTO
test_insert_col_db.test_restricted_column_table VALUES (8, 'Dummy4', 55)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: test_insert_col_db@test_restricted_column_table
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: _dummy_table
+ Row Limit Per Split: 1
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE
Column stats: COMPLETE
+ Select Operator
+ expressions: array(const struct(8,'Dummy4',55)) (type:
array<struct<col1:int,col2:string,col3:int>>)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 48 Basic stats:
COMPLETE Column stats: COMPLETE
+ UDTF Operator
+ Statistics: Num rows: 1 Data size: 48 Basic stats:
COMPLETE Column stats: COMPLETE
+ function name: inline
+ Select Operator
+ expressions: col1 (type: int), col2 (type: string),
col3 (type: int)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 1 Data size: 8 Basic stats:
COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats:
COMPLETE Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.mapred.TextInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name:
test_insert_col_db.test_restricted_column_table
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type:
string), _col2 (type: int)
+ outputColumnNames: id, name, age
+ Statistics: Num rows: 1 Data size: 8 Basic stats:
COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: min(id), max(id), count(1),
count(id), compute_bit_vector_hll(id), max(length(name)),
avg(COALESCE(length(name),0)), count(name), compute_bit_vector_hll(name),
min(age), max(age), count(age), compute_bit_vector_hll(age)
+ minReductionHashAggr: 0.4
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2, _col3,
_col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12
+ Statistics: Num rows: 1 Data size: 560 Basic
stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ null sort order:
+ sort order:
+ Statistics: Num rows: 1 Data size: 560 Basic
stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: int), _col1
(type: int), _col2 (type: bigint), _col3 (type: bigint), _col4 (type: binary),
_col5 (type: int), _col6 (type: struct<count:bigint,sum:double,input:int>),
_col7 (type: bigint), _col8 (type: binary), _col9 (type: int), _col10 (type:
int), _col11 (type: bigint), _col12 (type: binary)
+ Execution mode: llap
+ LLAP IO: no inputs
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: min(VALUE._col0), max(VALUE._col1),
count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4),
max(VALUE._col5), avg(VALUE._col6), count(VALUE._col7),
compute_bit_vector_hll(VALUE._col8), min(VALUE._col9), max(VALUE._col10),
count(VALUE._col11), compute_bit_vector_hll(VALUE._col12)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5,
_col6, _col7, _col8, _col9, _col10, _col11, _col12
+ Statistics: Num rows: 1 Data size: 492 Basic stats: COMPLETE
Column stats: COMPLETE
+ Select Operator
+ expressions: 'LONG' (type: string), UDFToLong(_col0) (type:
bigint), UDFToLong(_col1) (type: bigint), (_col2 - _col3) (type: bigint),
COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary),
'STRING' (type: string), UDFToLong(COALESCE(_col5,0)) (type: bigint),
COALESCE(_col6,0) (type: double), (_col2 - _col7) (type: bigint),
COALESCE(ndv_compute_bit_vector(_col8),0) (type: bigint), _col8 (type: binary),
'LONG' (type: string), UDFToLong(_col9) (typ [...]
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5,
_col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15,
_col16, _col17
+ Statistics: Num rows: 1 Data size: 794 Basic stats: COMPLETE
Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 794 Basic stats:
COMPLETE Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-2
+ Dependency Collection
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: test_insert_col_db.test_restricted_column_table
+
+ Stage: Stage-3
+ Stats Work
+ Basic Stats Work:
+ Column Stats Desc:
+ Columns: id, name, age
+ Column Types: int, string, int
+ Table: test_insert_col_db.test_restricted_column_table
+
+PREHOOK: query: DROP TABLE test_restricted_column_table
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: test_insert_col_db@test_restricted_column_table
+PREHOOK: Output: database:test_insert_col_db
+PREHOOK: Output: test_insert_col_db@test_restricted_column_table
+POSTHOOK: query: DROP TABLE test_restricted_column_table
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: test_insert_col_db@test_restricted_column_table
+POSTHOOK: Output: database:test_insert_col_db
+POSTHOOK: Output: test_insert_col_db@test_restricted_column_table
+PREHOOK: query: DROP DATABASE test_insert_col_db
+PREHOOK: type: DROPDATABASE
+PREHOOK: Input: database:test_insert_col_db
+PREHOOK: Output: database:test_insert_col_db
+POSTHOOK: query: DROP DATABASE test_insert_col_db
+POSTHOOK: type: DROPDATABASE
+POSTHOOK: Input: database:test_insert_col_db
+POSTHOOK: Output: database:test_insert_col_db
