Repository: hive Updated Branches: refs/heads/master 95929308b -> 0d93438a3
HIVE-10776 - Schema on insert for bucketed tables throwing NullPointerException(Eugene Koifman, reviewed by Alan Gates) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0d93438a Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0d93438a Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0d93438a Branch: refs/heads/master Commit: 0d93438a3543cb64cbe2ebcdc21e5b40c1dd86e6 Parents: 9592930 Author: Eugene Koifman <ekoif...@hortonworks.com> Authored: Fri May 22 13:50:40 2015 -0700 Committer: Eugene Koifman <ekoif...@hortonworks.com> Committed: Fri May 22 13:50:40 2015 -0700 ---------------------------------------------------------------------- .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 9 +- .../clientpositive/insert_into_with_schema2.q | 23 +++++ .../insert_into_with_schema2.q.out | 98 ++++++++++++++++++++ 3 files changed, 126 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/0d93438a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 4346f60..7f355e5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -3910,7 +3910,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { * @throws SemanticException */ private void handleInsertStatementSpec(List<ExprNodeDesc> col_list, String dest, - RowResolver out_rwsch, RowResolver inputRR, QB qb, + RowResolver outputRR, RowResolver inputRR, QB qb, ASTNode selExprList) throws SemanticException { //(z,x) List<String> targetTableSchema = qb.getParseInfo().getDestSchemaForClause(dest);//specified in the query @@ -3932,7 +3932,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { Map<String, ColumnInfo> targetCol2ColumnInfo = new HashMap<String, ColumnInfo>(); int colListPos = 0; for(String targetCol : targetTableSchema) { - targetCol2ColumnInfo.put(targetCol, out_rwsch.getColumnInfos().get(colListPos)); + targetCol2ColumnInfo.put(targetCol, outputRR.getColumnInfos().get(colListPos)); targetCol2Projection.put(targetCol, col_list.get(colListPos++)); } Table target = qb.getMetaData().getDestTableForAlias(dest); @@ -3976,16 +3976,17 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { t.setText("TOK_NULL"); ExprNodeDesc exp = genExprNodeDesc(new ASTNode(t), inputRR, tcCtx); new_col_list.add(exp); - final String tableAlias = "";//is this OK? this column doesn't come from any table + final String tableAlias = null;//this column doesn't come from any table ColumnInfo colInfo = new ColumnInfo(getColumnInternalName(colListPos), exp.getWritableObjectInspector(), tableAlias, false); newSchema.add(colInfo); + outputRR.addMappingOnly(colInfo.getTabAlias(), colInfo.getInternalName(), colInfo); } colListPos++; } col_list.clear(); col_list.addAll(new_col_list); - out_rwsch.setRowSchema(new RowSchema(newSchema)); + outputRR.setRowSchema(new RowSchema(newSchema)); } String recommendName(ExprNodeDesc exp, String colAlias) { if (!colAlias.startsWith(autogenColAliasPrfxLbl)) { http://git-wip-us.apache.org/repos/asf/hive/blob/0d93438a/ql/src/test/queries/clientpositive/insert_into_with_schema2.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/insert_into_with_schema2.q b/ql/src/test/queries/clientpositive/insert_into_with_schema2.q new file mode 100644 index 0000000..b7c6b58 --- /dev/null +++ b/ql/src/test/queries/clientpositive/insert_into_with_schema2.q @@ -0,0 +1,23 @@ +-- SORT_QUERY_RESULTS; + +set hive.enforce.bucketing=true; + +create table studenttab10k (age2 int); +insert into studenttab10k values(1); + +create table student_acid (age int, grade int) + clustered by (age) into 1 buckets; + +insert into student_acid(age) select * from studenttab10k; + +select * from student_acid; + +insert into student_acid(grade, age) select 3 g, * from studenttab10k; + +select * from student_acid; + +insert into student_acid(grade, age) values(20, 2); + +insert into student_acid(age) values(22); + +select * from student_acid; http://git-wip-us.apache.org/repos/asf/hive/blob/0d93438a/ql/src/test/results/clientpositive/insert_into_with_schema2.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/insert_into_with_schema2.q.out b/ql/src/test/results/clientpositive/insert_into_with_schema2.q.out new file mode 100644 index 0000000..a55a82f --- /dev/null +++ b/ql/src/test/results/clientpositive/insert_into_with_schema2.q.out @@ -0,0 +1,98 @@ +PREHOOK: query: create table studenttab10k (age2 int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@studenttab10k +POSTHOOK: query: create table studenttab10k (age2 int) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@studenttab10k +PREHOOK: query: insert into studenttab10k values(1) +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__1 +PREHOOK: Output: default@studenttab10k +POSTHOOK: query: insert into studenttab10k values(1) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__1 +POSTHOOK: Output: default@studenttab10k +POSTHOOK: Lineage: studenttab10k.age2 EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: create table student_acid (age int, grade int) + clustered by (age) into 1 buckets +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@student_acid +POSTHOOK: query: create table student_acid (age int, grade int) + clustered by (age) into 1 buckets +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@student_acid +PREHOOK: query: insert into student_acid(age) select * from studenttab10k +PREHOOK: type: QUERY +PREHOOK: Input: default@studenttab10k +PREHOOK: Output: default@student_acid +POSTHOOK: query: insert into student_acid(age) select * from studenttab10k +POSTHOOK: type: QUERY +POSTHOOK: Input: default@studenttab10k +POSTHOOK: Output: default@student_acid +POSTHOOK: Lineage: student_acid.age SIMPLE [(studenttab10k)studenttab10k.FieldSchema(name:age2, type:int, comment:null), ] +POSTHOOK: Lineage: student_acid.grade EXPRESSION [] +PREHOOK: query: select * from student_acid +PREHOOK: type: QUERY +PREHOOK: Input: default@student_acid +#### A masked pattern was here #### +POSTHOOK: query: select * from student_acid +POSTHOOK: type: QUERY +POSTHOOK: Input: default@student_acid +#### A masked pattern was here #### +1 NULL +PREHOOK: query: insert into student_acid(grade, age) select 3 g, * from studenttab10k +PREHOOK: type: QUERY +PREHOOK: Input: default@studenttab10k +PREHOOK: Output: default@student_acid +POSTHOOK: query: insert into student_acid(grade, age) select 3 g, * from studenttab10k +POSTHOOK: type: QUERY +POSTHOOK: Input: default@studenttab10k +POSTHOOK: Output: default@student_acid +POSTHOOK: Lineage: student_acid.age SIMPLE [(studenttab10k)studenttab10k.FieldSchema(name:age2, type:int, comment:null), ] +POSTHOOK: Lineage: student_acid.grade SIMPLE [] +PREHOOK: query: select * from student_acid +PREHOOK: type: QUERY +PREHOOK: Input: default@student_acid +#### A masked pattern was here #### +POSTHOOK: query: select * from student_acid +POSTHOOK: type: QUERY +POSTHOOK: Input: default@student_acid +#### A masked pattern was here #### +1 3 +1 NULL +PREHOOK: query: insert into student_acid(grade, age) values(20, 2) +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__2 +PREHOOK: Output: default@student_acid +POSTHOOK: query: insert into student_acid(grade, age) values(20, 2) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__2 +POSTHOOK: Output: default@student_acid +POSTHOOK: Lineage: student_acid.age EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +POSTHOOK: Lineage: student_acid.grade EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +PREHOOK: query: insert into student_acid(age) values(22) +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__3 +PREHOOK: Output: default@student_acid +POSTHOOK: query: insert into student_acid(age) values(22) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__3 +POSTHOOK: Output: default@student_acid +POSTHOOK: Lineage: student_acid.age EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: student_acid.grade EXPRESSION [] +PREHOOK: query: select * from student_acid +PREHOOK: type: QUERY +PREHOOK: Input: default@student_acid +#### A masked pattern was here #### +POSTHOOK: query: select * from student_acid +POSTHOOK: type: QUERY +POSTHOOK: Input: default@student_acid +#### A masked pattern was here #### +1 3 +1 NULL +2 20 +22 NULL