HIVE-12156: expanding view doesn't quote reserved keyword (Pengcheng Xiong, reviewed by Laljo John Pullokkaran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4d2df795 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4d2df795 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4d2df795 Branch: refs/heads/master-fixed Commit: 4d2df795c1e0ac3fe378486a6aba5fa0a3d9ffbb Parents: 3511df7 Author: Pengcheng Xiong <[email protected]> Authored: Thu Nov 5 11:26:05 2015 -0800 Committer: Pengcheng Xiong <[email protected]> Committed: Thu Nov 5 11:26:05 2015 -0800 ---------------------------------------------------------------------- .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 37 ++++++ .../queries/clientpositive/struct_in_view.q | 28 +++++ .../results/clientpositive/struct_in_view.q.out | 118 +++++++++++++++++++ 3 files changed, 183 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/4d2df795/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index d2c3a7c..f3d7057 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -26,6 +26,8 @@ import java.io.Serializable; import java.security.AccessControlException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -155,6 +157,7 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeColumnListDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils; +import org.apache.hadoop.hive.ql.plan.ExprNodeFieldDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc; import org.apache.hadoop.hive.ql.plan.FileSinkDesc; import org.apache.hadoop.hive.ql.plan.FilterDesc; @@ -10496,8 +10499,16 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { return nodeOutputs; } + Map<ExprNodeDesc,String> nodeToText = new HashMap<>(); + List<Entry<ASTNode, ExprNodeDesc>> fieldDescList = new ArrayList<>(); + for (Map.Entry<ASTNode, ExprNodeDesc> entry : nodeOutputs.entrySet()) { if (!(entry.getValue() instanceof ExprNodeColumnDesc)) { + // we need to translate the ExprNodeFieldDesc too, e.g., identifiers in + // struct<>. + if (entry.getValue() instanceof ExprNodeFieldDesc) { + fieldDescList.add(entry); + } continue; } ASTNode node = entry.getKey(); @@ -10513,9 +10524,35 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { replacementText.append(HiveUtils.unparseIdentifier(tmp[0], conf)); replacementText.append("."); replacementText.append(HiveUtils.unparseIdentifier(tmp[1], conf)); + nodeToText.put(columnDesc, replacementText.toString()); unparseTranslator.addTranslation(node, replacementText.toString()); } + if (fieldDescList.size() != 0) { + // Sorting the list based on the length of fieldName + // For example, in Column[a].b.c and Column[a].b, Column[a].b should be + // unparsed before Column[a].b.c + Collections.sort(fieldDescList, new Comparator<Map.Entry<ASTNode, ExprNodeDesc>>() { + public int compare(Entry<ASTNode, ExprNodeDesc> o1, Entry<ASTNode, ExprNodeDesc> o2) { + ExprNodeFieldDesc fieldDescO1 = (ExprNodeFieldDesc) o1.getValue(); + ExprNodeFieldDesc fieldDescO2 = (ExprNodeFieldDesc) o2.getValue(); + return fieldDescO1.toString().length() < fieldDescO2.toString().length() ? -1 : 1; + } + }); + for (Map.Entry<ASTNode, ExprNodeDesc> entry : fieldDescList) { + ASTNode node = entry.getKey(); + ExprNodeFieldDesc fieldDesc = (ExprNodeFieldDesc) entry.getValue(); + ExprNodeDesc exprNodeDesc = fieldDesc.getDesc(); + String fieldName = fieldDesc.getFieldName(); + StringBuilder replacementText = new StringBuilder(); + replacementText.append(nodeToText.get(exprNodeDesc)); + replacementText.append("."); + replacementText.append(HiveUtils.unparseIdentifier(fieldName, conf)); + nodeToText.put(fieldDesc, replacementText.toString()); + unparseTranslator.addTranslation(node, replacementText.toString()); + } + } + return nodeOutputs; } http://git-wip-us.apache.org/repos/asf/hive/blob/4d2df795/ql/src/test/queries/clientpositive/struct_in_view.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/struct_in_view.q b/ql/src/test/queries/clientpositive/struct_in_view.q new file mode 100644 index 0000000..d420030 --- /dev/null +++ b/ql/src/test/queries/clientpositive/struct_in_view.q @@ -0,0 +1,28 @@ +drop table testreserved; + +create table testreserved (data struct<`end`:string, id: string>); + +create view testreservedview as select data.`end` as data_end, data.id as data_id from testreserved; + +describe extended testreservedview; + +select data.`end` from testreserved; + +drop view testreservedview; + +drop table testreserved; + +create table s (default struct<src:struct<`end`:struct<key:string>, id: string>, id: string>); + +create view vs1 as select default.src.`end`.key from s; + +describe extended vs1; + +create view vs2 as select default.src.`end` from s; + +describe extended vs2; + +drop view vs1; + +drop view vs2; + http://git-wip-us.apache.org/repos/asf/hive/blob/4d2df795/ql/src/test/results/clientpositive/struct_in_view.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/struct_in_view.q.out b/ql/src/test/results/clientpositive/struct_in_view.q.out new file mode 100644 index 0000000..10b2f2e --- /dev/null +++ b/ql/src/test/results/clientpositive/struct_in_view.q.out @@ -0,0 +1,118 @@ +PREHOOK: query: drop table testreserved +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table testreserved +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table testreserved (data struct<`end`:string, id: string>) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@testreserved +POSTHOOK: query: create table testreserved (data struct<`end`:string, id: string>) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@testreserved +PREHOOK: query: create view testreservedview as select data.`end` as data_end, data.id as data_id from testreserved +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@testreserved +PREHOOK: Output: database:default +PREHOOK: Output: default@testreservedview +POSTHOOK: query: create view testreservedview as select data.`end` as data_end, data.id as data_id from testreserved +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@testreserved +POSTHOOK: Output: database:default +POSTHOOK: Output: default@testreservedview +PREHOOK: query: describe extended testreservedview +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@testreservedview +POSTHOOK: query: describe extended testreservedview +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@testreservedview +data_end string +data_id string + +#### A masked pattern was here #### +PREHOOK: query: select data.`end` from testreserved +PREHOOK: type: QUERY +PREHOOK: Input: default@testreserved +#### A masked pattern was here #### +POSTHOOK: query: select data.`end` from testreserved +POSTHOOK: type: QUERY +POSTHOOK: Input: default@testreserved +#### A masked pattern was here #### +PREHOOK: query: drop view testreservedview +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@testreservedview +PREHOOK: Output: default@testreservedview +POSTHOOK: query: drop view testreservedview +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@testreservedview +POSTHOOK: Output: default@testreservedview +PREHOOK: query: drop table testreserved +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@testreserved +PREHOOK: Output: default@testreserved +POSTHOOK: query: drop table testreserved +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@testreserved +POSTHOOK: Output: default@testreserved +PREHOOK: query: create table s (default struct<src:struct<`end`:struct<key:string>, id: string>, id: string>) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@s +POSTHOOK: query: create table s (default struct<src:struct<`end`:struct<key:string>, id: string>, id: string>) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@s +PREHOOK: query: create view vs1 as select default.src.`end`.key from s +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@s +PREHOOK: Output: database:default +PREHOOK: Output: default@vs1 +POSTHOOK: query: create view vs1 as select default.src.`end`.key from s +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@s +POSTHOOK: Output: database:default +POSTHOOK: Output: default@vs1 +PREHOOK: query: describe extended vs1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@vs1 +POSTHOOK: query: describe extended vs1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@vs1 +key string + +#### A masked pattern was here #### +PREHOOK: query: create view vs2 as select default.src.`end` from s +PREHOOK: type: CREATEVIEW +PREHOOK: Input: default@s +PREHOOK: Output: database:default +PREHOOK: Output: default@vs2 +POSTHOOK: query: create view vs2 as select default.src.`end` from s +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: default@s +POSTHOOK: Output: database:default +POSTHOOK: Output: default@vs2 +PREHOOK: query: describe extended vs2 +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@vs2 +POSTHOOK: query: describe extended vs2 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@vs2 +end struct<key:string> + +#### A masked pattern was here #### +PREHOOK: query: drop view vs1 +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@vs1 +PREHOOK: Output: default@vs1 +POSTHOOK: query: drop view vs1 +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@vs1 +POSTHOOK: Output: default@vs1 +PREHOOK: query: drop view vs2 +PREHOOK: type: DROPVIEW +PREHOOK: Input: default@vs2 +PREHOOK: Output: default@vs2 +POSTHOOK: query: drop view vs2 +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: default@vs2 +POSTHOOK: Output: default@vs2
