Author: hashutosh
Date: Fri Dec 6 14:59:57 2013
New Revision: 1548549
URL: http://svn.apache.org/r1548549
Log:
HIVE-5686 : partition column type validation doesn't quite work for dates
(Sergey Shelukhin via Ashutosh Chauhan)
Added:
hive/trunk/ql/src/test/queries/clientnegative/illegal_partition_type4.q
hive/trunk/ql/src/test/results/clientnegative/illegal_partition_type4.q.out
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1548549&r1=1548548&r2=1548549&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Fri
Dec 6 14:59:57 2013
@@ -2351,7 +2351,7 @@ public final class Utilities {
jobConf.set(serdeConstants.LIST_COLUMN_TYPES, columnTypesString);
}
- public static void validatePartSpec(Table tbl, Map<String, String> partSpec)
+ public static void validatePartSpecColumnNames(Table tbl, Map<String,
String> partSpec)
throws SemanticException {
List<FieldSchema> parts = tbl.getPartitionKeys();
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java?rev=1548549&r1=1548548&r2=1548549&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
Fri Dec 6 14:59:57 2013
@@ -1134,48 +1134,53 @@ public abstract class BaseSemanticAnalyz
return storedAsDirs;
}
- private static void getPartExprNodeDesc(ASTNode astNode,
- Map<ASTNode, ExprNodeDesc> astExprNodeMap)
- throws SemanticException, HiveException {
+ private static boolean getPartExprNodeDesc(ASTNode astNode,
+ Map<ASTNode, ExprNodeDesc> astExprNodeMap) throws SemanticException {
- if ((astNode == null) || (astNode.getChildren() == null) ||
- (astNode.getChildren().size() == 0)) {
- return;
+ if (astNode == null) {
+ return true;
+ } else if ((astNode.getChildren() == null) ||
(astNode.getChildren().size() == 0)) {
+ return astNode.getType() != HiveParser.TOK_PARTVAL;
}
TypeCheckCtx typeCheckCtx = new TypeCheckCtx(null);
+ boolean result = true;
for (Node childNode : astNode.getChildren()) {
ASTNode childASTNode = (ASTNode)childNode;
if (childASTNode.getType() != HiveParser.TOK_PARTVAL) {
- getPartExprNodeDesc(childASTNode, astExprNodeMap);
+ result = getPartExprNodeDesc(childASTNode, astExprNodeMap) && result;
} else {
- if (childASTNode.getChildren().size() <= 1) {
- throw new HiveException("This is dynamic partitioning");
+ boolean isDynamicPart = childASTNode.getChildren().size() <= 1;
+ result = !isDynamicPart && result;
+ if (!isDynamicPart) {
+ ASTNode partVal = (ASTNode)childASTNode.getChildren().get(1);
+ astExprNodeMap.put((ASTNode)childASTNode.getChildren().get(0),
+ TypeCheckProcFactory.genExprNode(partVal,
typeCheckCtx).get(partVal));
}
-
- ASTNode partValASTChild = (ASTNode)childASTNode.getChildren().get(1);
- astExprNodeMap.put((ASTNode)childASTNode.getChildren().get(0),
- TypeCheckProcFactory.genExprNode(partValASTChild,
typeCheckCtx).get(partValASTChild));
}
}
+ return result;
}
public static void validatePartSpec(Table tbl, Map<String, String> partSpec,
ASTNode astNode, HiveConf conf) throws SemanticException {
- Map<ASTNode, ExprNodeDesc> astExprNodeMap = new HashMap<ASTNode,
ExprNodeDesc>();
-
- Utilities.validatePartSpec(tbl, partSpec);
+ Utilities.validatePartSpecColumnNames(tbl, partSpec);
if (!HiveConf.getBoolVar(conf,
HiveConf.ConfVars.HIVE_TYPE_CHECK_ON_INSERT)) {
return;
}
- try {
- getPartExprNodeDesc(astNode, astExprNodeMap);
- } catch (HiveException e) {
- return;
+ Map<ASTNode, ExprNodeDesc> astExprNodeMap = new HashMap<ASTNode,
ExprNodeDesc>();
+ if (!getPartExprNodeDesc(astNode, astExprNodeMap)) {
+ STATIC_LOG.warn("Dynamic partitioning is used; only validating "
+ + astExprNodeMap.size() + " columns");
+ }
+
+ if (astExprNodeMap.isEmpty()) {
+ return; // All columns are dynamic, nothing to do.
}
+
List<FieldSchema> parts = tbl.getPartitionKeys();
Map<String, String> partCols = new HashMap<String, String>(parts.size());
for (FieldSchema col : parts) {
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1548549&r1=1548548&r2=1548549&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
Fri Dec 6 14:59:57 2013
@@ -5267,7 +5267,7 @@ public class SemanticAnalyzer extends Ba
}
dpCtx = qbm.getDPCtx(dest);
if (dpCtx == null) {
- Utilities.validatePartSpec(dest_tab, partSpec);
+ Utilities.validatePartSpecColumnNames(dest_tab, partSpec);
dpCtx = new DynamicPartitionCtx(dest_tab, partSpec,
conf.getVar(HiveConf.ConfVars.DEFAULTPARTITIONNAME),
conf.getIntVar(HiveConf.ConfVars.DYNAMICPARTITIONMAXPARTSPERNODE));
Added: hive/trunk/ql/src/test/queries/clientnegative/illegal_partition_type4.q
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientnegative/illegal_partition_type4.q?rev=1548549&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientnegative/illegal_partition_type4.q
(added)
+++ hive/trunk/ql/src/test/queries/clientnegative/illegal_partition_type4.q Fri
Dec 6 14:59:57 2013
@@ -0,0 +1,3 @@
+create table tab1(s string) PARTITIONED BY(dt date, st string);
+alter table tab1 add partition (dt=date 'foo', st='foo');
+drop table tab1;
Added:
hive/trunk/ql/src/test/results/clientnegative/illegal_partition_type4.q.out
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/illegal_partition_type4.q.out?rev=1548549&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientnegative/illegal_partition_type4.q.out
(added)
+++ hive/trunk/ql/src/test/results/clientnegative/illegal_partition_type4.q.out
Fri Dec 6 14:59:57 2013
@@ -0,0 +1,6 @@
+PREHOOK: query: create table tab1(s string) PARTITIONED BY(dt date, st string)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table tab1(s string) PARTITIONED BY(dt date, st string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@tab1
+FAILED: SemanticException Unable to convert date literal string to date value.
