Author: namit
Date: Tue Oct 25 00:02:04 2011
New Revision: 1188450
URL: http://svn.apache.org/viewvc?rev=1188450&view=rev
Log:
HIVE-2519 Dynamic partition insert should enforce the order of the
partition spec is the same as the one in schema (Ning Zhang via namit)
Added:
hive/trunk/ql/src/test/queries/clientnegative/dyn_part4.q
hive/trunk/ql/src/test/results/clientnegative/dyn_part4.q.out
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java?rev=1188450&r1=1188449&r2=1188450&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
Tue Oct 25 00:02:04 2011
@@ -23,6 +23,7 @@ import java.io.UnsupportedEncodingExcept
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
@@ -675,17 +676,30 @@ public abstract class BaseSemanticAnalyz
conf.getVar(HiveConf.ConfVars.DYNAMICPARTITIONINGMODE).equalsIgnoreCase("strict"))
{
throw new
SemanticException(ErrorMsg.DYNAMIC_PARTITION_STRICT_MODE.getMsg());
}
- for (FieldSchema fs: parts) {
- if (partSpec.get(fs.getName().toLowerCase()) == null) {
- if (numStaPart > 0) { // found a DP, but there exists ST as
subpartition
- throw new SemanticException(
-
ErrorMsg.PARTITION_DYN_STA_ORDER.getMsg(ast.getChild(childIndex)));
- }
- break;
- } else {
- --numStaPart;
- }
- }
+
+ // check the partitions in partSpec be the same as defined in table
schema
+ if (partSpec.keySet().size() != parts.size()) {
+ ErrorPartSpec(partSpec, parts);
+ }
+ Iterator<String> itrPsKeys = partSpec.keySet().iterator();
+ for (FieldSchema fs: parts) {
+ if
(!itrPsKeys.next().toLowerCase().equals(fs.getName().toLowerCase())) {
+ ErrorPartSpec(partSpec, parts);
+ }
+ }
+
+ // check if static partition appear after dynamic partitions
+ for (FieldSchema fs: parts) {
+ if (partSpec.get(fs.getName().toLowerCase()) == null) {
+ if (numStaPart > 0) { // found a DP, but there exists ST as
subpartition
+ throw new SemanticException(
+
ErrorMsg.PARTITION_DYN_STA_ORDER.getMsg(ast.getChild(childIndex)));
+ }
+ break;
+ } else {
+ --numStaPart;
+ }
+ }
partHandle = null;
specType = SpecType.DYNAMIC_PARTITION;
} else {
@@ -714,6 +728,26 @@ public abstract class BaseSemanticAnalyz
}
}
+ private void ErrorPartSpec(Map<String, String> partSpec, List<FieldSchema>
parts)
+ throws SemanticException {
+ StringBuilder sb = new StringBuilder("Partition columns in the table
schema are: (");
+ for (FieldSchema fs: parts) {
+ sb.append(fs.getName()).append(", ");
+ }
+ sb.setLength(sb.length() - 2); // remove the last ", "
+ sb.append("), while the partitions specified in the query are: (");
+
+ Iterator<String> itrPsKeys = partSpec.keySet().iterator();
+ while (itrPsKeys.hasNext()) {
+ sb.append(itrPsKeys.next()).append(", ");
+ }
+ sb.setLength(sb.length() - 2); // remove the last ", "
+ sb.append(").");
+
+ throw new SemanticException(
+ ErrorMsg.PARTSPEC_DIFFER_FROM_SCHEMA.getMsg(sb.toString()));
+ }
+
public Map<String, String> getPartSpec() {
return this.partSpec;
}
@@ -730,6 +764,7 @@ public abstract class BaseSemanticAnalyz
return tableHandle.toString();
}
}
+
}
/**
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java?rev=1188450&r1=1188449&r2=1188450&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java Tue
Oct 25 00:02:04 2011
@@ -186,7 +186,9 @@ public enum ErrorMsg {
NO_COMPARE_BIGINT_DOUBLE("In strict mode, comparing bigints and doubles is
not allowed, "
+ "it may result in a loss of precision. "
+ "If you really want to perform the operation, set
hive.mapred.mode=nonstrict"),
- FUNCTIONS_ARE_NOT_SUPPORTED_IN_ORDER_BY("functions are not supported in
order by"),
+ FUNCTIONS_ARE_NOT_SUPPORTED_IN_ORDER_BY("functions are not supported in
order by"),
+ PARTSPEC_DIFFER_FROM_SCHEMA("Partition columns in partition specification
are not the same as "
+ + "that defined in the table schema. The names and orders have to be
exactly the same."),
;
private String mesg;
Added: hive/trunk/ql/src/test/queries/clientnegative/dyn_part4.q
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientnegative/dyn_part4.q?rev=1188450&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientnegative/dyn_part4.q (added)
+++ hive/trunk/ql/src/test/queries/clientnegative/dyn_part4.q Tue Oct 25
00:02:04 2011
@@ -0,0 +1,7 @@
+create table nzhang_part4 (key string) partitioned by (ds string, hr string,
value string);
+
+set hive.exec.dynamic.partition=true;
+
+insert overwrite table nzhang_part4 partition(value = 'aaa', ds='11', hr)
select key, hr from srcpart where ds is not null;
+
+drop table nzhang_part4;
Added: hive/trunk/ql/src/test/results/clientnegative/dyn_part4.q.out
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/dyn_part4.q.out?rev=1188450&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientnegative/dyn_part4.q.out (added)
+++ hive/trunk/ql/src/test/results/clientnegative/dyn_part4.q.out Tue Oct 25
00:02:04 2011
@@ -0,0 +1,6 @@
+PREHOOK: query: create table nzhang_part4 (key string) partitioned by (ds
string, hr string, value string)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table nzhang_part4 (key string) partitioned by (ds
string, hr string, value string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@nzhang_part4
+FAILED: Error in semantic analysis: Partition columns in partition
specification are not the same as that defined in the table schema. The names
and orders have to be exactly the same. Partition columns in the table schema
are: (ds, hr, value), while the partitions specified in the query are: (value,
ds, hr).
