Author: xuefu
Date: Mon Dec 16 15:03:45 2013
New Revision: 1551221
URL: http://svn.apache.org/r1551221
Log:
HIVE-6022: Load statements with incorrect order of partitions put input files
to unreadable places (Teruyoshi Zenmyo via Xuefu)
Added:
hive/trunk/ql/src/test/queries/clientpositive/loadpart2.q
hive/trunk/ql/src/test/results/clientpositive/loadpart2.q.out
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
hive/trunk/ql/src/test/results/clientnegative/dyn_part4.q.out
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java?rev=1551221&r1=1551220&r2=1551221&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
Mon Dec 16 15:03:45 2013
@@ -750,7 +750,7 @@ public abstract class BaseSemanticAnalyz
ASTNode partspec = (ASTNode) ast.getChild(1);
partitions = new ArrayList<Partition>();
// partSpec is a mapping from partition column name to its value.
- partSpec = new LinkedHashMap<String, String>(partspec.getChildCount());
+ Map<String, String> tmpPartSpec = new HashMap<String,
String>(partspec.getChildCount());
for (int i = 0; i < partspec.getChildCount(); ++i) {
ASTNode partspec_val = (ASTNode) partspec.getChild(i);
String val = null;
@@ -765,15 +765,21 @@ public abstract class BaseSemanticAnalyz
} else { // in the form of T partition (ds="2010-03-03")
val = stripQuotes(partspec_val.getChild(1).getText());
}
- partSpec.put(colName, val);
+ tmpPartSpec.put(colName, val);
}
// check if the columns, as well as value types in the partition()
clause are valid
- validatePartSpec(tableHandle, partSpec, ast, conf);
+ validatePartSpec(tableHandle, tmpPartSpec, ast, conf);
+
+ List<FieldSchema> parts = tableHandle.getPartitionKeys();
+ partSpec = new LinkedHashMap<String, String>(partspec.getChildCount());
+ for (FieldSchema fs : parts) {
+ String partKey = fs.getName();
+ partSpec.put(partKey, tmpPartSpec.get(partKey));
+ }
// check if the partition spec is valid
if (numDynParts > 0) {
- List<FieldSchema> parts = tableHandle.getPartitionKeys();
int numStaPart = parts.size() - numDynParts;
if (numStaPart == 0 &&
conf.getVar(HiveConf.ConfVars.DYNAMICPARTITIONINGMODE).equalsIgnoreCase("strict"))
{
Added: hive/trunk/ql/src/test/queries/clientpositive/loadpart2.q
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/loadpart2.q?rev=1551221&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/loadpart2.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/loadpart2.q Mon Dec 16
15:03:45 2013
@@ -0,0 +1,9 @@
+
+create table hive_test ( col1 string ) partitioned by ( pcol1 string , pcol2
string) stored as textfile;
+load data local inpath '../../data/files/test.dat' overwrite into table
hive_test partition (pcol1='part1',pcol2='part1') ;
+load data local inpath '../../data/files/test.dat' overwrite into table
hive_test partition (pcol2='part2',pcol1='part2') ;
+select * from hive_test where pcol1='part1' and pcol2='part1';
+select * from hive_test where pcol1='part2' and pcol2='part2';
+
+
+
Modified: hive/trunk/ql/src/test/results/clientnegative/dyn_part4.q.out
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/dyn_part4.q.out?rev=1551221&r1=1551220&r2=1551221&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientnegative/dyn_part4.q.out (original)
+++ hive/trunk/ql/src/test/results/clientnegative/dyn_part4.q.out Mon Dec 16
15:03:45 2013
@@ -3,4 +3,4 @@ PREHOOK: type: CREATETABLE
POSTHOOK: query: create table nzhang_part4 (key string) partitioned by (ds
string, hr string, value string)
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: default@nzhang_part4
-FAILED: SemanticException [Error 10125]: Partition columns in partition
specification are not the same as that defined in the table schema. The names
and orders have to be exactly the same. Partition columns in the table schema
are: (ds, hr, value), while the partitions specified in the query are: (value,
ds, hr).
+FAILED: SemanticException [Error 10094]: Line 3:46 Dynamic partition cannot be
the parent of a static partition 'hr'
Added: hive/trunk/ql/src/test/results/clientpositive/loadpart2.q.out
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/loadpart2.q.out?rev=1551221&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/loadpart2.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/loadpart2.q.out Mon Dec 16
15:03:45 2013
@@ -0,0 +1,51 @@
+PREHOOK: query: create table hive_test ( col1 string ) partitioned by ( pcol1
string , pcol2 string) stored as textfile
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table hive_test ( col1 string ) partitioned by ( pcol1
string , pcol2 string) stored as textfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@hive_test
+PREHOOK: query: load data local inpath '../../data/files/test.dat' overwrite
into table hive_test partition (pcol1='part1',pcol2='part1')
+PREHOOK: type: LOAD
+PREHOOK: Output: default@hive_test
+POSTHOOK: query: load data local inpath '../../data/files/test.dat' overwrite
into table hive_test partition (pcol1='part1',pcol2='part1')
+POSTHOOK: type: LOAD
+POSTHOOK: Output: default@hive_test
+POSTHOOK: Output: default@hive_test@pcol1=part1/pcol2=part1
+PREHOOK: query: load data local inpath '../../data/files/test.dat' overwrite
into table hive_test partition (pcol2='part2',pcol1='part2')
+PREHOOK: type: LOAD
+PREHOOK: Output: default@hive_test
+POSTHOOK: query: load data local inpath '../../data/files/test.dat' overwrite
into table hive_test partition (pcol2='part2',pcol1='part2')
+POSTHOOK: type: LOAD
+POSTHOOK: Output: default@hive_test
+POSTHOOK: Output: default@hive_test@pcol1=part2/pcol2=part2
+PREHOOK: query: select * from hive_test where pcol1='part1' and pcol2='part1'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hive_test
+PREHOOK: Input: default@hive_test@pcol1=part1/pcol2=part1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from hive_test where pcol1='part1' and pcol2='part1'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hive_test
+POSTHOOK: Input: default@hive_test@pcol1=part1/pcol2=part1
+#### A masked pattern was here ####
+1 part1 part1
+2 part1 part1
+3 part1 part1
+4 part1 part1
+5 part1 part1
+6 part1 part1
+PREHOOK: query: select * from hive_test where pcol1='part2' and pcol2='part2'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hive_test
+PREHOOK: Input: default@hive_test@pcol1=part2/pcol2=part2
+#### A masked pattern was here ####
+POSTHOOK: query: select * from hive_test where pcol1='part2' and pcol2='part2'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hive_test
+POSTHOOK: Input: default@hive_test@pcol1=part2/pcol2=part2
+#### A masked pattern was here ####
+1 part2 part2
+2 part2 part2
+3 part2 part2
+4 part2 part2
+5 part2 part2
+6 part2 part2
