Author: amareshwari
Date: Thu Aug 25 11:38:02 2011
New Revision: 1161516
URL: http://svn.apache.org/viewvc?rev=1161516&view=rev
Log:
HIVE-2384. Fixes overwriting of files in case of import of multiple partitions
from a partitioned table with external location. (Krishna Kumar via amareshwari)
Added:
hive/trunk/data/files/test2.dat
hive/trunk/ql/src/test/queries/clientpositive/exim_19_00_part_external_location.q
hive/trunk/ql/src/test/results/clientpositive/exim_19_00_part_external_location.q.out
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
Added: hive/trunk/data/files/test2.dat
URL:
http://svn.apache.org/viewvc/hive/trunk/data/files/test2.dat?rev=1161516&view=auto
==============================================================================
--- hive/trunk/data/files/test2.dat (added)
+++ hive/trunk/data/files/test2.dat Thu Aug 25 11:38:02 2011
@@ -0,0 +1,6 @@
+101
+202
+303
+404
+505
+606
\ No newline at end of file
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java?rev=1161516&r1=1161515&r2=1161516&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
Thu Aug 25 11:38:02 2011
@@ -309,7 +309,8 @@ public class ImportSemanticAnalyzer exte
Warehouse.makePartPath(addPartitionDesc.getPartSpec()));
}
} else {
- tgtPath = new Path(tblDesc.getLocation());
+ tgtPath = new Path(tblDesc.getLocation(),
+ Warehouse.makePartPath(addPartitionDesc.getPartSpec()));
}
checkTargetLocationEmpty(fs, tgtPath);
addPartitionDesc.setLocation(tgtPath.toString());
Added:
hive/trunk/ql/src/test/queries/clientpositive/exim_19_00_part_external_location.q
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/exim_19_00_part_external_location.q?rev=1161516&view=auto
==============================================================================
---
hive/trunk/ql/src/test/queries/clientpositive/exim_19_00_part_external_location.q
(added)
+++
hive/trunk/ql/src/test/queries/clientpositive/exim_19_00_part_external_location.q
Thu Aug 25 11:38:02 2011
@@ -0,0 +1,36 @@
+set hive.test.mode=true;
+set hive.test.mode.prefix=;
+set hive.test.mode.nosamplelist=exim_department,exim_employee;
+
+create table exim_employee ( emp_id int comment "employee id")
+ comment "employee table"
+ partitioned by (emp_country string comment "two char iso code",
emp_state string comment "free text")
+ stored as textfile
+ tblproperties("creator"="krishna");
+load data local inpath "../data/files/test.dat"
+ into table exim_employee partition (emp_country="in", emp_state="tn");
+load data local inpath "../data/files/test2.dat"
+ into table exim_employee partition (emp_country="in", emp_state="ka");
+!rm -rf ../build/ql/test/data/exports/exim_employee;
+export table exim_employee to 'ql/test/data/exports/exim_employee';
+drop table exim_employee;
+
+create database importer;
+use importer;
+
+!rm -rf ../build/ql/test/data/tablestore/exim_employee;
+
+import external table exim_employee
+ from 'ql/test/data/exports/exim_employee'
+ location 'ql/test/data/tablestore/exim_employee';
+describe extended exim_employee;
+show table extended like exim_employee;
+show table extended like exim_employee partition (emp_country="in",
emp_state="tn");
+show table extended like exim_employee partition (emp_country="in",
emp_state="ka");
+!rm -rf ../build/ql/test/data/exports/exim_employee;
+select * from exim_employee;
+!rm -rf ../build/ql/test/data/tablestore/exim_employee;
+select * from exim_employee;
+drop table exim_employee;
+
+drop database importer;
Added:
hive/trunk/ql/src/test/results/clientpositive/exim_19_00_part_external_location.q.out
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/exim_19_00_part_external_location.q.out?rev=1161516&view=auto
==============================================================================
---
hive/trunk/ql/src/test/results/clientpositive/exim_19_00_part_external_location.q.out
(added)
+++
hive/trunk/ql/src/test/results/clientpositive/exim_19_00_part_external_location.q.out
Thu Aug 25 11:38:02 2011
@@ -0,0 +1,178 @@
+PREHOOK: query: create table exim_employee ( emp_id int comment "employee id")
+ comment "employee table"
+ partitioned by (emp_country string comment "two char iso code",
emp_state string comment "free text")
+ stored as textfile
+ tblproperties("creator"="krishna")
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table exim_employee ( emp_id int comment "employee
id")
+ comment "employee table"
+ partitioned by (emp_country string comment "two char iso code",
emp_state string comment "free text")
+ stored as textfile
+ tblproperties("creator"="krishna")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@exim_employee
+PREHOOK: query: load data local inpath "../data/files/test.dat"
+ into table exim_employee partition (emp_country="in", emp_state="tn")
+PREHOOK: type: LOAD
+PREHOOK: Output: default@exim_employee
+POSTHOOK: query: load data local inpath "../data/files/test.dat"
+ into table exim_employee partition (emp_country="in", emp_state="tn")
+POSTHOOK: type: LOAD
+POSTHOOK: Output: default@exim_employee
+POSTHOOK: Output: default@exim_employee@emp_country=in/emp_state=tn
+PREHOOK: query: load data local inpath "../data/files/test2.dat"
+ into table exim_employee partition (emp_country="in", emp_state="ka")
+PREHOOK: type: LOAD
+PREHOOK: Output: default@exim_employee
+POSTHOOK: query: load data local inpath "../data/files/test2.dat"
+ into table exim_employee partition (emp_country="in", emp_state="ka")
+POSTHOOK: type: LOAD
+POSTHOOK: Output: default@exim_employee
+POSTHOOK: Output: default@exim_employee@emp_country=in/emp_state=ka
+PREHOOK: query: export table exim_employee to
'ql/test/data/exports/exim_employee'
+PREHOOK: type: EXPORT
+PREHOOK: Input: default@exim_employee@emp_country=in/emp_state=ka
+PREHOOK: Input: default@exim_employee@emp_country=in/emp_state=tn
+PREHOOK: Output:
pfile:/Users/krishnak/Projects/hdp/sources/hive-git-apache/build/ql/test/data/exports/exim_employee
+POSTHOOK: query: export table exim_employee to
'ql/test/data/exports/exim_employee'
+POSTHOOK: type: EXPORT
+POSTHOOK: Input: default@exim_employee@emp_country=in/emp_state=ka
+POSTHOOK: Input: default@exim_employee@emp_country=in/emp_state=tn
+POSTHOOK: Output:
pfile:/Users/krishnak/Projects/hdp/sources/hive-git-apache/build/ql/test/data/exports/exim_employee
+PREHOOK: query: drop table exim_employee
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@exim_employee
+PREHOOK: Output: default@exim_employee
+POSTHOOK: query: drop table exim_employee
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@exim_employee
+POSTHOOK: Output: default@exim_employee
+PREHOOK: query: create database importer
+PREHOOK: type: CREATEDATABASE
+POSTHOOK: query: create database importer
+POSTHOOK: type: CREATEDATABASE
+PREHOOK: query: use importer
+PREHOOK: type: SWITCHDATABASE
+POSTHOOK: query: use importer
+POSTHOOK: type: SWITCHDATABASE
+PREHOOK: query: import external table exim_employee
+ from 'ql/test/data/exports/exim_employee'
+ location 'ql/test/data/tablestore/exim_employee'
+PREHOOK: type: IMPORT
+POSTHOOK: query: import external table exim_employee
+ from 'ql/test/data/exports/exim_employee'
+ location 'ql/test/data/tablestore/exim_employee'
+POSTHOOK: type: IMPORT
+POSTHOOK: Output: importer@exim_employee
+POSTHOOK: Output: importer@exim_employee@emp_country=in/emp_state=ka
+POSTHOOK: Output: importer@exim_employee@emp_country=in/emp_state=tn
+PREHOOK: query: describe extended exim_employee
+PREHOOK: type: DESCTABLE
+POSTHOOK: query: describe extended exim_employee
+POSTHOOK: type: DESCTABLE
+emp_id int employee id
+emp_country string two char iso code
+emp_state string free text
+
+Detailed Table Information Table(tableName:exim_employee, dbName:importer,
owner:krishnak, createTime:1313569893, lastAccessTime:0, retention:0,
sd:StorageDescriptor(cols:[FieldSchema(name:emp_id, type:int, comment:employee
id), FieldSchema(name:emp_country, type:string, comment:two char iso code),
FieldSchema(name:emp_state, type:string, comment:free text)],
location:pfile:/Users/krishnak/Projects/hdp/sources/hive-git-apache/build/ql/test/data/tablestore/exim_employee,
inputFormat:org.apache.hadoop.mapred.TextInputFormat,
outputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat,
compressed:false, numBuckets:-1, serdeInfo:SerDeInfo(name:null,
serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe,
parameters:{serialization.format=1}), bucketCols:[], sortCols:[],
parameters:{}), partitionKeys:[FieldSchema(name:emp_country, type:string,
comment:two char iso code), FieldSchema(name:emp_state, type:string,
comment:free text)], parameters:{EXTERNAL=TRU
E, transient_lastDdlTime=1313569893, comment=employee table, creator=krishna},
viewOriginalText:null, viewExpandedText:null, tableType:EXTERNAL_TABLE)
+PREHOOK: query: show table extended like exim_employee
+PREHOOK: type: SHOW_TABLESTATUS
+POSTHOOK: query: show table extended like exim_employee
+POSTHOOK: type: SHOW_TABLESTATUS
+tableName:exim_employee
+owner:krishnak
+location:pfile:/Users/krishnak/Projects/hdp/sources/hive-git-apache/build/ql/test/data/tablestore/exim_employee
+inputformat:org.apache.hadoop.mapred.TextInputFormat
+outputformat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+columns:struct columns { i32 emp_id}
+partitioned:true
+partitionColumns:struct partition_columns { string emp_country, string
emp_state}
+totalNumberFiles:2
+totalFileSize:34
+maxFileSize:23
+minFileSize:11
+lastAccessTime:0
+lastUpdateTime:1313569893000
+
+PREHOOK: query: show table extended like exim_employee partition
(emp_country="in", emp_state="tn")
+PREHOOK: type: SHOW_TABLESTATUS
+POSTHOOK: query: show table extended like exim_employee partition
(emp_country="in", emp_state="tn")
+POSTHOOK: type: SHOW_TABLESTATUS
+tableName:exim_employee
+owner:krishnak
+location:pfile:/Users/krishnak/Projects/hdp/sources/hive-git-apache/build/ql/test/data/tablestore/exim_employee/emp_country=in/emp_state=tn
+inputformat:org.apache.hadoop.mapred.TextInputFormat
+outputformat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+columns:struct columns { i32 emp_id}
+partitioned:true
+partitionColumns:struct partition_columns { string emp_country, string
emp_state}
+totalNumberFiles:1
+totalFileSize:11
+maxFileSize:11
+minFileSize:11
+lastAccessTime:0
+lastUpdateTime:1313569893000
+
+PREHOOK: query: show table extended like exim_employee partition
(emp_country="in", emp_state="ka")
+PREHOOK: type: SHOW_TABLESTATUS
+POSTHOOK: query: show table extended like exim_employee partition
(emp_country="in", emp_state="ka")
+POSTHOOK: type: SHOW_TABLESTATUS
+tableName:exim_employee
+owner:krishnak
+location:pfile:/Users/krishnak/Projects/hdp/sources/hive-git-apache/build/ql/test/data/tablestore/exim_employee/emp_country=in/emp_state=ka
+inputformat:org.apache.hadoop.mapred.TextInputFormat
+outputformat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+columns:struct columns { i32 emp_id}
+partitioned:true
+partitionColumns:struct partition_columns { string emp_country, string
emp_state}
+totalNumberFiles:1
+totalFileSize:23
+maxFileSize:23
+minFileSize:23
+lastAccessTime:0
+lastUpdateTime:1313569893000
+
+PREHOOK: query: select * from exim_employee
+PREHOOK: type: QUERY
+PREHOOK: Input: importer@exim_employee@emp_country=in/emp_state=ka
+PREHOOK: Input: importer@exim_employee@emp_country=in/emp_state=tn
+PREHOOK: Output:
file:/var/folders/67/67R3POPtF90VG63KSmCbcU++F0U/-Tmp-/krishnak/hive_2011-08-17_01-31-36_679_7845969666559300040/-mr-10000
+POSTHOOK: query: select * from exim_employee
+POSTHOOK: type: QUERY
+POSTHOOK: Input: importer@exim_employee@emp_country=in/emp_state=ka
+POSTHOOK: Input: importer@exim_employee@emp_country=in/emp_state=tn
+POSTHOOK: Output:
file:/var/folders/67/67R3POPtF90VG63KSmCbcU++F0U/-Tmp-/krishnak/hive_2011-08-17_01-31-36_679_7845969666559300040/-mr-10000
+101 in ka
+202 in ka
+303 in ka
+404 in ka
+505 in ka
+606 in ka
+1 in tn
+2 in tn
+3 in tn
+4 in tn
+5 in tn
+6 in tn
+PREHOOK: query: select * from exim_employee
+PREHOOK: type: QUERY
+PREHOOK: Input: importer@exim_employee@emp_country=in/emp_state=ka
+PREHOOK: Input: importer@exim_employee@emp_country=in/emp_state=tn
+PREHOOK: Output:
file:/var/folders/67/67R3POPtF90VG63KSmCbcU++F0U/-Tmp-/krishnak/hive_2011-08-17_01-31-37_400_8599999083947155620/-mr-10000
+POSTHOOK: query: select * from exim_employee
+POSTHOOK: type: QUERY
+POSTHOOK: Input: importer@exim_employee@emp_country=in/emp_state=ka
+POSTHOOK: Input: importer@exim_employee@emp_country=in/emp_state=tn
+POSTHOOK: Output:
file:/var/folders/67/67R3POPtF90VG63KSmCbcU++F0U/-Tmp-/krishnak/hive_2011-08-17_01-31-37_400_8599999083947155620/-mr-10000
+PREHOOK: query: drop table exim_employee
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: importer@exim_employee
+PREHOOK: Output: importer@exim_employee
+POSTHOOK: query: drop table exim_employee
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: importer@exim_employee
+POSTHOOK: Output: importer@exim_employee
+PREHOOK: query: drop database importer
+PREHOOK: type: DROPDATABASE
+POSTHOOK: query: drop database importer
+POSTHOOK: type: DROPDATABASE
