Author: daijy
Date: Thu Mar 21 19:01:11 2013
New Revision: 1459476
URL: http://svn.apache.org/r1459476
Log:
HCAT-628 Committer deletes the original partition when a duplicate partition
fails registration
Modified:
incubator/hcatalog/trunk/CHANGES.txt
incubator/hcatalog/trunk/core/src/main/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java
incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatDynamicPartitioned.java
Modified: incubator/hcatalog/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/CHANGES.txt?rev=1459476&r1=1459475&r2=1459476&view=diff
==============================================================================
--- incubator/hcatalog/trunk/CHANGES.txt (original)
+++ incubator/hcatalog/trunk/CHANGES.txt Thu Mar 21 19:01:11 2013
@@ -31,6 +31,9 @@ Trunk (unreleased changes)
OPTIMIZATIONS
BUG FIXES
+
+ HCAT-628 Committer deletes the original partition when a duplicate partition
fails registration (thiruvel via daijy)
+
HCAT-554 Loading data using HCatLoader() from a table on non default
namenode fails. (amalakar via avandana)
HCAT-629 First request always fails for webhcat (kamrul via gates)
Modified:
incubator/hcatalog/trunk/core/src/main/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/core/src/main/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java?rev=1459476&r1=1459475&r2=1459476&view=diff
==============================================================================
---
incubator/hcatalog/trunk/core/src/main/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java
(original)
+++
incubator/hcatalog/trunk/core/src/main/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java
Thu Mar 21 19:01:11 2013
@@ -681,14 +681,14 @@ class FileOutputCommitterContainer exten
// no harProcessor, regular operation
updateTableSchema(client, table, jobInfo.getOutputSchema());
LOG.info("HAR not is not being used. The table {} has new
partitions {}.", table.getTableName(), ptnInfos);
- partitionsAdded = partitionsToAdd;
- if (dynamicPartitioningUsed && (partitionsAdded.size()>0)){
+ if (dynamicPartitioningUsed && (partitionsToAdd.size()>0)){
Path src = new Path(ptnRootLocation);
moveTaskOutputs(fs, src, src, tblPath, true);
moveTaskOutputs(fs, src, src, tblPath, false);
fs.delete(src, true);
}
client.add_partitions(partitionsToAdd);
+ partitionsAdded = partitionsToAdd;
}
} catch (Exception e) {
if (partitionsAdded.size() > 0) {
Modified:
incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatDynamicPartitioned.java
URL:
http://svn.apache.org/viewvc/incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatDynamicPartitioned.java?rev=1459476&r1=1459475&r2=1459476&view=diff
==============================================================================
---
incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatDynamicPartitioned.java
(original)
+++
incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatDynamicPartitioned.java
Thu Mar 21 19:01:11 2013
@@ -160,6 +160,24 @@ public class TestHCatDynamicPartitioned
|| (ErrorType.ERROR_MOVE_FAILED ==
((HCatException) exc).getErrorType())
);
}
+
+ query = "show partitions " + tableName;
+ retCode = driver.run(query).getResponseCode();
+ if (retCode != 0) {
+ throw new Exception("Error " + retCode + " running query " +
query);
+ }
+ res = new ArrayList<String>();
+ driver.getResults(res);
+ assertEquals(NUM_PARTITIONS, res.size());
+
+ query = "select * from " + tableName;
+ retCode = driver.run(query).getResponseCode();
+ if (retCode != 0) {
+ throw new Exception("Error " + retCode + " running query " +
query);
+ }
+ res = new ArrayList<String>();
+ driver.getResults(res);
+ assertEquals(NUM_RECORDS, res.size());
}
//TODO 1.0 miniCluster is slow this test times out, make it work
