Author: daijy
Date: Wed May 26 18:33:09 2010
New Revision: 948526
URL: http://svn.apache.org/viewvc?rev=948526&view=rev
Log:
PIG-1347: Clear up output directory for a failed job
Modified:
hadoop/pig/trunk/CHANGES.txt
hadoop/pig/trunk/src/org/apache/pig/PigServer.java
hadoop/pig/trunk/src/org/apache/pig/impl/io/FileLocalizer.java
hadoop/pig/trunk/test/org/apache/pig/test/TestMultiQueryLocal.java
Modified: hadoop/pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=948526&r1=948525&r2=948526&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Wed May 26 18:33:09 2010
@@ -70,6 +70,8 @@ PIG-1309: Map-side Cogroup (ashutoshc)
BUG FIXES
+PIG-1347: Clear up output directory for a failed job (daijy)
+
PIG-1419: Remove "user.name" from JobConf (daijy)
PIG-1359: bin/pig script does not pick up correct jar libraries (zjffdu)
Modified: hadoop/pig/trunk/src/org/apache/pig/PigServer.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/PigServer.java?rev=948526&r1=948525&r2=948526&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/PigServer.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/PigServer.java Wed May 26 18:33:09 2010
@@ -1066,7 +1066,6 @@ public class PigServer {
private List<ExecJob> executeCompiledLogicalPlan(LogicalPlan compiledLp)
throws ExecException {
PhysicalPlan pp = compilePp(compiledLp);
// execute using appropriate engine
- FileLocalizer.clearDeleteOnFail();
List<ExecJob> execJobs = pigContext.getExecutionEngine().execute(pp,
"job_pigexec_");
for (ExecJob execJob: execJobs) {
if (execJob.getStatus()==ExecJob.JOB_STATUS.FAILED) {
Modified: hadoop/pig/trunk/src/org/apache/pig/impl/io/FileLocalizer.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/impl/io/FileLocalizer.java?rev=948526&r1=948525&r2=948526&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/impl/io/FileLocalizer.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/impl/io/FileLocalizer.java Wed May 26
18:33:09 2010
@@ -430,19 +430,6 @@ public class FileLocalizer {
};
/**
- * Thread local deleteOnFail Stack to hold descriptors to be deleted upon
- * calling triggerDeleteOnFail. Use the deleteOnFail() method to access
this
- * stack.
- */
- private static ThreadLocal<Stack<ElementDescriptor>> deleteOnFail =
- new ThreadLocal<Stack<ElementDescriptor>>() {
-
- protected Stack<ElementDescriptor> initialValue() {
- return new Stack<ElementDescriptor>();
- }
- };
-
- /**
* Thread local relativeRoot ContainerDescriptor. Do not access this object
* directly, since it's lazy initialized in the relativeRoot(PigContext)
* method, which should be used instead.
@@ -460,14 +447,6 @@ public class FileLocalizer {
}
/**
- * Convenience accessor method to the deleteOnFail Stack bound to this
thread.
- * @return A Stack of ElementDescriptors that should be deleted upon
failure.
- */
- private static Stack<ElementDescriptor> deleteOnFail() {
- return deleteOnFail.get();
- }
-
- /**
* This method is only used by test code to reset state.
* @param initialized
*/
@@ -636,35 +615,6 @@ public class FileLocalizer {
FileLocalizer.r = r;
}
- public static void clearDeleteOnFail()
- {
- deleteOnFail().clear();
- }
- public static void registerDeleteOnFail(String filename, PigContext
pigContext) throws IOException
- {
- try {
- ElementDescriptor elem =
pigContext.getDfs().asElement(filename);
- if (!toDelete().contains(elem))
- deleteOnFail().push(elem);
- }
- catch (DataStorageException e) {
- log.warn("Unable to register output file to delete on failure: " +
filename);
- }
- }
- public static void triggerDeleteOnFail()
- {
- ElementDescriptor elem = null;
- while (!deleteOnFail().empty()) {
- try {
- elem = deleteOnFail().pop();
- if (elem.exists())
- elem.delete();
- }
- catch (IOException e) {
- log.warn("Unable to delete output file on failure: " +
elem.toString());
- }
- }
- }
/**
* Convert path from Windows convention to Unix convention. Invoked under
* cygwin.
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestMultiQueryLocal.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestMultiQueryLocal.java?rev=948526&r1=948525&r2=948526&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestMultiQueryLocal.java
(original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestMultiQueryLocal.java Wed May
26 18:33:09 2010
@@ -550,12 +550,10 @@ public class TestMultiQueryLocal extends
private boolean executePlan(PhysicalPlan pp) throws IOException {
boolean failed = true;
- FileLocalizer.clearDeleteOnFail();
List<ExecJob> jobs =
myPig.getPigContext().getExecutionEngine().execute(pp, "execute");
for (ExecJob job: jobs) {
failed = (job.getStatus() == ExecJob.JOB_STATUS.FAILED);
if (failed) {
- FileLocalizer.triggerDeleteOnFail();
break;
}
}
