Author: olga
Date: Wed Jul 9 17:10:15 2008
New Revision: 675385
URL: http://svn.apache.org/viewvc?rev=675385&view=rev
Log:
PIG-258: Pig should cleanup output directory of a failed query
Modified:
incubator/pig/branches/types/CHANGES.txt
incubator/pig/branches/types/src/org/apache/pig/PigServer.java
incubator/pig/branches/types/src/org/apache/pig/impl/io/FileLocalizer.java
incubator/pig/branches/types/src/org/apache/pig/impl/mapReduceLayer/MRCompiler.java
Modified: incubator/pig/branches/types/CHANGES.txt
URL:
http://svn.apache.org/viewvc/incubator/pig/branches/types/CHANGES.txt?rev=675385&r1=675384&r2=675385&view=diff
==============================================================================
--- incubator/pig/branches/types/CHANGES.txt (original)
+++ incubator/pig/branches/types/CHANGES.txt Wed Jul 9 17:10:15 2008
@@ -137,3 +137,5 @@
PIG-284: target for building source jar (oae via olgan)
PIG-294: string comparator unit tests (sms via pi_song)
+
+ PIG-258: cleaning up directories on failure (daijy via olgan)
Modified: incubator/pig/branches/types/src/org/apache/pig/PigServer.java
URL:
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/PigServer.java?rev=675385&r1=675384&r2=675385&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/PigServer.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/PigServer.java Wed Jul 9
17:10:15 2008
@@ -466,7 +466,11 @@
// typeCheckedLp.explain(System.out, System.err);
PhysicalPlan pp = compilePp(typeCheckedLp);
// execute using appropriate engine
- return pigContext.getExecutionEngine().execute(pp, "execute");
+ FileLocalizer.clearDeleteOnFail();
+ ExecJob execJob = pigContext.getExecutionEngine().execute(pp,
"execute");
+ if (execJob.getStatus()==ExecJob.JOB_STATUS.FAILED)
+ FileLocalizer.triggerDeleteOnFail();
+ return execJob;
}
private LogicalPlan compileLp(
Modified:
incubator/pig/branches/types/src/org/apache/pig/impl/io/FileLocalizer.java
URL:
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/io/FileLocalizer.java?rev=675385&r1=675384&r2=675385&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/io/FileLocalizer.java
(original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/io/FileLocalizer.java
Wed Jul 9 17:10:15 2008
@@ -303,6 +303,8 @@
static Stack<ElementDescriptor> toDelete =
new Stack<ElementDescriptor>();
+ static Stack<ElementDescriptor> deleteOnFail =
+ new Stack<ElementDescriptor>();
static Random r = new Random();
static ContainerDescriptor relativeRoot;
static boolean initialized = false;
@@ -439,5 +441,34 @@
public static void setR(Random r) {
FileLocalizer.r = r;
}
-
+
+ public static void clearDeleteOnFail()
+ {
+ deleteOnFail.clear();
+ }
+ public static void registerDeleteOnFail(String filename, PigContext
pigContext) throws IOException
+ {
+ try {
+ ElementDescriptor elem =
pigContext.getDfs().asElement(filename);
+ if (!toDelete.contains(elem))
+ deleteOnFail.push(elem);
+ }
+ catch (DataStorageException e) {
+ log.warn("Unable to register output file to delete on failure: " +
filename);
+ }
+ }
+ public static void triggerDeleteOnFail()
+ {
+ ElementDescriptor elem = null;
+ while (!deleteOnFail.empty()) {
+ try {
+ elem = deleteOnFail.pop();
+ if (elem.exists())
+ elem.delete();
+ }
+ catch (IOException e) {
+ log.warn("Unable to delete output file on failure: " +
elem.toString());
+ }
+ }
+ }
}
Modified:
incubator/pig/branches/types/src/org/apache/pig/impl/mapReduceLayer/MRCompiler.java
URL:
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/mapReduceLayer/MRCompiler.java?rev=675385&r1=675384&r2=675385&view=diff
==============================================================================
---
incubator/pig/branches/types/src/org/apache/pig/impl/mapReduceLayer/MRCompiler.java
(original)
+++
incubator/pig/branches/types/src/org/apache/pig/impl/mapReduceLayer/MRCompiler.java
Wed Jul 9 17:10:15 2008
@@ -180,6 +180,7 @@
public MROperPlan compile() throws IOException, PlanException,
VisitorException {
List<PhysicalOperator> leaves = plan.getLeaves();
POStore store = (POStore)leaves.get(0);
+ FileLocalizer.registerDeleteOnFail(store.getSFile().getFileName(),
pigContext);
compile(store);
return MRPlan;
