Author: gates
Date: Wed Feb 6 12:56:43 2008
New Revision: 619151
URL: http://svn.apache.org/viewvc?rev=619151&view=rev
Log:
PIG-89: Fix DefaultDataBag, DistinctDataBag, SortedDataBag to close spill files
when they are done spilling.
Modified:
incubator/pig/trunk/CHANGES.txt
incubator/pig/trunk/src/org/apache/pig/data/DefaultDataBag.java
incubator/pig/trunk/src/org/apache/pig/data/DistinctDataBag.java
incubator/pig/trunk/src/org/apache/pig/data/SortedDataBag.java
Modified: incubator/pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/incubator/pig/trunk/CHANGES.txt?rev=619151&r1=619150&r2=619151&view=diff
==============================================================================
--- incubator/pig/trunk/CHANGES.txt (original)
+++ incubator/pig/trunk/CHANGES.txt Wed Feb 6 12:56:43 2008
@@ -83,3 +83,7 @@
PIG-87: Fix pig.pl to find java via JAVA_HOME instead of hardcoded
default
path. Also fix it to not die if pigclient.conf is missing. (craigm via
gates).
+
+ PIG-89: Fix DefaultDataBag, DistinctDataBag, SortedDataBag to close
spill
+ files when they are done spilling (contributions by craigm, breed, and
+ gates, committed by gates).
Modified: incubator/pig/trunk/src/org/apache/pig/data/DefaultDataBag.java
URL:
http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/data/DefaultDataBag.java?rev=619151&r1=619150&r2=619151&view=diff
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/data/DefaultDataBag.java (original)
+++ incubator/pig/trunk/src/org/apache/pig/data/DefaultDataBag.java Wed Feb 6
12:56:43 2008
@@ -65,8 +65,17 @@
// trying to read while I'm mucking with the container.
long spilled = 0;
synchronized (mContents) {
+ DataOutputStream out = null;
+ try {
+ out = getSpillFile();
+ } catch (IOException ioe) {
+ // Do not remove last file from spilled array. It was not
+ // added as File.createTmpFile threw an IOException
+ PigLogger.getLogger().error(
+ "Unable to create tmp file to spill to disk", ioe);
+ return 0;
+ }
try {
- DataOutputStream out = getSpillFile();
Iterator<Tuple> i = mContents.iterator();
while (i.hasNext()) {
i.next().write(out);
@@ -82,6 +91,14 @@
PigLogger.getLogger().error(
"Unable to spill contents to disk", ioe);
return 0;
+ } finally {
+ if (out != null) {
+ try {
+ out.close();
+ } catch (IOException e) {
+ PigLogger.getLogger().error("Error closing spill", e);
+ }
+ }
}
mContents.clear();
}
Modified: incubator/pig/trunk/src/org/apache/pig/data/DistinctDataBag.java
URL:
http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/data/DistinctDataBag.java?rev=619151&r1=619150&r2=619151&view=diff
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/data/DistinctDataBag.java (original)
+++ incubator/pig/trunk/src/org/apache/pig/data/DistinctDataBag.java Wed Feb 6
12:56:43 2008
@@ -102,8 +102,17 @@
// trying to read while I'm mucking with the container.
long spilled = 0;
synchronized (mContents) {
+ DataOutputStream out = null;
+ try {
+ out = getSpillFile();
+ } catch (IOException ioe) {
+ // Do not remove last file from spilled array. It was not
+ // added as File.createTmpFile threw an IOException
+ PigLogger.getLogger().error(
+ "Unable to create tmp file to spill to disk", ioe);
+ return 0;
+ }
try {
- DataOutputStream out = getSpillFile();
// If we've already started reading, then it will already be
// sorted into an array list. If not, we need to sort it
// before writing.
@@ -134,6 +143,14 @@
PigLogger.getLogger().error(
"Unable to spill contents to disk", ioe);
return 0;
+ } finally {
+ if (out != null) {
+ try {
+ out.close();
+ } catch (IOException e) {
+ PigLogger.getLogger().error("Error closing spill", e);
+ }
+ }
}
mContents.clear();
}
Modified: incubator/pig/trunk/src/org/apache/pig/data/SortedDataBag.java
URL:
http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/data/SortedDataBag.java?rev=619151&r1=619150&r2=619151&view=diff
==============================================================================
--- incubator/pig/trunk/src/org/apache/pig/data/SortedDataBag.java (original)
+++ incubator/pig/trunk/src/org/apache/pig/data/SortedDataBag.java Wed Feb 6
12:56:43 2008
@@ -102,8 +102,17 @@
// trying to read while I'm mucking with the container.
long spilled = 0;
synchronized (mContents) {
+ DataOutputStream out = null;
+ try {
+ out = getSpillFile();
+ } catch (IOException ioe) {
+ // Do not remove last file from spilled array. It was not
+ // added as File.createTmpFile threw an IOException
+ PigLogger.getLogger().error(
+ "Unable to create tmp file to spill to disk", ioe);
+ return 0;
+ }
try {
- DataOutputStream out = getSpillFile();
// Have to sort the data before we can dump it. It's bogus
// that we have to do this under the lock, but there's no way
// around it. If the reads alread started, then we've
@@ -129,6 +138,14 @@
PigLogger.getLogger().error(
"Unable to spill contents to disk", ioe);
return 0;
+ } finally {
+ if (out != null) {
+ try {
+ out.close();
+ } catch (IOException e) {
+ PigLogger.getLogger().error("Error closing spill", e);
+ }
+ }
}
mContents.clear();
}
