Author: daijy Date: Thu Jan 14 23:40:28 2010 New Revision: 899477 URL: http://svn.apache.org/viewvc?rev=899477&view=rev Log: PIG-1185: Data bags do not close spill files after using iterator to read tuples
Modified: hadoop/pig/trunk/CHANGES.txt hadoop/pig/trunk/src/org/apache/pig/data/DefaultDataBag.java hadoop/pig/trunk/src/org/apache/pig/data/DistinctDataBag.java hadoop/pig/trunk/src/org/apache/pig/data/InternalDistinctBag.java hadoop/pig/trunk/src/org/apache/pig/data/InternalSortedBag.java hadoop/pig/trunk/src/org/apache/pig/data/SortedDataBag.java Modified: hadoop/pig/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=899477&r1=899476&r2=899477&view=diff ============================================================================== --- hadoop/pig/trunk/CHANGES.txt (original) +++ hadoop/pig/trunk/CHANGES.txt Thu Jan 14 23:40:28 2010 @@ -340,6 +340,9 @@ PIG-1180: Piggybank should compile even if we only have "pig-withouthadoop.jar" but no "pig.jar" in the pig home directory (daijy) +PIG-1185: Data bags do not close spill files after using iterator to read +tuples (yinghe via daijy) + Release 0.5.0 INCOMPATIBLE CHANGES Modified: hadoop/pig/trunk/src/org/apache/pig/data/DefaultDataBag.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/data/DefaultDataBag.java?rev=899477&r1=899476&r2=899477&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/data/DefaultDataBag.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/data/DefaultDataBag.java Thu Jan 14 23:40:28 2010 @@ -242,6 +242,11 @@ } catch (EOFException eof) { // Fall through to the next case where we find the // next file, or go to memory + try { + mIn.close(); + }catch(IOException e) { + log.warn("Failed to close spill file.", e); + } } catch (IOException ioe) { String msg = "Unable to read our spill file."; log.fatal(msg, ioe); Modified: hadoop/pig/trunk/src/org/apache/pig/data/DistinctDataBag.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/data/DistinctDataBag.java?rev=899477&r1=899476&r2=899477&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/data/DistinctDataBag.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/data/DistinctDataBag.java Thu Jan 14 23:40:28 2010 @@ -434,6 +434,11 @@ // Out of tuples in this file. Set our slot in the // array to null so we don't keep trying to read from // this file. + try { + in.close(); + }catch(IOException e) { + log.warn("Failed to close spill file.", e); + } mStreams.set(fileNum, null); return; } catch (IOException ioe) { @@ -522,6 +527,7 @@ t.write(out); } out.flush(); + out.close(); } catch (IOException ioe) { String msg = "Unable to find our spill file."; log.fatal(msg, ioe); Modified: hadoop/pig/trunk/src/org/apache/pig/data/InternalDistinctBag.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/data/InternalDistinctBag.java?rev=899477&r1=899476&r2=899477&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/data/InternalDistinctBag.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/data/InternalDistinctBag.java Thu Jan 14 23:40:28 2010 @@ -409,6 +409,11 @@ // Out of tuples in this file. Set our slot in the // array to null so we don't keep trying to read from // this file. + try { + in.close(); + }catch(IOException e) { + log.warn("Failed to close spill file.", e); + } mStreams.set(fileNum, null); return; } catch (IOException ioe) { @@ -497,6 +502,7 @@ t.write(out); } out.flush(); + out.close(); } catch (IOException ioe) { String msg = "Unable to find our spill file."; log.fatal(msg, ioe); Modified: hadoop/pig/trunk/src/org/apache/pig/data/InternalSortedBag.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/data/InternalSortedBag.java?rev=899477&r1=899476&r2=899477&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/data/InternalSortedBag.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/data/InternalSortedBag.java Thu Jan 14 23:40:28 2010 @@ -404,6 +404,11 @@ // Out of tuples in this file. Set our slot in the // array to null so we don't keep trying to read from // this file. + try { + in.close(); + }catch(IOException e) { + log.warn("Failed to close spill file.", e); + } mStreams.set(fileNum, null); } catch (IOException ioe) { String msg = "Unable to find our spill file."; @@ -488,6 +493,7 @@ t.write(out); } out.flush(); + out.close(); } catch (IOException ioe) { String msg = "Unable to find our spill file."; log.fatal(msg, ioe); Modified: hadoop/pig/trunk/src/org/apache/pig/data/SortedDataBag.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/data/SortedDataBag.java?rev=899477&r1=899476&r2=899477&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/data/SortedDataBag.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/data/SortedDataBag.java Thu Jan 14 23:40:28 2010 @@ -396,6 +396,11 @@ // Out of tuples in this file. Set our slot in the // array to null so we don't keep trying to read from // this file. + try { + in.close(); + }catch(IOException e) { + log.warn("Failed to close spill file.", e); + } mStreams.set(fileNum, null); } catch (IOException ioe) { String msg = "Unable to find our spill file."; @@ -480,6 +485,7 @@ t.write(out); } out.flush(); + out.close(); } catch (IOException ioe) { String msg = "Unable to find our spill file."; log.fatal(msg, ioe);