Author: rding Date: Thu Mar 18 22:49:12 2010 New Revision: 925034 URL: http://svn.apache.org/viewvc?rev=925034&view=rev Log: PIG-1266: Show spill count on the pig console at the end of the job
Modified: hadoop/pig/trunk/CHANGES.txt hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java Modified: hadoop/pig/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=925034&r1=925033&r2=925034&view=diff ============================================================================== --- hadoop/pig/trunk/CHANGES.txt (original) +++ hadoop/pig/trunk/CHANGES.txt Thu Mar 18 22:49:12 2010 @@ -157,6 +157,9 @@ OPTIMIZATIONS BUG FIXES +PIG-1266: Show spill count on the pig console at the end of the job (sriranjan +via rding) + PIG-1296: Skewed join fail due to negative partition index (daijy) PIG-1293: pig wrapper script tends to fail if pig is in the path and PIG_HOME Modified: hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java?rev=925034&r1=925033&r2=925034&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java Thu Mar 18 22:49:12 2010 @@ -336,6 +336,16 @@ public class MapReduceLauncher extends L } else { log.info("Bytes written : " + stats.getBytesWritten()); } + if(stats.getSMMSpillCount()==-1) { + log.info("Spillable Memory Manager spill count : Unable to determine spillable memory manager spill count"); + } else { + log.info("Spillable Memory Manager spill count : " + stats.getSMMSpillCount()); + } + if(stats.getProactiveSpillCount() == -1) { + log.info("Proactive spill count : Unable to determine proactive spill count"); + } else { + log.info("Proactive spill count : " + stats.getProactiveSpillCount()); + } } if (!failed) { Modified: hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java?rev=925034&r1=925033&r2=925034&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java Thu Mar 18 22:49:12 2010 @@ -316,6 +316,21 @@ public class PigStats { return spillCount; } + public long getProactiveSpillCount() { + long spillCount = 0; + for (String jid : rootJobIDs) { + Map<String, String> jobStats = stats.get(jid); + if (jobStats == null) continue; + if (Long.parseLong(jobStats.get("PIG_STATS_PROACTIVE_SPILL_COUNT"))==-1L) + { + spillCount = -1L; + break; + } + spillCount += Long.parseLong(jobStats.get("PIG_STATS_PROACTIVE_SPILL_COUNT")); + } + return spillCount; + } + private long getLocalBytesWritten() { for(PhysicalOperator op : php.getLeaves()) return Long.parseLong(stats.get(op.toString()).get("PIG_STATS_LOCAL_BYTES_WRITTEN"));