Author: rding
Date: Thu Mar 18 22:49:12 2010
New Revision: 925034
URL: http://svn.apache.org/viewvc?rev=925034&view=rev
Log:
PIG-1266: Show spill count on the pig console at the end of the job
Modified:
hadoop/pig/trunk/CHANGES.txt
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java
Modified: hadoop/pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=925034&r1=925033&r2=925034&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Thu Mar 18 22:49:12 2010
@@ -157,6 +157,9 @@ OPTIMIZATIONS
BUG FIXES
+PIG-1266: Show spill count on the pig console at the end of the job (sriranjan
+via rding)
+
PIG-1296: Skewed join fail due to negative partition index (daijy)
PIG-1293: pig wrapper script tends to fail if pig is in the path and PIG_HOME
Modified:
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java?rev=925034&r1=925033&r2=925034&view=diff
==============================================================================
---
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
(original)
+++
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
Thu Mar 18 22:49:12 2010
@@ -336,6 +336,16 @@ public class MapReduceLauncher extends L
} else {
log.info("Bytes written : " + stats.getBytesWritten());
}
+ if(stats.getSMMSpillCount()==-1) {
+ log.info("Spillable Memory Manager spill count : Unable to
determine spillable memory manager spill count");
+ } else {
+ log.info("Spillable Memory Manager spill count : " +
stats.getSMMSpillCount());
+ }
+ if(stats.getProactiveSpillCount() == -1) {
+ log.info("Proactive spill count : Unable to determine
proactive spill count");
+ } else {
+ log.info("Proactive spill count : " +
stats.getProactiveSpillCount());
+ }
}
if (!failed) {
Modified: hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java?rev=925034&r1=925033&r2=925034&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/PigStats.java Thu Mar 18
22:49:12 2010
@@ -316,6 +316,21 @@ public class PigStats {
return spillCount;
}
+ public long getProactiveSpillCount() {
+ long spillCount = 0;
+ for (String jid : rootJobIDs) {
+ Map<String, String> jobStats = stats.get(jid);
+ if (jobStats == null) continue;
+ if
(Long.parseLong(jobStats.get("PIG_STATS_PROACTIVE_SPILL_COUNT"))==-1L)
+ {
+ spillCount = -1L;
+ break;
+ }
+ spillCount +=
Long.parseLong(jobStats.get("PIG_STATS_PROACTIVE_SPILL_COUNT"));
+ }
+ return spillCount;
+ }
+
private long getLocalBytesWritten() {
for(PhysicalOperator op : php.getLeaves())
return
Long.parseLong(stats.get(op.toString()).get("PIG_STATS_LOCAL_BYTES_WRITTEN"));
