Author: thejas
Date: Thu Aug 19 20:14:12 2010
New Revision: 987291
URL: http://svn.apache.org/viewvc?rev=987291&view=rev
Log:
PIG-1466: Improve log messages for memory usage
Modified:
hadoop/pig/trunk/CHANGES.txt
hadoop/pig/trunk/src/org/apache/pig/impl/util/SpillableMemoryManager.java
Modified: hadoop/pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=987291&r1=987290&r2=987291&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Thu Aug 19 20:14:12 2010
@@ -26,6 +26,8 @@ PIG-1249: Safe-guards against misconfigu
IMPROVEMENTS
+PIG-1466: Improve log messages for memory usage (thejas)
+
PIG-1404: added PigUnit, a framework fo building unit tests of Pig Latin
scripts (romainr via gates)
PIG-1452: to remove hadoop20.jar from lib and use hadoop from the apache maven
Modified:
hadoop/pig/trunk/src/org/apache/pig/impl/util/SpillableMemoryManager.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/impl/util/SpillableMemoryManager.java?rev=987291&r1=987290&r2=987291&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/impl/util/SpillableMemoryManager.java
(original)
+++ hadoop/pig/trunk/src/org/apache/pig/impl/util/SpillableMemoryManager.java
Thu Aug 19 20:14:12 2010
@@ -72,6 +72,12 @@ public class SpillableMemoryManager impl
// "collection threshold exceeded" notifications
private static double collectionMemoryThresholdFraction = 0.5;
+ // log notification on usage threshold exceeded only the first time
+ private boolean firstUsageThreshExceededLogged = false;
+
+ // log notification on collection threshold exceeded only the first time
+ private boolean firstCollectionThreshExceededLogged = false;
+
public SpillableMemoryManager() {
((NotificationEmitter)ManagementFactory.getMemoryMXBean()).addNotificationListener(this,
null, null);
List<MemoryPoolMXBean> mpbeans =
ManagementFactory.getMemoryPoolMXBeans();
@@ -140,11 +146,30 @@ public class SpillableMemoryManager impl
if(n.getType().equals(MemoryNotificationInfo.MEMORY_THRESHOLD_EXCEEDED)) {
long threshold = (long)(info.getUsage().getMax() *
memoryThresholdFraction);
toFree = info.getUsage().getUsed() - threshold + (long)(threshold
* 0.5);
- log.info("low memory handler called (Usage threshold exceeded) " +
info.getUsage());
+
+ //log
+ String msg = "memory handler call- Usage threshold "
+ + info.getUsage();
+ if(!firstUsageThreshExceededLogged){
+ log.info("first " + msg);
+ firstUsageThreshExceededLogged = true;
+ }else{
+ log.debug(msg);
+ }
} else { // MEMORY_COLLECTION_THRESHOLD_EXCEEDED CASE
long threshold = (long)(info.getUsage().getMax() *
collectionMemoryThresholdFraction);
toFree = info.getUsage().getUsed() - threshold + (long)(threshold
* 0.5);
- log.info("low memory handler called (Collection threshold
exceeded) " + info.getUsage());
+
+ //log
+ String msg = "memory handler call - Collection threshold "
+ + info.getUsage();
+ if(!firstCollectionThreshExceededLogged){
+ log.info("first " + msg);
+ firstCollectionThreshExceededLogged = true;
+ }else{
+ log.debug(msg);
+ }
+
}
if (toFree < 0) {
@@ -195,6 +220,7 @@ public class SpillableMemoryManager impl
}
});
long estimatedFreed = 0;
+ int numObjSpilled = 0;
boolean invokeGC = false;
for (i = spillables.iterator(); i.hasNext();) {
Spillable s = i.next().get();
@@ -213,6 +239,7 @@ public class SpillableMemoryManager impl
break ;
}
s.spill();
+ numObjSpilled++;
estimatedFreed += toBeFreed;
accumulatedFreeSize += toBeFreed;
// This should significantly reduce the number of small files
@@ -226,14 +253,19 @@ public class SpillableMemoryManager impl
invokeGC = true;
break;
}
- }
-
+ }
/* Poke the GC again to see if we successfully freed enough memory
*/
if(invokeGC) {
System.gc();
// now that we have invoked the GC, reset accumulatedFreeSize
accumulatedFreeSize = 0;
}
+ if(estimatedFreed > 0){
+ String msg = "Spilled an estimate of " + estimatedFreed +
+ " bytes from " + numObjSpilled + " objects. " +
info.getUsage();;
+ log.info(msg);
+ }
+
}
}
