Author: thejas Date: Thu Aug 19 20:14:12 2010 New Revision: 987291 URL: http://svn.apache.org/viewvc?rev=987291&view=rev Log: PIG-1466: Improve log messages for memory usage
Modified: hadoop/pig/trunk/CHANGES.txt hadoop/pig/trunk/src/org/apache/pig/impl/util/SpillableMemoryManager.java Modified: hadoop/pig/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=987291&r1=987290&r2=987291&view=diff ============================================================================== --- hadoop/pig/trunk/CHANGES.txt (original) +++ hadoop/pig/trunk/CHANGES.txt Thu Aug 19 20:14:12 2010 @@ -26,6 +26,8 @@ PIG-1249: Safe-guards against misconfigu IMPROVEMENTS +PIG-1466: Improve log messages for memory usage (thejas) + PIG-1404: added PigUnit, a framework fo building unit tests of Pig Latin scripts (romainr via gates) PIG-1452: to remove hadoop20.jar from lib and use hadoop from the apache maven Modified: hadoop/pig/trunk/src/org/apache/pig/impl/util/SpillableMemoryManager.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/impl/util/SpillableMemoryManager.java?rev=987291&r1=987290&r2=987291&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/impl/util/SpillableMemoryManager.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/impl/util/SpillableMemoryManager.java Thu Aug 19 20:14:12 2010 @@ -72,6 +72,12 @@ public class SpillableMemoryManager impl // "collection threshold exceeded" notifications private static double collectionMemoryThresholdFraction = 0.5; + // log notification on usage threshold exceeded only the first time + private boolean firstUsageThreshExceededLogged = false; + + // log notification on collection threshold exceeded only the first time + private boolean firstCollectionThreshExceededLogged = false; + public SpillableMemoryManager() { ((NotificationEmitter)ManagementFactory.getMemoryMXBean()).addNotificationListener(this, null, null); List<MemoryPoolMXBean> mpbeans = ManagementFactory.getMemoryPoolMXBeans(); @@ -140,11 +146,30 @@ public class SpillableMemoryManager impl if(n.getType().equals(MemoryNotificationInfo.MEMORY_THRESHOLD_EXCEEDED)) { long threshold = (long)(info.getUsage().getMax() * memoryThresholdFraction); toFree = info.getUsage().getUsed() - threshold + (long)(threshold * 0.5); - log.info("low memory handler called (Usage threshold exceeded) " + info.getUsage()); + + //log + String msg = "memory handler call- Usage threshold " + + info.getUsage(); + if(!firstUsageThreshExceededLogged){ + log.info("first " + msg); + firstUsageThreshExceededLogged = true; + }else{ + log.debug(msg); + } } else { // MEMORY_COLLECTION_THRESHOLD_EXCEEDED CASE long threshold = (long)(info.getUsage().getMax() * collectionMemoryThresholdFraction); toFree = info.getUsage().getUsed() - threshold + (long)(threshold * 0.5); - log.info("low memory handler called (Collection threshold exceeded) " + info.getUsage()); + + //log + String msg = "memory handler call - Collection threshold " + + info.getUsage(); + if(!firstCollectionThreshExceededLogged){ + log.info("first " + msg); + firstCollectionThreshExceededLogged = true; + }else{ + log.debug(msg); + } + } if (toFree < 0) { @@ -195,6 +220,7 @@ public class SpillableMemoryManager impl } }); long estimatedFreed = 0; + int numObjSpilled = 0; boolean invokeGC = false; for (i = spillables.iterator(); i.hasNext();) { Spillable s = i.next().get(); @@ -213,6 +239,7 @@ public class SpillableMemoryManager impl break ; } s.spill(); + numObjSpilled++; estimatedFreed += toBeFreed; accumulatedFreeSize += toBeFreed; // This should significantly reduce the number of small files @@ -226,14 +253,19 @@ public class SpillableMemoryManager impl invokeGC = true; break; } - } - + } /* Poke the GC again to see if we successfully freed enough memory */ if(invokeGC) { System.gc(); // now that we have invoked the GC, reset accumulatedFreeSize accumulatedFreeSize = 0; } + if(estimatedFreed > 0){ + String msg = "Spilled an estimate of " + estimatedFreed + + " bytes from " + numObjSpilled + " objects. " + info.getUsage();; + log.info(msg); + } + } }