YARN-5551. Ignore file backed pages from memory computation when smaps is enabled. Contributed by Rajesh Balamohan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ecb51b85 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ecb51b85 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ecb51b85 Branch: refs/heads/YARN-3368 Commit: ecb51b857ac7faceff981b2b6f22ea1af0d42ab1 Parents: 96b1266 Author: Jason Lowe <jl...@apache.org> Authored: Tue Oct 11 15:12:43 2016 +0000 Committer: Jason Lowe <jl...@apache.org> Committed: Tue Oct 11 15:12:43 2016 +0000 ---------------------------------------------------------------------- .../yarn/util/ProcfsBasedProcessTree.java | 26 ++++++----- .../yarn/util/TestProcfsBasedProcessTree.java | 46 ++++++++++---------- 2 files changed, 39 insertions(+), 33 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ecb51b85/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java index 80d49c3..29bc277 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java @@ -406,15 +406,14 @@ public class ProcfsBasedProcessTree extends ResourceCalculatorProcessTree { continue; } - total += - Math.min(info.sharedDirty, info.pss) + info.privateDirty - + info.privateClean; + // Account for anonymous to know the amount of + // memory reclaimable by killing the process + total += info.anonymous; + if (LOG.isDebugEnabled()) { LOG.debug(" total(" + olderThanAge + "): PID : " + p.getPid() - + ", SharedDirty : " + info.sharedDirty + ", PSS : " - + info.pss + ", Private_Dirty : " + info.privateDirty - + ", Private_Clean : " + info.privateClean + ", total : " - + (total * KB_TO_BYTES)); + + ", info : " + info.toString() + + ", total : " + (total * KB_TO_BYTES)); } } } @@ -877,6 +876,7 @@ public class ProcfsBasedProcessTree extends ResourceCalculatorProcessTree { private int sharedDirty; private int privateClean; private int privateDirty; + private int anonymous; private int referenced; private String regionName; private String permission; @@ -929,6 +929,10 @@ public class ProcfsBasedProcessTree extends ResourceCalculatorProcessTree { return referenced; } + public int getAnonymous() { + return anonymous; + } + public void setMemInfo(String key, String value) { MemInfo info = MemInfo.getMemInfoByName(key); int val = 0; @@ -969,6 +973,9 @@ public class ProcfsBasedProcessTree extends ResourceCalculatorProcessTree { case REFERENCED: referenced = val; break; + case ANONYMOUS: + anonymous = val; + break; default: break; } @@ -999,10 +1006,7 @@ public class ProcfsBasedProcessTree extends ResourceCalculatorProcessTree { .append(MemInfo.REFERENCED.name + ":" + this.getReferenced()) .append(" kB\n"); sb.append("\t") - .append(MemInfo.PRIVATE_DIRTY.name + ":" + this.getPrivateDirty()) - .append(" kB\n"); - sb.append("\t") - .append(MemInfo.PRIVATE_DIRTY.name + ":" + this.getPrivateDirty()) + .append(MemInfo.ANONYMOUS.name + ":" + this.getAnonymous()) .append(" kB\n"); return sb.toString(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/ecb51b85/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java index fa4e8c8..841d333 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestProcfsBasedProcessTree.java @@ -369,21 +369,24 @@ public class TestProcfsBasedProcessTree { List<ProcessSmapMemoryInfo> memoryMappingList = procMemInfo[i].getMemoryInfoList(); memoryMappingList.add(constructMemoryMappingInfo( - "7f56c177c000-7f56c177d000 " + "7f56c177c000-7f56c177d000 " + "rw-p 00010000 08:02 40371558 " + "/grid/0/jdk1.7.0_25/jre/lib/amd64/libnio.so", - new String[] { "4", "4", "25", "4", "25", "15", "10", "4", "0", "0", - "0", "4", "4" })); + // Format: size, rss, pss, shared_clean, shared_dirty, private_clean + // private_dirty, referenced, anon, anon-huge-pages, swap, + // kernel_page_size, mmu_page_size + new String[] {"4", "4", "25", "4", "25", "15", "10", "4", "10", "0", + "0", "4", "4"})); memoryMappingList.add(constructMemoryMappingInfo( - "7fb09382e000-7fb09382f000 r--s 00003000 " + "08:02 25953545", - new String[] { "4", "4", "25", "4", "0", "15", "10", "4", "0", "0", - "0", "4", "4" })); + "7fb09382e000-7fb09382f000 r--s 00003000 " + "08:02 25953545", + new String[] {"4", "4", "25", "4", "0", "15", "10", "4", "10", "0", + "0", "4", "4"})); memoryMappingList.add(constructMemoryMappingInfo( - "7e8790000-7e8b80000 r-xs 00000000 00:00 0", new String[] { "4", "4", - "25", "4", "0", "15", "10", "4", "0", "0", "0", "4", "4" })); + "7e8790000-7e8b80000 r-xs 00000000 00:00 0", new String[] {"4", "4", + "25", "4", "0", "15", "10", "4", "10", "0", "0", "4", "4"})); memoryMappingList.add(constructMemoryMappingInfo( - "7da677000-7e0dcf000 rw-p 00000000 00:00 0", new String[] { "4", "4", - "25", "4", "50", "15", "10", "4", "0", "0", "0", "4", "4" })); + "7da677000-7e0dcf000 rw-p 00000000 00:00 0", new String[] {"4", "4", + "25", "4", "50", "15", "10", "4", "10", "0", "0", "4", "4"})); } } @@ -471,13 +474,12 @@ public class TestProcfsBasedProcessTree { // Check by enabling smaps setSmapsInProceTree(processTree, true); - // RSS=Min(shared_dirty,PSS)+PrivateClean+PrivateDirty (exclude r-xs, - // r--s) + // anon (exclude r-xs,r--s) Assert.assertEquals("rss memory does not match", - (100 * KB_TO_BYTES * 3), processTree.getRssMemorySize()); + (20 * KB_TO_BYTES * 3), processTree.getRssMemorySize()); // verify old API Assert.assertEquals("rss memory (old API) does not match", - (100 * KB_TO_BYTES * 3), processTree.getCumulativeRssmem()); + (20 * KB_TO_BYTES * 3), processTree.getCumulativeRssmem()); // test the cpu time again to see if it cumulates procInfos[0] = @@ -621,10 +623,10 @@ public class TestProcfsBasedProcessTree { cumuRssMem, processTree.getCumulativeRssmem()); } else { Assert.assertEquals("rssmem does not include new process", - 100 * KB_TO_BYTES * 4, processTree.getRssMemorySize()); + 20 * KB_TO_BYTES * 4, processTree.getRssMemorySize()); // verify old API Assert.assertEquals("rssmem (old API) does not include new process", - 100 * KB_TO_BYTES * 4, processTree.getCumulativeRssmem()); + 20 * KB_TO_BYTES * 4, processTree.getCumulativeRssmem()); } // however processes older than 1 iteration will retain the older value @@ -650,11 +652,11 @@ public class TestProcfsBasedProcessTree { } else { Assert.assertEquals( "rssmem shouldn't have included new process", - 100 * KB_TO_BYTES * 3, processTree.getRssMemorySize(1)); + 20 * KB_TO_BYTES * 3, processTree.getRssMemorySize(1)); // Verify old API Assert.assertEquals( "rssmem (old API) shouldn't have included new process", - 100 * KB_TO_BYTES * 3, processTree.getCumulativeRssmem(1)); + 20 * KB_TO_BYTES * 3, processTree.getCumulativeRssmem(1)); } // one more process @@ -696,11 +698,11 @@ public class TestProcfsBasedProcessTree { } else { Assert.assertEquals( "rssmem shouldn't have included new processes", - 100 * KB_TO_BYTES * 3, processTree.getRssMemorySize(2)); + 20 * KB_TO_BYTES * 3, processTree.getRssMemorySize(2)); // Verify old API Assert.assertEquals( "rssmem (old API) shouldn't have included new processes", - 100 * KB_TO_BYTES * 3, processTree.getCumulativeRssmem(2)); + 20 * KB_TO_BYTES * 3, processTree.getCumulativeRssmem(2)); } // processes older than 1 iteration should not include new process, @@ -727,10 +729,10 @@ public class TestProcfsBasedProcessTree { } else { Assert.assertEquals( "rssmem shouldn't have included new processes", - 100 * KB_TO_BYTES * 4, processTree.getRssMemorySize(1)); + 20 * KB_TO_BYTES * 4, processTree.getRssMemorySize(1)); Assert.assertEquals( "rssmem (old API) shouldn't have included new processes", - 100 * KB_TO_BYTES * 4, processTree.getCumulativeRssmem(1)); + 20 * KB_TO_BYTES * 4, processTree.getCumulativeRssmem(1)); } // no processes older than 3 iterations --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
