John Vandenberg has uploaded a new change for review. https://gerrit.wikimedia.org/r/264028
Change subject: Sync ps_mem.py from origin ...................................................................... Sync ps_mem.py from origin Sync ps_mem.py from https://github.com/pixelb/ps_mem/ which has been improved to pass pyflakes. Change-Id: If24ac49e96adb1e471764b615c3e87a647627c3f --- M modules/admin/files/home/ori/.binned/ps_mem.py 1 file changed, 103 insertions(+), 127 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/puppet refs/changes/28/264028/1 diff --git a/modules/admin/files/home/ori/.binned/ps_mem.py b/modules/admin/files/home/ori/.binned/ps_mem.py index 274c955..f3fd2e5 100755 --- a/modules/admin/files/home/ori/.binned/ps_mem.py +++ b/modules/admin/files/home/ori/.binned/ps_mem.py @@ -3,9 +3,9 @@ # Try to determine how much RAM is currently being used per program. # Note per _program_, not per process. So for example this script # will report RAM used by all httpd process together. In detail it reports: -# sum(private RAM for program processes) + sum(Shared RAM for program -# processes) The shared RAM is problematic to calculate, and this script -# automatically selects the most accurate method available for your kernel. +# sum(private RAM for program processes) + sum(Shared RAM for program processes) +# The shared RAM is problematic to calculate, and this script automatically +# selects the most accurate method available for your kernel. # Licence: LGPLv2 # Author: [email protected] @@ -31,13 +31,12 @@ # which fixes the possible race between reading # RSS with ps, and shared memory with this program. # Also we can show non truncated command names. -# V1.8 28 Sep 2007 More accurate matching for stats in -# /proc/$pid/smaps as otherwise could match libraries -# causing a crash. +# V1.8 28 Sep 2007 More accurate matching for stats in /proc/$pid/smaps +# as otherwise could match libraries causing a crash. # Patch from [email protected] # V1.9 20 Feb 2008 Fix invalid values reported when PSS is available. # Reported by Andrey Borzenkov <[email protected]> -# V3.3 24 Jun 2014 +# V3.6 16 Oct 2015 # http://github.com/pixelb/scripts/commits/master/scripts/ps_mem.py # Notes: @@ -45,7 +44,7 @@ # All interpreted programs where the interpreter is started # by the shell or with env, will be merged to the interpreter # (as that's what's given to exec). For e.g. all python programs -# starting with "# !/usr/bin/env python" will be grouped under python. +# starting with "#!/usr/bin/env python" will be grouped under python. # You can change this by using the full command line but that will # have the undesirable affect of splitting up programs started with # differing parameters (for e.g. mingetty tty[1-6]). @@ -80,16 +79,6 @@ import os import sys -try: - # md5 module is deprecated on python 2.6 - # so try the newer hashlib first - import hashlib - md5_new = hashlib.md5 -except ImportError: - import md5 - md5_new = md5.new - - # The following exits cleanly on Ctrl-C or EPIPE # while treating other exceptions as before. def std_exceptions(etype, value, tb): @@ -106,24 +95,12 @@ # Define some global variables # -PAGESIZE = os.sysconf("SC_PAGE_SIZE") / 1024 # KiB +PAGESIZE = os.sysconf("SC_PAGE_SIZE") / 1024 #KiB our_pid = os.getpid() have_pss = 0 -help_msg = """Usage: ps_mem [OPTION]... -Show program core memory usage - - -h, -help Show this help - -p <pid>[,pid2,...pidN] Only show memory usage PIDs in the specified list - -s, --split-args Show and separate by, all command line arguments - -t, --total Show only the total value - -w <N> Measure and show process memory every N seconds - -""" - class Proc: - def __init__(self): uname = os.uname() if uname[0] == "FreeBSD": @@ -136,11 +113,14 @@ def open(self, *args): try: - return open(self.path(*args)) + if sys.version_info < (3,): + return open(self.path(*args)) + else: + return open(self.path(*args), errors='ignore') except (IOError, OSError): val = sys.exc_info()[1] - if (val.errno == errno.ENOENT or # kernel thread or process gone - val.errno == errno.EPERM): + if (val.errno == errno.ENOENT or # kernel thread or process gone + val.errno == errno.EPERM): raise LookupError raise @@ -192,14 +172,19 @@ return (split_args, pids_to_show, watch, only_total) - def help(): - global help_msg + help_msg = 'Usage: ps_mem [OPTION]...\n' \ + 'Show program core memory usage\n' \ + '\n' \ + ' -h, -help Show this help\n' \ + ' -p <pid>[,pid2,...pidN] Only show memory usage PIDs in the specified list\n' \ + ' -s, --split-args Show and separate by, all command line arguments\n' \ + ' -t, --total Show only the total value\n' \ + ' -w <N> Measure and show process memory every N seconds\n' + return help_msg -# (major,minor,release) - - +#(major,minor,release) def kernel_ver(): kv = proc.open('sys/kernel/osrelease').readline().split(".")[:3] last = len(kv) @@ -217,22 +202,22 @@ return (int(kv[0]), int(kv[1]), int(kv[2])) -# return Private,Shared -# Note shared is always a subset of rss (trs is not always) +#return Private,Shared +#Note shared is always a subset of rss (trs is not always) def getMemStats(pid): global have_pss - mem_id = pid # unique + mem_id = pid #unique Private_lines = [] Shared_lines = [] Pss_lines = [] - Rss = (int(proc.open(pid, 'statm').readline().split()[1]) * - PAGESIZE) - if os.path.exists(proc.path(pid, 'smaps')): # stat - digester = md5_new() - for line in proc.open(pid, 'smaps').readlines(): # open - # Note we checksum smaps as maps is usually but - # not always different for separate processes. - digester.update(line.encode('latin1')) + Rss = (int(proc.open(pid, 'statm').readline().split()[1]) + * PAGESIZE) + if os.path.exists(proc.path(pid, 'smaps')): #stat + lines = proc.open(pid, 'smaps').readlines() #open + # Note we checksum smaps as maps is usually but + # not always different for separate processes. + mem_id = hash(''.join(lines)) + for line in lines: if line.startswith("Shared"): Shared_lines.append(line) elif line.startswith("Private"): @@ -240,18 +225,16 @@ elif line.startswith("Pss"): have_pss = 1 Pss_lines.append(line) - mem_id = digester.hexdigest() Shared = sum([int(line.split()[1]) for line in Shared_lines]) Private = sum([int(line.split()[1]) for line in Private_lines]) - # Note Shared + Private = Rss above - # The Rss in smaps includes video card mem etc. + #Note Shared + Private = Rss above + #The Rss in smaps includes video card mem etc. if have_pss: - pss_adjust = 0.5 # add 0.5KiB as this avg error due to trunctation - Pss = sum( - [float(line.split()[1]) + pss_adjust for line in Pss_lines]) + pss_adjust = 0.5 # add 0.5KiB as this avg error due to trunctation + Pss = sum([float(line.split()[1])+pss_adjust for line in Pss_lines]) Shared = Pss - Private - elif (2, 6, 1) <= kernel_ver() <= (2, 6, 9): - Shared = 0 # lots of overestimation, but what can we do? + elif (2,6,1) <= kernel_ver() <= (2,6,9): + Shared = 0 #lots of overestimation, but what can we do? Private = Rss else: Shared = int(proc.open(pid, 'statm').readline().split()[2]) @@ -273,7 +256,8 @@ path = path.split('\0')[0] except OSError: val = sys.exc_info()[1] - if (val.errno == errno.ENOENT or val.errno == errno.EPERM): + if (val.errno == errno.ENOENT or # either kernel thread or process gone + val.errno == errno.EPERM): raise LookupError raise @@ -284,8 +268,9 @@ if os.path.exists(path): path += " [updated]" else: - # The path could be have prelink stuff so try cmdline - # which might have the full path present. + #The path could be have prelink stuff so try cmdline + #which might have the full path present. This helped for: + #/usr/libexec/notification-area-applet.#prelink#.fX7LCT (deleted) if os.path.exists(cmdline[0]): path = cmdline[0] + " [updated]" else: @@ -293,22 +278,25 @@ exe = os.path.basename(path) cmd = proc.open(pid, 'status').readline()[6:-1] if exe.startswith(cmd): - cmd = exe # show non truncated version - # Note because we show the non truncated name - # one can have separated programs as follows: - # 584.0 KiB + 1.0 MiB = 1.6 MiB mozilla-thunder (exe -> bash) + cmd = exe #show non truncated version + #Note because we show the non truncated name + #one can have separated programs as follows: + #584.0 KiB + 1.0 MiB = 1.6 MiB mozilla-thunder (exe -> bash) # 56.0 MiB + 22.2 MiB = 78.2 MiB mozilla-thunderbird-bin - return cmd + if sys.version_info < (3,): + return cmd + else: + return cmd.encode(errors='replace').decode() -# The following matches "du -h" output -# see also human.py +#The following matches "du -h" output +#see also human.py def human(num, power="Ki", units=None): if units is None: powers = ["Ki", "Mi", "Gi", "Ti"] - while num >= 1000: # 4 digits + while num >= 1000: #4 digits num /= 1024.0 - power = powers[powers.index(power) + 1] + power = powers[powers.index(power)+1] return "%.1f %sB" % (num, power) else: return "%.f" % ((num * 1024) / units) @@ -320,67 +308,59 @@ else: return cmd -# Warn of possible inaccuracies -# 2 = accurate & can total -# 1 = accurate only considering each process in isolation -# 0 = some shared mem not reported -# -1= all shared mem not reported - - +#Warn of possible inaccuracies +#2 = accurate & can total +#1 = accurate only considering each process in isolation +#0 = some shared mem not reported +#-1= all shared mem not reported def shared_val_accuracy(): """http://wiki.apache.org/spamassassin/TopSharedMemoryBug"""; kv = kernel_ver() - if kv[:2] == (2, 4): + pid = os.getpid() + if kv[:2] == (2,4): if proc.open('meminfo').read().find("Inact_") == -1: return 1 return 0 - elif kv[:2] == (2, 6): - pid = os.getpid() + elif kv[:2] == (2,6): if os.path.exists(proc.path(pid, 'smaps')): - if proc.open(pid, 'smaps').read().find("Pss:") != -1: + if proc.open(pid, 'smaps').read().find("Pss:")!=-1: return 2 else: return 1 - if (2, 6, 1) <= kv <= (2, 6, 9): + if (2,6,1) <= kv <= (2,6,9): return -1 return 0 - elif kv[0] > 2: + elif kv[0] > 2 and os.path.exists(proc.path(pid, 'smaps')): return 2 else: return 1 - -def show_shared_val_accuracy(possible_inacc, only_total=False): - level = ("Warning", "Error")[only_total] +def show_shared_val_accuracy( possible_inacc, only_total=False ): + level = ("Warning","Error")[only_total] if possible_inacc == -1: sys.stderr.write( - "%s: Shared memory is not reported by this system.\n" % level + "%s: Shared memory is not reported by this system.\n" % level ) sys.stderr.write( - "Values reported will be too large, and totals are not reported\n" + "Values reported will be too large, and totals are not reported\n" ) elif possible_inacc == 0: sys.stderr.write( - "%s: Shared memory is not reported accurately by this system.\n" % - level) + "%s: Shared memory is not reported accurately by this system.\n" % level + ) sys.stderr.write( - "Values reported could be too large, and totals are not reported\n" + "Values reported could be too large, and totals are not reported\n" ) elif possible_inacc == 1: sys.stderr.write( - "%s: Shared memory is slightly over-estimated by this system\n" - "for each program, so totals are not reported.\n" % level + "%s: Shared memory is slightly over-estimated by this system\n" + "for each program, so totals are not reported.\n" % level ) sys.stderr.close() if only_total and possible_inacc != 2: sys.exit(1) - -def get_memory_usage( - pids_to_show, - split_args, - include_self=False, - only_self=False): +def get_memory_usage( pids_to_show, split_args, include_self=False, only_self=False ): cmds = {} shareds = {} mem_ids = {} @@ -401,19 +381,19 @@ try: cmd = getCmdName(pid, split_args) except LookupError: - # operation not permitted - # kernel threads don't have exe links or - # process gone + #operation not permitted + #kernel threads don't have exe links or + #process gone continue try: private, shared, mem_id = getMemStats(pid) except RuntimeError: - continue # process gone + continue #process gone if shareds.get(cmd): - if have_pss: # add shared portion of PSS together + if have_pss: #add shared portion of PSS together shareds[cmd] += shared - elif shareds[cmd] < shared: # just take largest shared val + elif shareds[cmd] < shared: #just take largest shared val shareds[cmd] = shared else: shareds[cmd] = shared @@ -422,9 +402,9 @@ count[cmd] += 1 else: count[cmd] = 1 - mem_ids.setdefault(cmd, {}).update({mem_id: None}) + mem_ids.setdefault(cmd, {}).update({mem_id:None}) - # Add shared mem for each program + #Add shared mem for each program total = 0 for cmd in cmds: cmd_count = count[cmd] @@ -435,49 +415,45 @@ if have_pss: shareds[cmd] /= cmd_count cmds[cmd] = cmds[cmd] + shareds[cmd] - total += cmds[cmd] # valid if PSS available + total += cmds[cmd] #valid if PSS available - sorted_cmds = sorted(cmds.items(), key=lambda x: x[1]) + sorted_cmds = sorted(cmds.items(), key=lambda x:x[1]) sorted_cmds = [x for x in sorted_cmds if x[1]] return sorted_cmds, shareds, count, total - def print_header(): sys.stdout.write(" Private + Shared = RAM used\tProgram\n\n") - def print_memory_usage(sorted_cmds, shareds, count, total): for cmd in sorted_cmds: sys.stdout.write("%9s + %9s = %9s\t%s\n" % - (human(cmd[1] - shareds[cmd[0]]), + (human(cmd[1]-shareds[cmd[0]]), human(shareds[cmd[0]]), human(cmd[1]), cmd_with_count(cmd[0], count[cmd[0]]))) if have_pss: sys.stdout.write("%s\n%s%9s\n%s\n" % ("-" * 33, " " * 24, human(total), "=" * 33)) - def verify_environment(): if os.geteuid() != 0: sys.stderr.write("Sorry, root permission required.\n") - if __name__ == '__main__': - sys.stderr.close() - sys.exit(1) + sys.stderr.close() + sys.exit(1) try: - kv = kernel_ver() + kernel_ver() except (IOError, OSError): val = sys.exc_info()[1] if val.errno == errno.ENOENT: sys.stderr.write( - "Couldn't access " + proc.path('') + "\n" - "Only GNU/Linux and FreeBSD (with linprocfs) are supported\n") + "Couldn't access " + proc.path('') + "\n" + "Only GNU/Linux and FreeBSD (with linprocfs) are supported\n") sys.exit(2) else: raise -if __name__ == '__main__': +def main(): split_args, pids_to_show, watch, only_total = parse_options() verify_environment() @@ -488,10 +464,9 @@ try: sorted_cmds = True while sorted_cmds: - sorted_cmds, shareds, count, total = get_memory_usage( - pids_to_show, split_args) + sorted_cmds, shareds, count, total = get_memory_usage( pids_to_show, split_args ) if only_total and have_pss: - sys.stdout.write(human(total, units=1) + '\n') + sys.stdout.write(human(total, units=1)+'\n') elif not only_total: print_memory_usage(sorted_cmds, shareds, count, total) time.sleep(watch) @@ -501,10 +476,9 @@ pass else: # This is the default behavior - sorted_cmds, shareds, count, total = get_memory_usage( - pids_to_show, split_args) + sorted_cmds, shareds, count, total = get_memory_usage( pids_to_show, split_args ) if only_total and have_pss: - sys.stdout.write(human(total, units=1) + '\n') + sys.stdout.write(human(total, units=1)+'\n') elif not only_total: print_memory_usage(sorted_cmds, shareds, count, total) @@ -514,4 +488,6 @@ sys.stdout.close() vm_accuracy = shared_val_accuracy() - show_shared_val_accuracy(vm_accuracy, only_total) + show_shared_val_accuracy( vm_accuracy, only_total ) + +if __name__ == '__main__': main() -- To view, visit https://gerrit.wikimedia.org/r/264028 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: If24ac49e96adb1e471764b615c3e87a647627c3f Gerrit-PatchSet: 1 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: John Vandenberg <[email protected]> _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
