From 0b05c4858fa61483fb038aac95431e4e1ed8b41c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A1draig=20Brady?= Date: Mon, 11 Jul 2022 23:28:06 +0100 Subject: [PATCH] latest upstream (3.14) --- ps_mem.1 | 13 +- ps_mem.py | 405 +++++++++++++++++++++++++++++++++++++--------------- ps_mem.spec | 7 +- 3 files changed, 306 insertions(+), 119 deletions(-) diff --git a/ps_mem.1 b/ps_mem.1 index 12bf337..f2771c1 100644 --- a/ps_mem.1 +++ b/ps_mem.1 @@ -1,6 +1,6 @@ .\" Simple man page to ps_mem script .\" Contact fholec@redhat.com -.TH ps_mem 1 "04 October 2014" "" "" +.TH ps_mem 1 "25 May 2018" "" "" .SH NAME ps_mem \- Memory profiling tool .SH SYNOPSIS @@ -25,9 +25,20 @@ The shared RAM is problematic to calculate, and the tool automatically selects t \-h \-\-help Show help message .TP +\-\-version +Show version info +.TP \-p PID[,PID2,...PIDN] Show memory consumption for the specified processes .TP +\-S \-\-swap +Show swap information of each item +(and Shared swap if the kernel supports SwapPss). +.TP +\-d \-\-discriminate-by-pid +Show and separate memory usage entries by process +rather than by program. +.TP \-s \-\-split\-args Show and separate memory usage entries by command line arguments and not just the program name. diff --git a/ps_mem.py b/ps_mem.py index 8e30bce..d03a987 100644 --- a/ps_mem.py +++ b/ps_mem.py @@ -9,7 +9,7 @@ # Licence: LGPLv2 # Author: P@draigBrady.com -# Source: http://www.pixelbeat.org/scripts/ps_mem.py +# Source: https://www.pixelbeat.org/scripts/ps_mem.py # V1.0 06 Jul 2005 Initial release # V1.1 11 Aug 2006 root permission required for accuracy @@ -36,8 +36,8 @@ # Patch from patrice.bouchand.fedora@gmail.com # V1.9 20 Feb 2008 Fix invalid values reported when PSS is available. # Reported by Andrey Borzenkov -# V3.6 16 Oct 2015 -# http://github.com/pixelb/scripts/commits/master/scripts/ps_mem.py +# V3.14 28 May 2022 +# https://github.com/pixelb/ps_mem/commits/master/ps_mem.py # Notes: # @@ -73,11 +73,12 @@ # FreeBSD is supported if linprocfs is mounted at /compat/linux/proc/ # FreeBSD 8.0 supports up to a level of Linux 2.6.16 -import getopt -import time +import argparse import errno import os import sys +import time +import io # The following exits cleanly on Ctrl-C or EPIPE # while treating other exceptions as before. @@ -99,6 +100,17 @@ PAGESIZE = os.sysconf("SC_PAGE_SIZE") / 1024 #KiB our_pid = os.getpid() have_pss = 0 +have_swap_pss = 0 + +class Unbuffered(io.TextIOBase): + def __init__(self, stream): + super(Unbuffered, self).__init__() + self.stream = stream + def write(self, data): + self.stream.write(data) + self.stream.flush() + def close(self): + self.stream.close() class Proc: def __init__(self): @@ -118,9 +130,12 @@ class Proc: else: return open(self.path(*args), errors='ignore') except (IOError, OSError): + if type(args[0]) is not int: + raise val = sys.exc_info()[1] if (val.errno == errno.ENOENT or # kernel thread or process gone - val.errno == errno.EPERM): + val.errno == errno.EPERM or + val.errno == errno.EACCES): raise LookupError raise @@ -132,59 +147,68 @@ proc = Proc() # def parse_options(): - try: - long_options = ['split-args', 'help', 'total'] - opts, args = getopt.getopt(sys.argv[1:], "shtp:w:", long_options) - except getopt.GetoptError: - sys.stderr.write(help()) - sys.exit(3) + help_msg = 'Show program core memory usage.' + parser = argparse.ArgumentParser(prog='ps_mem', description=help_msg) + parser.add_argument('--version', action='version', version='3.14') + parser.add_argument( + '-s', '--split-args', + action='store_true', + help='Show and separate by, all command line arguments', + ) + parser.add_argument( + '-t', '--total', + dest='only_total', + action='store_true', + help='Show only the total value', + ) + parser.add_argument( + '-d', '--discriminate-by-pid', + action='store_true', + help='Show by process rather than by program', + ) + parser.add_argument( + '-S', '--swap', + dest='show_swap', + action='store_true', + help='Show swap information', + ) + parser.add_argument( + '-p', + dest='pids', + metavar='[,pid2,...pidN]', + help='Only show memory usage PIDs in the specified list', + ) + parser.add_argument( + '-w', + dest='watch', + metavar='', + type=int, + help='Measure and show process memory every N seconds', + ) + args = parser.parse_args() - if len(args): - sys.stderr.write("Extraneous arguments: %s\n" % args) - sys.exit(3) + args.pids_to_show = [] + if args.pids: + try: + args.pids_to_show = [int(x) for x in args.pids.split(',')] + except ValueError: + parser.error('Invalid PID(s): %s' % args.pids) - # ps_mem.py options - split_args = False - pids_to_show = None - watch = None - only_total = False + if args.watch is not None: + if args.watch <= 0: + parser.error('Seconds must be positive! (%s)' % args.watch) - for o, a in opts: - if o in ('-s', '--split-args'): - split_args = True - if o in ('-t', '--total'): - only_total = True - if o in ('-h', '--help'): - sys.stdout.write(help()) - sys.exit(0) - if o in ('-p',): - try: - pids_to_show = [int(x) for x in a.split(',')] - except: - sys.stderr.write(help()) - sys.exit(3) - if o in ('-w',): - try: - watch = int(a) - except: - sys.stderr.write(help()) - sys.exit(3) + return ( + args.split_args, + args.pids_to_show, + args.watch, + args.only_total, + args.discriminate_by_pid, + args.show_swap, + ) - return (split_args, pids_to_show, watch, only_total) -def help(): - help_msg = 'Usage: ps_mem [OPTION]...\n' \ - 'Show program core memory usage\n' \ - '\n' \ - ' -h, -help Show this help\n' \ - ' -p [,pid2,...pidN] Only show memory usage PIDs in the specified list\n' \ - ' -s, --split-args Show and separate by, all command line arguments\n' \ - ' -t, --total Show only the total value\n' \ - ' -w Measure and show process memory every N seconds\n' - - return help_msg - -#(major,minor,release) +# (major,minor,release) def kernel_ver(): kv = proc.open('sys/kernel/osrelease').readline().split(".")[:3] last = len(kv) @@ -202,50 +226,84 @@ def kernel_ver(): return (int(kv[0]), int(kv[1]), int(kv[2])) -#return Private,Shared +#return Private,Shared,Swap(Pss),unique_id #Note shared is always a subset of rss (trs is not always) def getMemStats(pid): global have_pss + global have_swap_pss mem_id = pid #unique Private_lines = [] Shared_lines = [] + Private_huge_lines = [] + Shared_huge_lines = [] Pss_lines = [] Rss = (int(proc.open(pid, 'statm').readline().split()[1]) * PAGESIZE) - if os.path.exists(proc.path(pid, 'smaps')): #stat - lines = proc.open(pid, 'smaps').readlines() #open + Swap_lines = [] + Swap_pss_lines = [] + + Swap = 0 + + if os.path.exists(proc.path(pid, 'smaps')): # stat + smaps = 'smaps' + if os.path.exists(proc.path(pid, 'smaps_rollup')): + smaps = 'smaps_rollup' # faster to process + lines = proc.open(pid, smaps).readlines() # open # Note we checksum smaps as maps is usually but # not always different for separate processes. mem_id = hash(''.join(lines)) for line in lines: - if line.startswith("Shared"): + # {Private,Shared}_Hugetlb is not included in Pss (why?) + # so we need to account for separately. + if line.startswith("Private_Hugetlb:"): + Private_huge_lines.append(line) + elif line.startswith("Shared_Hugetlb:"): + Shared_huge_lines.append(line) + elif line.startswith("Shared"): Shared_lines.append(line) elif line.startswith("Private"): Private_lines.append(line) - elif line.startswith("Pss"): + elif line.startswith("Pss:"): have_pss = 1 Pss_lines.append(line) + elif line.startswith("Swap:"): + Swap_lines.append(line) + elif line.startswith("SwapPss:"): + have_swap_pss = 1 + Swap_pss_lines.append(line) Shared = sum([int(line.split()[1]) for line in Shared_lines]) Private = sum([int(line.split()[1]) for line in Private_lines]) + Shared_huge = sum([int(line.split()[1]) for line in Shared_huge_lines]) + Private_huge = sum([int(line.split()[1]) for line in Private_huge_lines]) #Note Shared + Private = Rss above #The Rss in smaps includes video card mem etc. if have_pss: - pss_adjust = 0.5 # add 0.5KiB as this avg error due to trunctation + pss_adjust = 0.5 # add 0.5KiB as this avg error due to truncation Pss = sum([float(line.split()[1])+pss_adjust for line in Pss_lines]) Shared = Pss - Private + Private += Private_huge # Add after as PSS doesn't a/c for huge pages + if have_swap_pss: + # The kernel supports SwapPss, that shows proportional swap share. + # Note that Swap - SwapPss is not Private Swap. + Swap = sum([int(line.split()[1]) for line in Swap_pss_lines]) + else: + # Note that Swap = Private swap + Shared swap. + Swap = sum([int(line.split()[1]) for line in Swap_lines]) elif (2,6,1) <= kernel_ver() <= (2,6,9): Shared = 0 #lots of overestimation, but what can we do? + Shared_huge = 0 Private = Rss else: Shared = int(proc.open(pid, 'statm').readline().split()[2]) Shared *= PAGESIZE + Shared_huge = 0 Private = Rss - Shared - return (Private, Shared, mem_id) + return (Private, Shared, Shared_huge, Swap, mem_id) -def getCmdName(pid, split_args): +def getCmdName(pid, split_args, discriminate_by_pid, exe_only=False): cmdline = proc.open(pid, 'cmdline').read().split("\0") - if cmdline[-1] == '' and len(cmdline) > 1: + while cmdline[-1] == '' and len(cmdline) > 1: cmdline = cmdline[:-1] path = proc.path(pid, 'exe') @@ -257,12 +315,13 @@ def getCmdName(pid, split_args): except OSError: val = sys.exc_info()[1] if (val.errno == errno.ENOENT or # either kernel thread or process gone - val.errno == errno.EPERM): + val.errno == errno.EPERM or + val.errno == errno.EACCES): raise LookupError raise if split_args: - return " ".join(cmdline) + return ' '.join(cmdline).replace('\n', ' ') if path.endswith(" (deleted)"): path = path[:-10] if os.path.exists(path): @@ -276,17 +335,38 @@ def getCmdName(pid, split_args): else: path += " [deleted]" exe = os.path.basename(path) - cmd = proc.open(pid, 'status').readline()[6:-1] + if exe_only: return exe + + proc_status = proc.open(pid, 'status').readlines() + cmd = proc_status[0][6:-1] if exe.startswith(cmd): cmd = exe #show non truncated version #Note because we show the non truncated name #one can have separated programs as follows: #584.0 KiB + 1.0 MiB = 1.6 MiB mozilla-thunder (exe -> bash) # 56.0 MiB + 22.2 MiB = 78.2 MiB mozilla-thunderbird-bin - if sys.version_info < (3,): - return cmd else: - return cmd.encode(errors='replace').decode() + #Lookup the parent's exe and use that if matching + #which will merge "Web Content" with "firefox" for example + ppid = 0 + for l in range(10): + ps_line = proc_status[l] + if ps_line.startswith('PPid:'): + ppid = int(ps_line[6:-1]) + break + if ppid: + try: + p_exe = getCmdName(ppid, False, False, exe_only=True) + except LookupError: + pass + else: + if exe == p_exe: + cmd = exe + if sys.version_info >= (3,): + cmd = cmd.encode(errors='replace').decode() + if discriminate_by_pid: + cmd = '%s [%d]' % (cmd, pid) + return cmd #The following matches "du -h" output @@ -309,62 +389,99 @@ def cmd_with_count(cmd, count): return cmd #Warn of possible inaccuracies +#RAM: #2 = accurate & can total #1 = accurate only considering each process in isolation #0 = some shared mem not reported #-1= all shared mem not reported -def shared_val_accuracy(): +#SWAP: +#2 = accurate & can total +#1 = accurate only considering each process in isolation +#-1= not available +def val_accuracy(show_swap): """http://wiki.apache.org/spamassassin/TopSharedMemoryBug""" kv = kernel_ver() pid = os.getpid() + swap_accuracy = -1 if kv[:2] == (2,4): if proc.open('meminfo').read().find("Inact_") == -1: - return 1 - return 0 + return 1, swap_accuracy + return 0, swap_accuracy elif kv[:2] == (2,6): if os.path.exists(proc.path(pid, 'smaps')): + swap_accuracy = 1 if proc.open(pid, 'smaps').read().find("Pss:")!=-1: - return 2 + return 2, swap_accuracy else: - return 1 + return 1, swap_accuracy if (2,6,1) <= kv <= (2,6,9): - return -1 - return 0 + return -1, swap_accuracy + return 0, swap_accuracy elif kv[0] > 2 and os.path.exists(proc.path(pid, 'smaps')): - return 2 + swap_accuracy = 1 + if show_swap and proc.open(pid, 'smaps').read().find("SwapPss:")!=-1: + swap_accuracy = 2 + return 2, swap_accuracy else: - return 1 + return 1, swap_accuracy -def show_shared_val_accuracy( possible_inacc, only_total=False ): +def show_val_accuracy( ram_inacc, swap_inacc, only_total, show_swap ): level = ("Warning","Error")[only_total] - if possible_inacc == -1: + + # Only show significant warnings + if not show_swap: + swap_inacc = 2 + elif only_total: + ram_inacc = 2 + + if ram_inacc == -1: sys.stderr.write( "%s: Shared memory is not reported by this system.\n" % level ) sys.stderr.write( "Values reported will be too large, and totals are not reported\n" ) - elif possible_inacc == 0: + elif ram_inacc == 0: sys.stderr.write( "%s: Shared memory is not reported accurately by this system.\n" % level ) sys.stderr.write( "Values reported could be too large, and totals are not reported\n" ) - elif possible_inacc == 1: + elif ram_inacc == 1: sys.stderr.write( "%s: Shared memory is slightly over-estimated by this system\n" "for each program, so totals are not reported.\n" % level ) - sys.stderr.close() - if only_total and possible_inacc != 2: - sys.exit(1) -def get_memory_usage( pids_to_show, split_args, include_self=False, only_self=False ): + if swap_inacc == -1: + sys.stderr.write( + "%s: Swap is not reported by this system.\n" % level + ) + elif swap_inacc == 1: + sys.stderr.write( + "%s: Swap is over-estimated by this system for each program,\n" + "so totals are not reported.\n" % level + ) + + sys.stderr.close() + if only_total: + if show_swap: + accuracy = swap_inacc + else: + accuracy = ram_inacc + if accuracy != 2: + sys.exit(1) + + +def get_memory_usage(pids_to_show, split_args, discriminate_by_pid, + include_self=False, only_self=False): cmds = {} shareds = {} + shared_huges = {} mem_ids = {} count = {} + swaps = {} for pid in os.listdir(proc.path('')): if not pid.isdigit(): continue @@ -375,11 +492,11 @@ def get_memory_usage( pids_to_show, split_args, include_self=False, only_self=Fa continue if pid == our_pid and not include_self: continue - if pids_to_show is not None and pid not in pids_to_show: + if pids_to_show and pid not in pids_to_show: continue try: - cmd = getCmdName(pid, split_args) + cmd = getCmdName(pid, split_args, discriminate_by_pid) except LookupError: #operation not permitted #kernel threads don't have exe links or @@ -387,7 +504,7 @@ def get_memory_usage( pids_to_show, split_args, include_self=False, only_self=Fa continue try: - private, shared, mem_id = getMemStats(pid) + private, shared, shared_huge, swap, mem_id = getMemStats(pid) except RuntimeError: continue #process gone if shareds.get(cmd): @@ -397,15 +514,27 @@ def get_memory_usage( pids_to_show, split_args, include_self=False, only_self=Fa shareds[cmd] = shared else: shareds[cmd] = shared + if shared_huges.get(cmd): + if shared_huges[cmd] < shared_huge: #just take largest shared_huge + shared_huges[cmd] = shared_huge + else: + shared_huges[cmd] = shared_huge cmds[cmd] = cmds.setdefault(cmd, 0) + private if cmd in count: count[cmd] += 1 else: count[cmd] = 1 - mem_ids.setdefault(cmd, {}).update({mem_id:None}) + mem_ids.setdefault(cmd, {}).update({mem_id: None}) - #Add shared mem for each program + # Swap (overcounting for now...) + swaps[cmd] = swaps.setdefault(cmd, 0) + swap + + # Total swaped mem for each program + total_swap = 0 + + # Add shared mem for each program total = 0 + for cmd in cmds: cmd_count = count[cmd] if len(mem_ids[cmd]) == 1 and cmd_count > 1: @@ -414,35 +543,61 @@ def get_memory_usage( pids_to_show, split_args, include_self=False, only_self=Fa cmds[cmd] /= cmd_count if have_pss: shareds[cmd] /= cmd_count + # overestimation possible if shared_huges shared across commands + shareds[cmd] += shared_huges[cmd] cmds[cmd] = cmds[cmd] + shareds[cmd] - total += cmds[cmd] #valid if PSS available + total += cmds[cmd] # valid if PSS available + total_swap += swaps[cmd] sorted_cmds = sorted(cmds.items(), key=lambda x:x[1]) sorted_cmds = [x for x in sorted_cmds if x[1]] - return sorted_cmds, shareds, count, total + return sorted_cmds, shareds, count, total, swaps, total_swap -def print_header(): - sys.stdout.write(" Private + Shared = RAM used\tProgram\n\n") +def print_header(show_swap, discriminate_by_pid): + output_string = " Private + Shared = RAM used" + if show_swap: + output_string += " Swap used" + output_string += "\tProgram" + if discriminate_by_pid: + output_string += "[pid]" + output_string += "\n\n" + sys.stdout.write(output_string) -def print_memory_usage(sorted_cmds, shareds, count, total): + +def print_memory_usage(sorted_cmds, shareds, count, total, swaps, total_swap, + show_swap): for cmd in sorted_cmds: - sys.stdout.write("%9s + %9s = %9s\t%s\n" % - (human(cmd[1]-shareds[cmd[0]]), - human(shareds[cmd[0]]), human(cmd[1]), - cmd_with_count(cmd[0], count[cmd[0]]))) - if have_pss: + + output_string = "%9s + %9s = %9s" + output_data = (human(cmd[1]-shareds[cmd[0]]), + human(shareds[cmd[0]]), human(cmd[1])) + if show_swap: + output_string += " %9s" + output_data += (human(swaps[cmd[0]]),) + output_string += "\t%s\n" + output_data += (cmd_with_count(cmd[0], count[cmd[0]]),) + + sys.stdout.write(output_string % output_data) + + # Only show totals if appropriate + if have_swap_pss and show_swap: # kernel will have_pss + sys.stdout.write("%s\n%s%9s%s%9s\n%s\n" % + ("-" * 45, " " * 24, human(total), " " * 3, + human(total_swap), "=" * 45)) + elif have_pss: sys.stdout.write("%s\n%s%9s\n%s\n" % ("-" * 33, " " * 24, human(total), "=" * 33)) -def verify_environment(): - if os.geteuid() != 0: - sys.stderr.write("Sorry, root permission required.\n") + +def verify_environment(pids_to_show): + if os.geteuid() != 0 and not pids_to_show: + sys.stderr.write("Sorry, root permission required, or specify pids with -p\n") sys.stderr.close() sys.exit(1) try: - kv = kernel_ver() + kernel_ver() except (IOError, OSError): val = sys.exc_info()[1] if val.errno == errno.ENOENT: @@ -454,21 +609,34 @@ def verify_environment(): raise def main(): - split_args, pids_to_show, watch, only_total = parse_options() - verify_environment() + # Force the stdout and stderr streams to be unbuffered + sys.stdout = Unbuffered(sys.stdout) + sys.stderr = Unbuffered(sys.stderr) + + split_args, pids_to_show, watch, only_total, discriminate_by_pid, \ + show_swap = parse_options() + + verify_environment(pids_to_show) if not only_total: - print_header() + print_header(show_swap, discriminate_by_pid) if watch is not None: try: sorted_cmds = True while sorted_cmds: - sorted_cmds, shareds, count, total = get_memory_usage( pids_to_show, split_args ) - if only_total and have_pss: + sorted_cmds, shareds, count, total, swaps, total_swap = \ + get_memory_usage(pids_to_show, split_args, + discriminate_by_pid) + if only_total and show_swap and have_swap_pss: + sys.stdout.write(human(total_swap, units=1)+'\n') + elif only_total and not show_swap and have_pss: sys.stdout.write(human(total, units=1)+'\n') elif not only_total: - print_memory_usage(sorted_cmds, shareds, count, total) + print_memory_usage(sorted_cmds, shareds, count, total, + swaps, total_swap, show_swap) + + sys.stdout.flush() time.sleep(watch) else: sys.stdout.write('Process does not exist anymore.\n') @@ -476,18 +644,23 @@ def main(): pass else: # This is the default behavior - sorted_cmds, shareds, count, total = get_memory_usage( pids_to_show, split_args ) - if only_total and have_pss: + sorted_cmds, shareds, count, total, swaps, total_swap = \ + get_memory_usage(pids_to_show, split_args, + discriminate_by_pid) + if only_total and show_swap and have_swap_pss: + sys.stdout.write(human(total_swap, units=1)+'\n') + elif only_total and not show_swap and have_pss: sys.stdout.write(human(total, units=1)+'\n') elif not only_total: - print_memory_usage(sorted_cmds, shareds, count, total) + print_memory_usage(sorted_cmds, shareds, count, total, swaps, + total_swap, show_swap) # We must close explicitly, so that any EPIPE exception # is handled by our excepthook, rather than the default # one which is reenabled after this script finishes. sys.stdout.close() - vm_accuracy = shared_val_accuracy() - show_shared_val_accuracy( vm_accuracy, only_total ) + ram_accuracy, swap_accuracy = val_accuracy( show_swap ) + show_val_accuracy( ram_accuracy, swap_accuracy, only_total, show_swap ) if __name__ == '__main__': main() diff --git a/ps_mem.spec b/ps_mem.spec index 2aa178a..dceb27c 100644 --- a/ps_mem.spec +++ b/ps_mem.spec @@ -1,7 +1,7 @@ Name: ps_mem -Version: 3.6 -Release: 15%{?dist} +Version: 3.14 +Release: 1%{?dist} Summary: Memory profiling tool License: LGPLv2 URL: https://github.com/pixelb/ps_mem @@ -46,6 +46,9 @@ install -Dpm644 %{name}.1 %{buildroot}%{_mandir}/man1/%{name}.1 %changelog +* Mon Jul 11 2022 Pádraig Brady - 3.14-1 +- Latest upstream + * Fri Jan 21 2022 Fedora Release Engineering - 3.6-15 - Rebuilt for https://fedoraproject.org/wiki/Fedora_36_Mass_Rebuild