397 lines
14 KiB
Python
Executable File
397 lines
14 KiB
Python
Executable File
#!/usr/bin/python2
|
|
|
|
# The MIT License (MIT)
|
|
#
|
|
# Copyright (c) 2015 Lance W. Shelton
|
|
#
|
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
# of this software and associated documentation files (the "Software"), to deal
|
|
# in the Software without restriction, including without limitation the rights
|
|
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
# copies of the Software, and to permit persons to whom the Software is
|
|
# furnished to do so, subject to the following conditions:
|
|
#
|
|
# The above copyright notice and this permission notice shall be included in all
|
|
# copies or substantial portions of the Software.
|
|
#
|
|
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
# SOFTWARE.
|
|
|
|
"""
|
|
A better way to watch /proc/interrupts, especially on large NUMA machines with
|
|
so many CPUs that /proc/interrupts is wider than the screen. Press '0'-'9'
|
|
for node views, 't' for node totals
|
|
"""
|
|
|
|
__version__ = '1.0.1-pre'
|
|
|
|
import os
|
|
import sys
|
|
import tty
|
|
import termios
|
|
import time
|
|
from time import sleep
|
|
import subprocess
|
|
from optparse import OptionParser
|
|
import thread
|
|
import threading
|
|
|
|
KEYEVENT = threading.Event()
|
|
|
|
|
|
def gen_numa(numafile):
|
|
"""Generate NUMA info"""
|
|
cpunodes = {}
|
|
numacores = {}
|
|
err_str = ""
|
|
|
|
try:
|
|
if not numafile:
|
|
temp = subprocess.Popen(['numactl', '--hardware'],
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE)
|
|
(output, error) = temp.communicate()
|
|
temp.wait()
|
|
|
|
if error:
|
|
print("NUMACTL ERROR:")
|
|
print(error)
|
|
exit(1)
|
|
else:
|
|
numa_file = open(numafile, 'r')
|
|
output = numa_file.read()
|
|
numa_file.close()
|
|
|
|
output = output.split("\n")
|
|
for line in output:
|
|
arr = line.split()
|
|
if len(arr) < 3:
|
|
continue
|
|
if arr[0] == "node" and arr[2] == "cpus:":
|
|
node = arr[1]
|
|
numacores[node] = arr[3:]
|
|
for core in arr[3:]:
|
|
cpunodes[core] = node
|
|
continue
|
|
return numacores, cpunodes
|
|
except (OSError, IOError) as err:
|
|
if err.errno == 2: # No such file or directory
|
|
if numafile:
|
|
err_str = " (does '" + numafile + "' exist?)"
|
|
else:
|
|
err_str = err.strerror + " (is numactl installed?)"
|
|
print("ERROR: " + err.strerror + err_str)
|
|
exit(1)
|
|
|
|
# input character, passed between threads
|
|
INCHAR = ''
|
|
|
|
|
|
def wait_for_input():
|
|
"""Get a single character of input, validate"""
|
|
global INCHAR
|
|
|
|
acceptable_keys = ['0', '1', '2', '3', '4', '5', '6', '7',
|
|
'8', '9', '0', 't']
|
|
while True:
|
|
key = sys.stdin.read(1)
|
|
|
|
# simple just to exit on any invalid input
|
|
if not key in acceptable_keys:
|
|
thread.interrupt_main()
|
|
|
|
# set the new input and notify the main thread
|
|
INCHAR = key
|
|
KEYEVENT.set()
|
|
|
|
|
|
def filter_found(name, filter_list):
|
|
"""Check if IRQ name matches anything in the filter list"""
|
|
for filt in filter_list:
|
|
if filt in name:
|
|
return True
|
|
return False
|
|
|
|
|
|
def display_itop(batch, seconds, rowcnt, iterations, sort, totals, dispnode,
|
|
zero, filters, file1, file2, overall, numafile):
|
|
"""Main I/O loop"""
|
|
irqs = {}
|
|
cpunodes = {}
|
|
numacores = {}
|
|
loops = 0
|
|
width = len('NODEXX')
|
|
|
|
if not file1 and batch:
|
|
print("Running in batch mode")
|
|
elif not file1:
|
|
print("interactive commands -- "
|
|
"t: view totals, 0-9: view node, any other key: quit\r")
|
|
|
|
if file1:
|
|
intr_filename = file1
|
|
else:
|
|
intr_filename = '/proc/interrupts'
|
|
|
|
while True:
|
|
# Grab the new display type at a time when nothing is in flux
|
|
if KEYEVENT.isSet():
|
|
KEYEVENT.clear()
|
|
dispnode = INCHAR if INCHAR in numacores else '-1'
|
|
|
|
with open(intr_filename, 'r') as intr_file:
|
|
header = intr_file.readline()
|
|
cpus = []
|
|
for name in header.split():
|
|
num = name[3:]
|
|
cpus.append(num)
|
|
|
|
# Only query the numa information when something is missing.
|
|
# This is effectively the first time and when any disabled CPUs
|
|
# are enabled
|
|
if not num in cpunodes:
|
|
numacores, cpunodes = gen_numa(numafile)
|
|
|
|
for line in intr_file.readlines():
|
|
vals = line.split()
|
|
irqnum = vals[0].rstrip(':')
|
|
|
|
# Optionally exclude rows that are not an IRQ number
|
|
if totals is None:
|
|
try:
|
|
num = int(irqnum)
|
|
except ValueError:
|
|
continue
|
|
|
|
irq = {}
|
|
irq['cpus'] = [int(x) for x in vals[1:len(cpus)+1]]
|
|
irq['oldcpus'] = (irqs[irqnum]['cpus'] if irqnum in irqs
|
|
else [0] * len(cpus))
|
|
irq['name'] = ' '.join(vals[len(cpus)+1:])
|
|
irq['oldsum'] = irqs[irqnum]['sum'] if irqnum in irqs else 0
|
|
irq['sum'] = sum(irq['cpus'])
|
|
irq['num'] = irqnum
|
|
|
|
for node in numacores:
|
|
oldkey = 'oldsum' + node
|
|
key = 'sum' + node
|
|
irq[oldkey] = (irqs[irqnum][key] if irqnum in irqs
|
|
and key in irqs[irqnum] else 0)
|
|
irq[key] = 0
|
|
|
|
for idx, val in enumerate(irq['cpus']):
|
|
key = 'sum' + cpunodes[cpus[idx]]
|
|
irq[key] = irq[key] + val if key in irq else val
|
|
|
|
# save old
|
|
irqs[irqnum] = irq
|
|
|
|
def sort_func(val):
|
|
"""Sort output"""
|
|
sortnum = -1
|
|
try:
|
|
sortnum = int(sort)
|
|
except ValueError:
|
|
pass
|
|
|
|
if sortnum >= 0:
|
|
for node in numacores:
|
|
if sortnum == int(node):
|
|
return val['sum' + node] - val['oldsum' + node]
|
|
if sort == 't':
|
|
return val['sum'] - val['oldsum']
|
|
if sort == 'i':
|
|
if val['num'].isdigit():
|
|
return int(val['num'])
|
|
return sys.maxsize
|
|
if sort == 'n':
|
|
return val['name']
|
|
raise Exception('Invalid sort type {}'.format(sort))
|
|
|
|
# reverse sort all IRQ count sorts
|
|
rev = sort not in ['i', 'n']
|
|
rows = sorted(irqs.values(), key=sort_func, reverse=rev)
|
|
|
|
# determine the width required for the count field
|
|
for idx, irq in enumerate(rows):
|
|
width = max(width, len(str(irq['sum'] - irq['oldsum'])))
|
|
|
|
if overall and loops > 0:
|
|
print("" + '\r')
|
|
if not file1 and (overall or loops > 0):
|
|
print(time.ctime() + '\r')
|
|
print("IRQs / " + str(seconds) + " second(s)" + '\r')
|
|
fmtstr = ('IRQ# %' + str(width) + 's') % 'TOTAL'
|
|
|
|
# node view header
|
|
if int(dispnode) >= 0:
|
|
node = 'NODE%s' % dispnode
|
|
fmtstr += (' %' + str(width) + 's ') % node
|
|
for idx, val in enumerate(irq['cpus']):
|
|
if cpunodes[cpus[idx]] == dispnode:
|
|
cpu = 'CPU%s' % cpus[idx]
|
|
fmtstr += (' %' + str(width) + 's ') % cpu
|
|
# top view header
|
|
else:
|
|
for node in sorted(numacores):
|
|
node = 'NODE%s' % node
|
|
fmtstr += (' %' + str(width) + 's ') % node
|
|
|
|
fmtstr += ' NAME'
|
|
if overall or loops > 0:
|
|
print(fmtstr + '\r')
|
|
|
|
displayed_rows = 0
|
|
for idx, irq in enumerate(rows):
|
|
if filters and not filter_found(irq['name'], filters):
|
|
continue
|
|
|
|
total = irq['sum'] - irq['oldsum']
|
|
if zero and not total:
|
|
continue
|
|
|
|
# IRQ# TOTAL
|
|
fmtstr = ('%4s %' + str(width) + 'd') % (irq['num'], total)
|
|
|
|
# node view
|
|
if int(dispnode) >= 0:
|
|
oldnodesum = 'oldsum' + dispnode
|
|
nodesum = 'sum' + dispnode
|
|
nodecnt = irq[nodesum] - irq[oldnodesum]
|
|
if zero and not nodecnt:
|
|
continue
|
|
fmtstr += (' %' + str(width) + 's ') % str(nodecnt)
|
|
for cpu, val in enumerate(irq['cpus']):
|
|
if cpunodes[cpus[cpu]] == dispnode:
|
|
fmtstr += ((' %' + str(width) + 's ') %
|
|
str(irq['cpus'][cpu] - irq['oldcpus'][cpu]))
|
|
|
|
# top view
|
|
else:
|
|
for node in sorted(numacores):
|
|
oldnodesum = 'oldsum' + node
|
|
nodesum = 'sum' + node
|
|
nodecnt = irq[nodesum] - irq[oldnodesum]
|
|
fmtstr += ((' %' + str(width) + 's ') % str(nodecnt))
|
|
fmtstr += ' ' + irq['name']
|
|
if overall or loops > 0:
|
|
print(fmtstr + '\r')
|
|
displayed_rows += 1
|
|
if displayed_rows == rowcnt:
|
|
break
|
|
|
|
# Update field widths after the first iteration. Data changes
|
|
# significantly between the all-time stats and the interval stats, so
|
|
# this compresses the fields quite a bit. Updating every iteration
|
|
# is too jumpy.
|
|
if loops == 0:
|
|
width = len('NODEXX')
|
|
|
|
loops += 1
|
|
if loops == iterations:
|
|
break
|
|
|
|
if file2 and loops == 1:
|
|
intr_filename = file2
|
|
continue
|
|
|
|
# thread.interrupt_main() does not seem to interrupt a sleep, so break
|
|
# it into tenth-of-a-second sleeps to improve user response time on exit
|
|
for _ in range(0, seconds * 10):
|
|
sleep(.1)
|
|
|
|
|
|
def main(args):
|
|
"""Parse arguments, call main loop"""
|
|
|
|
parser = OptionParser(description=__doc__)
|
|
parser.add_option("-b", "--batch", action="store_true",
|
|
help="run under batch mode")
|
|
parser.add_option("-i", "--iterations", default='-1',
|
|
help="iterations to run")
|
|
parser.add_option("-n", "--node", default='-1',
|
|
help="view a single node")
|
|
parser.add_option("-r", "--rows", default='10',
|
|
help="rows to display (default 10)")
|
|
parser.add_option("-s", "--sort", default='t',
|
|
help="column to sort on ('t':total, 'n': name, "
|
|
"'i':IRQ number, '1':node1, etc) (default: 't')")
|
|
parser.add_option("-t", "--time", default='5',
|
|
help="update interval in seconds")
|
|
parser.add_option("-z", "--zero", action="store_true",
|
|
help="exclude inactive IRQs")
|
|
parser.add_option("-v", "--version", action="store_true",
|
|
help="get version")
|
|
parser.add_option("--filter", default="",
|
|
help="filter IRQs based on name matching comma "
|
|
"separated filters")
|
|
parser.add_option("--totals", action="store_true",
|
|
help="include total rows")
|
|
parser.add_option("-f", "--file1", default="",
|
|
help="read a file instead of /proc/interrupts")
|
|
parser.add_option("-F", "--file2", default="",
|
|
help="no monitoring. Compare the samples from two files "
|
|
"instead")
|
|
parser.add_option("-O", "--overall", action="store_true",
|
|
help="print all-time stats at the beginning")
|
|
parser.add_option("-N", "--numafile", default="",
|
|
help="read the NUMA info from a file, instead of "
|
|
"calling numactl")
|
|
|
|
options = parser.parse_args(args)[0]
|
|
|
|
if options.version:
|
|
print __version__
|
|
return 0
|
|
|
|
if options.filter:
|
|
options.filter = options.filter.split(',')
|
|
else:
|
|
options.filter = []
|
|
|
|
# If file is specified, no iterations
|
|
if options.file1:
|
|
options.iterations = 1
|
|
options.batch = True
|
|
|
|
if options.file2:
|
|
if not options.file1:
|
|
print("ERROR: --file2 requires --file1")
|
|
return -1
|
|
options.iterations = 2
|
|
|
|
if options.file1 and not options.file2:
|
|
options.overall = True
|
|
|
|
# Set the terminal to unbuffered, to catch a single keypress
|
|
if not options.batch:
|
|
out = sys.stdin.fileno()
|
|
old_settings = termios.tcgetattr(out)
|
|
tty.setraw(sys.stdin.fileno())
|
|
|
|
# input thread
|
|
thread.start_new_thread(wait_for_input, tuple())
|
|
else:
|
|
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
|
|
|
|
try:
|
|
display_itop(options.batch, int(options.time), int(options.rows),
|
|
int(options.iterations), options.sort, options.totals,
|
|
options.node, options.zero, options.filter, options.file1,
|
|
options.file2, options.overall, options.numafile)
|
|
except (KeyboardInterrupt, SystemExit):
|
|
pass
|
|
finally:
|
|
if not options.batch:
|
|
termios.tcsetattr(out, termios.TCSADRAIN, old_settings)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main(sys.argv))
|