412 lines
12 KiB
Perl
Executable File
412 lines
12 KiB
Perl
Executable File
#!/usr/bin/perl
|
|
# irqtop
|
|
#
|
|
# by Robert Elliott, HP
|
|
# contributed to the sysstat project
|
|
#
|
|
#########################################################################
|
|
# This program is free software; you can redistribute it and/or modify it
|
|
# under the terms of the GNU General Public License as published by the
|
|
# Free Software Foundation; either version 2 of the License, or (at your
|
|
# option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful, but
|
|
# WITHOUT ANY WARRANTY; without the implied warranty of MERCHANTABILITY
|
|
# or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
|
|
# for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License along
|
|
# with this program; if not, write to the Free Software Foundation, Inc.,
|
|
# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
#########################################################################
|
|
#
|
|
# Monitor differences in /proc/interrupts and /proc/softirqs
|
|
# per CPU, along with CPU statistics
|
|
#
|
|
# Usage: irqtop [interval]
|
|
#
|
|
# Displays interrupts that have occurred since this program
|
|
# began, filtering those that have always been zeros.
|
|
#
|
|
# TODO features:
|
|
# * increase column widths automatically
|
|
# * add an all-CPU column
|
|
# * add option to choose between
|
|
# - online CPUs
|
|
# - just the all-CPU column
|
|
# - select CPUs (e.g., ranges like in smp_affinity_list)
|
|
# * automatically determine interrupts to total (e.g.
|
|
# all mpt3sas or hpsaN interrupts) based on them having
|
|
# common prefixes in their names or common names
|
|
#
|
|
use strict;
|
|
use Getopt::Long;
|
|
|
|
my $myname = "irqtop";
|
|
my $myversion = "0.2";
|
|
|
|
sub print_usage {
|
|
print "Usage: $myname [interval] [-V | --version] [-h | --help]\n\n";
|
|
print " [internal] time in seconds between updates (default: 1 second)\n";
|
|
print " [-V | --version] display version number\n";
|
|
print " [-h | --help] display usage\n\n";
|
|
print "Use Control-C to exit\n";
|
|
}
|
|
|
|
#my $mpstat = "../sysstat/mpstat";
|
|
my $mpstat = "mpstat";
|
|
|
|
# calculate total interrupt stats for this list
|
|
#my @track_list = ();
|
|
my @track_list = ("eth", "hpsa1", "hpsa2", "mpt3sas");
|
|
|
|
# exclude these from per-vector display (e.g., eth while studying storage)
|
|
# still included in totals if also in track_list
|
|
#my @exclude_list = ();
|
|
my @exclude_list = ("eth");
|
|
|
|
my $interval = 1; # default interval in seconds
|
|
|
|
# hashes of arrays
|
|
# key is the interrupt number or name (e.g., 95 or TIMER)
|
|
# array is the interrupt counts and the description on the right
|
|
my %current;
|
|
my %delta; # hold the deltas between collection and printing
|
|
|
|
# hash of values
|
|
my %ever_changed; # indicates if this row ever changed
|
|
my %track_interrupts; # number of each type
|
|
|
|
my $online_cpus; # as seen by process_hardirq, used to filter columns in softirq
|
|
|
|
# return 1 to exclude, 0 to not
|
|
sub is_excluded {
|
|
my ($line) = @_;
|
|
|
|
foreach (@exclude_list) {
|
|
return 1 if ($line =~ /$_/);
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
# convert affinity_list bitmask (64-bit binary) into numbered list
|
|
# Example: 5800a000f -> "0-3,17,19,31-32,34"
|
|
# used for parsing /proc/irq/NN/affinity_hint
|
|
sub bitmask_to_list {
|
|
my ($bitmask) = @_;
|
|
my $inrange;
|
|
my $rangelist;
|
|
my $rangenew;
|
|
my $lowcpu;
|
|
my $highcpu;
|
|
|
|
for (my $i = 0; $i < 63; $i++) {
|
|
if (!$inrange && $bitmask & 1) {
|
|
$inrange = 1;
|
|
$lowcpu = $i;
|
|
$highcpu = $i;
|
|
} elsif ($inrange && $bitmask & 1) {
|
|
$highcpu = $i;
|
|
} elsif ($inrange && !($bitmask & 1)) {
|
|
$inrange = 0;
|
|
if ($lowcpu == $highcpu) {
|
|
$rangenew = "$lowcpu";
|
|
} else {
|
|
$rangenew = "$lowcpu-$highcpu";
|
|
}
|
|
|
|
if ($rangelist) {
|
|
$rangelist = "$rangelist,$rangenew";
|
|
} else {
|
|
$rangelist = "$rangenew";
|
|
}
|
|
}
|
|
|
|
$bitmask = $bitmask >> 1;
|
|
}
|
|
if ($inrange) {
|
|
$rangelist = "$rangelist,$lowcpu-$highcpu";
|
|
}
|
|
if (!$rangelist) {
|
|
$rangelist = "none";
|
|
}
|
|
return $rangelist;
|
|
}
|
|
|
|
# process /proc/interrupts
|
|
# argument:
|
|
# 0 do not display - use the first time to not display values since reboot
|
|
# 1 display - use on all subsequent calls
|
|
sub collect_hardirqs {
|
|
my ($firstpass) = @_;
|
|
|
|
open HARDIRQFILE, "/proc/interrupts" or die "Cannot open $_";
|
|
|
|
foreach (@track_list) {
|
|
$track_interrupts{$_} = 0;
|
|
}
|
|
|
|
# /proc/interrupts lists only online cpus
|
|
# first line contains CPUnn headers for each column
|
|
# CPU0 CPU1 CPU2 CPU3 CPU4 CPU5 CPU6 CPU7 CPU8 CPU9 CPU10 CPU11
|
|
$_ = <HARDIRQFILE>;
|
|
my @cpulist = split;
|
|
$online_cpus = $#cpulist + 1; # hardirqs are a source of online_cpus
|
|
|
|
# remaining lines contain vector number: per-cpu counts, interrupt type, device
|
|
# or vector name, per-CPU counts, interrupt type, description
|
|
while (<HARDIRQFILE>) {
|
|
my @values = split;
|
|
my $irqname = $values[0];
|
|
my $cpu;
|
|
my $line = $_;
|
|
|
|
for ($cpu = 0; $cpu < $online_cpus; $cpu++) {
|
|
$delta{$irqname}[$cpu] = $values[$cpu + 1] - $current{$irqname}[$cpu];
|
|
$current{$irqname}[$cpu] = $values[$cpu + 1];
|
|
|
|
# if this is not the first pass,
|
|
# keep track of whether the values have changed
|
|
if ($delta{$irqname}[$cpu] && !$firstpass && !is_excluded($line)) {
|
|
$ever_changed{$irqname} = 1;
|
|
}
|
|
|
|
foreach (@track_list) {
|
|
if ($line =~ /$_/) {
|
|
$track_interrupts{$_} += $delta{$irqname}[$cpu];
|
|
}
|
|
}
|
|
}
|
|
# capture the rest of the line (interrupt type, handlers)
|
|
# these are not really per-cpu values, but continue storing
|
|
# each word in %current since it's convenient
|
|
for (; $cpu < $#values + 1; $cpu++ ) {
|
|
$current{$irqname}[$cpu] = $values[$cpu + 1];
|
|
}
|
|
}
|
|
close HARDIRQFILE;
|
|
|
|
}
|
|
|
|
# process /proc/softirqs
|
|
# argument:
|
|
# 0 do not display - use the first time to not display values since reboot
|
|
# 1 display - use on all subsequent calls
|
|
sub collect_softirqs {
|
|
my ($firstpass) = @_;
|
|
|
|
open SOFTIRQFILE, "/proc/softirqs" or die "Cannot open $_";
|
|
|
|
# /proc/softirqs includes all possible cpus, not all online cpus
|
|
# this function ignores all those extra cpus
|
|
# first line contains CPUnn headers for each column
|
|
# CPU0 CPU1 CPU2 CPU3 CPU4 CPU5 CPU6 CPU7 CPU8 CPU9 CPU10 CPU11
|
|
$_ = <SOFTIRQFILE>;
|
|
my @cpulist = split; # discard the first line
|
|
|
|
# remaining lines contain
|
|
# vector number: per-cpu counts, interrupt type, device
|
|
# or vector name: per-cpu counts, interrupt type, description
|
|
while (<SOFTIRQFILE>) {
|
|
my @values = split;
|
|
my $irqname = $values[0];
|
|
my $line = $_;
|
|
|
|
# just remember the values for online cpus, not offline cpus
|
|
for (my $cpu = 0; $cpu < $online_cpus; $cpu++) {
|
|
$delta{$irqname}[$cpu] = $values[$cpu + 1] - $current{$irqname}[$cpu];
|
|
$current{$irqname}[$cpu] = $values[$cpu + 1];
|
|
if ($delta{$irqname}[$cpu] && !$firstpass && !is_excluded($line)) {
|
|
$ever_changed{$irqname} = 1;
|
|
}
|
|
}
|
|
}
|
|
close SOFTIRQFILE;
|
|
}
|
|
|
|
# print the CPU0 CPU1 CPU2... header line for the online cpus
|
|
sub print_cpulist {
|
|
for (my $cpu = 0; $cpu < $online_cpus; $cpu++) {
|
|
my $cpustring;
|
|
$cpustring = sprintf("CPU%d", $cpu);
|
|
printf("%6s ", $cpustring);
|
|
}
|
|
print "\n";
|
|
}
|
|
|
|
# print hardirqs and softirqs from %delta that have ever %changed
|
|
sub print_irqs {
|
|
|
|
# header line
|
|
printf("%13s ", "--- IRQs ---");
|
|
# print_cpulist(); # uncomment if print_mpstat is not used, since that prints the header line
|
|
print "\n";
|
|
|
|
foreach (sort keys %delta) {
|
|
my $irqname = $_;
|
|
# if any values have ever changed for an interrupt, print the delta values
|
|
if ($ever_changed{$irqname}) {
|
|
# match the width of softirq names 12 characters plus :
|
|
printf("%13s ", $irqname);
|
|
my $cpu;
|
|
for ($cpu = 0; $cpu < $online_cpus; $cpu++) {
|
|
printf("%6d ", $delta{$irqname}[$cpu]);
|
|
}
|
|
# print the rest of the line (interrupt type, handlers)
|
|
for (; $cpu < @{$current{$irqname}} + 1; $cpu++ ) {
|
|
print "$current{$irqname}[$cpu] ";
|
|
}
|
|
|
|
# print the irq smp affinity list, if any
|
|
my $irqname_nocolon = $irqname;
|
|
$irqname_nocolon =~ s/://;
|
|
|
|
if (!($irqname =~ /[A-Z]/)) {
|
|
my $affinity_hint_file = "/proc/irq/$irqname_nocolon/affinity_hint";
|
|
if (-e $affinity_hint_file) {
|
|
open IRQAFF, "<$affinity_hint_file";
|
|
my $affhint = <IRQAFF>;
|
|
close IRQAFF;
|
|
chomp $affhint;
|
|
my ($affhigh,$afflow) = $affhint =~ /(.*),(.*)/;
|
|
my $aff = hex $affhigh << 32 | hex $afflow;
|
|
printf("hint=%s,", bitmask_to_list($aff));
|
|
} else {
|
|
print "hint=none,";
|
|
}
|
|
my $smp_affinity_list_file = "/proc/irq/$irqname_nocolon/smp_affinity_list";
|
|
if (-e $smp_affinity_list_file) {
|
|
open IRQAFF, "<$smp_affinity_list_file";
|
|
my $afflist = <IRQAFF>;
|
|
close IRQAFF;
|
|
chomp $afflist;
|
|
print "aff=$afflist";
|
|
} else {
|
|
print "aff=none";
|
|
}
|
|
}
|
|
|
|
print "\n";
|
|
}
|
|
}
|
|
|
|
# print summary of selected sets of interrupts
|
|
foreach (@track_list) {
|
|
printf("%12s: total=%-7d, average=%-7d; total/s=%-7d, average/s=%-7d\n",
|
|
$_,
|
|
$track_interrupts{$_},
|
|
$track_interrupts{$_} / $online_cpus,
|
|
$track_interrupts{$_} / $interval,
|
|
$track_interrupts{$_} / $interval / $online_cpus);
|
|
}
|
|
}
|
|
|
|
# collect interesting CPU statistics from mpstat
|
|
# %usr, %sys, %iowait, %idle, %irq, %soft
|
|
# mpstat displays:
|
|
# 06:14:48 PM CPU %usr %nice %sys %iowait %irq %soft %steal %guest %idle
|
|
# plain "mpstat" shows averages since boot, which is not useful
|
|
# so, use the interval option based on the first argument to this function,
|
|
# which also causes this function to incur the delay.
|
|
# per-cpu arrays of value strings
|
|
my @usr;
|
|
my @sys;
|
|
my @irq;
|
|
my @soft;
|
|
my @iowait;
|
|
my @idle;
|
|
|
|
sub collect_mpstat {
|
|
my ($delay) = @_;
|
|
|
|
open MPSTATFILE, "$mpstat -P ON -u $delay 1 |" or sleep($delay);
|
|
while (<MPSTATFILE>) {
|
|
if (/CPU/ || /all/ || /^$/ || /Average/) {
|
|
next;
|
|
}
|
|
my ($time, $ampm, $cpu, $usr, $nice, $sys, $iowait, $irq, $soft, $steal, $guest, $idle) = split;
|
|
$usr[$cpu] = $usr;
|
|
$sys[$cpu] = $sys;
|
|
$irq[$cpu] = $irq;
|
|
$soft[$cpu] = $soft;
|
|
$iowait[$cpu] = $iowait;
|
|
$idle[$cpu] = $idle;
|
|
$online_cpus = $cpu + 1; # mpstat is a source for # online CPUs
|
|
}
|
|
close MPSTATFILE;
|
|
}
|
|
|
|
# print the interesting CPU statistics from mpstat
|
|
# collected in collect_mpstat()
|
|
sub print_mpstat {
|
|
|
|
# header line
|
|
printf("%13s ", "CPU usage:");
|
|
print_cpulist();
|
|
|
|
printf("%13s ", "\%usr:");
|
|
for (my $cpu = 0; $cpu < $online_cpus; $cpu++) {
|
|
printf("%6s ", $usr[$cpu]);
|
|
}
|
|
print "\n";
|
|
printf("%13s ", "\%sys:");
|
|
for (my $cpu = 0; $cpu < $online_cpus; $cpu++) {
|
|
printf("%6s ", $sys[$cpu]);
|
|
}
|
|
print "\n";
|
|
printf("%13s ", "\%irq:");
|
|
for (my $cpu = 0; $cpu < $online_cpus; $cpu++) {
|
|
printf("%6s ", $irq[$cpu]);
|
|
}
|
|
print "\n";
|
|
printf("%13s ", "\%soft:");
|
|
for (my $cpu = 0; $cpu < $online_cpus; $cpu++) {
|
|
printf("%6s ", $soft[$cpu]);
|
|
}
|
|
print "\n";
|
|
printf("%13s ", "\%iowait idle:"); # clarify that iowait is really idle time
|
|
for (my $cpu = 0; $cpu < $online_cpus; $cpu++) {
|
|
printf("%6s ", $iowait[$cpu]);
|
|
}
|
|
print "\n";
|
|
printf("%13s ", "\%idle:");
|
|
for (my $cpu = 0; $cpu < $online_cpus; $cpu++) {
|
|
printf("%6s ", $idle[$cpu]);
|
|
}
|
|
print "\n";
|
|
}
|
|
|
|
#
|
|
# start of program
|
|
#
|
|
foreach (@ARGV) {
|
|
if (/--version/ || /-V/) {
|
|
print "$myname version $myversion\n";
|
|
exit;
|
|
} elsif (/--help/ || /-h/) {
|
|
print_usage();
|
|
exit;
|
|
} elsif (!/[a-z]/) {
|
|
# assume an all-numeric argument is the interval
|
|
$interval = $_;
|
|
}
|
|
print "Collecting CPU and interrupt activity for $interval seconds between updates\n";
|
|
}
|
|
|
|
# remember the clear screen characters to avoid system() during run time
|
|
my $clearscreen = `clear`;
|
|
collect_hardirqs(1);
|
|
collect_softirqs(1);
|
|
while (1) {
|
|
collect_hardirqs(0);
|
|
collect_softirqs(0);
|
|
collect_mpstat($interval);
|
|
my $datestring = localtime();
|
|
print "${clearscreen}$myname $datestring interval: $interval s\n";
|
|
print_mpstat();
|
|
print_irqs();
|
|
|
|
# if collect_mpstat(), which delays, is commented out, sleep here
|
|
# sleep($interval);
|
|
}
|