678 lines
15 KiB
Perl
Executable File
678 lines
15 KiB
Perl
Executable File
#!/usr/bin/perl
|
|
|
|
# * Copyright (c) 2009-2011 Mellanox Technologies Ltd. All rights reserved.
|
|
# * Copyright (c) 2009-2011 System Fabric Works, Inc. All rights reserved.
|
|
# *
|
|
# * This software is available to you under a choice of one of two
|
|
# * licenses. You may choose to be licensed under the terms of the GNU
|
|
# * General Public License (GPL) Version 2, available from the file
|
|
# * COPYING in the main directory of this source tree, or the
|
|
# * OpenIB.org BSD license below:
|
|
# *
|
|
# * Redistribution and use in source and binary forms, with or
|
|
# * without modification, are permitted provided that the following
|
|
# * conditions are met:
|
|
# *
|
|
# * - Redistributions of source code must retain the above
|
|
# * copyright notice, this list of conditions and the following
|
|
# * disclaimer.
|
|
# *
|
|
# * - Redistributions in binary form must reproduce the above
|
|
# * copyright notice, this list of conditions and the following
|
|
# * disclaimer in the documentation and/or other materials
|
|
# * provided with the distribution.
|
|
# *
|
|
# * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
|
# * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
|
# * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
|
# * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
|
|
# * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
|
|
# * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
|
# * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
# * SOFTWARE.
|
|
#
|
|
|
|
use warnings;
|
|
use strict;
|
|
|
|
use File::Basename;
|
|
use File::Path qw(make_path);
|
|
use Getopt::Long;
|
|
|
|
my $help = 0;
|
|
my $no_persist = 0;
|
|
my $debug = 0;
|
|
my $force = 0;
|
|
my $linkonly = 0;
|
|
my $parms = "/sys/module/rdma_rxe/parameters";
|
|
my $modprobe_opt = "";
|
|
my $modprobe_checked = "0";
|
|
my $persistence_path = "/var/lib/rxe";
|
|
my $persistence_file = "${persistence_path}/rxe";
|
|
my $num_persistent = 0;
|
|
my $sys = "/sys/module/rdma_rxe/parameters";
|
|
my %rxe_names;
|
|
my @rxe_array;
|
|
my %eth_names;
|
|
my @eth_list;
|
|
my %eth_driver;
|
|
my %link_state;
|
|
my %link_speed;
|
|
my %eth_mtu;
|
|
my %ipv4_addr;
|
|
my %rxe_mtu;
|
|
my @persistence_array;
|
|
my %persistence_hash;
|
|
my @mlx4_port;
|
|
my @mlx4_ether;
|
|
my @roce_list;
|
|
|
|
# Read a file and return its contents as a string.
|
|
sub read_file {
|
|
my $filename = shift;
|
|
my $result = "";
|
|
|
|
if (open(FILE, $filename)) {
|
|
$result = <FILE>;
|
|
close FILE;
|
|
}
|
|
return $result;
|
|
}
|
|
|
|
#get mapping between rxe and eth devices
|
|
sub get_names {
|
|
my $i = 0;
|
|
|
|
foreach my $rxe (glob("/sys/class/infiniband/rxe*")) {
|
|
$rxe = basename($rxe);
|
|
my $eth = read_file("/sys/class/infiniband/$rxe/parent");
|
|
chomp($eth);
|
|
|
|
if (($eth =~ /[\w]+[\d]/)
|
|
&& ($rxe =~ /rxe[0123456789]/)) {
|
|
|
|
# hash ethername to rxename
|
|
$rxe_names{$eth} = $rxe;
|
|
$rxe_array[$i++] = $rxe;
|
|
|
|
# hash rxename to ethername
|
|
$eth_names{$rxe} = $eth;
|
|
}
|
|
}
|
|
}
|
|
|
|
# get list of Mellanox RoCE ports
|
|
sub get_mlx4_list {
|
|
my $i = 0;
|
|
|
|
foreach my $mlx4 (glob("/sys/class/infiniband/mlx4_*")) {
|
|
$mlx4 = basename($mlx4);
|
|
foreach my $port (glob("/sys/class/infiniband/$mlx4/ports/*")) {
|
|
$port = basename($port);
|
|
my $link = read_file("$port/link_layer");
|
|
chomp($link);
|
|
|
|
if ($link =~ "Ethernet") {
|
|
$roce_list[$i++] = "$mlx4:$port";
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
#collect per device information
|
|
sub get_dev_info {
|
|
my @list;
|
|
my @fields;
|
|
my @lines;
|
|
my $line;
|
|
my $eth;
|
|
my $drv;
|
|
my $np;
|
|
my $i = 0;
|
|
my $j = 0;
|
|
|
|
get_mlx4_list();
|
|
|
|
my @my_eth_list = ();
|
|
foreach my $my_eth_dev (glob("/sys/class/net/*")) {
|
|
$my_eth_dev = basename($my_eth_dev);
|
|
if ($my_eth_dev ne "bonding_masters"){
|
|
my $my_dev_type = read_file("/sys/class/net/${my_eth_dev}/type");
|
|
chomp($my_dev_type);
|
|
if ($my_dev_type == "1") {
|
|
push(@my_eth_list, "$my_eth_dev");
|
|
}
|
|
}
|
|
}
|
|
|
|
@list = @my_eth_list;
|
|
foreach $eth (@list) {
|
|
chomp($eth);
|
|
|
|
$eth_list[$i++] = $eth;
|
|
|
|
@lines = `ethtool -i $eth`;
|
|
foreach $line (@lines) {
|
|
chomp($line);
|
|
|
|
@fields = split(/\s+/, $line);
|
|
chomp($fields[0]);
|
|
|
|
if ($fields[0] =~ /driver:/) {
|
|
$drv = $fields[1];
|
|
$eth_driver{$eth} = $drv;
|
|
|
|
if ($drv =~ /mlx4_en/ && scalar(@roce_list) > 0 ) {
|
|
$eth_names{$roce_list[$j++]} = $eth;
|
|
}
|
|
}
|
|
}
|
|
|
|
# get link status
|
|
$link_state{$eth} = "";
|
|
$link_speed{$eth} = "";
|
|
|
|
@lines = `ethtool $eth`;
|
|
foreach $line (@lines) {
|
|
chomp($line);
|
|
|
|
@fields = split(/:/, $line);
|
|
if (defined($fields[1])) {
|
|
$fields[1] =~ s/^\s+//g;
|
|
if ($fields[0] =~ "Link detected") {
|
|
$link_state{$eth} = $fields[1];
|
|
}
|
|
}
|
|
elsif ($line =~ "10000baseT") {
|
|
$link_speed{$eth} = "10GigE";
|
|
}
|
|
}
|
|
|
|
$ipv4_addr{$eth} = " ";
|
|
$eth_mtu{$eth} = "";
|
|
|
|
@lines = `ip addr show $eth`;
|
|
foreach $line (@lines) {
|
|
# get IP address
|
|
if ($line =~ /inet /) {
|
|
$line =~ s/^\s+inet ([0-9.]+)\//$1 /g;
|
|
@fields = split(/\s+/, $line);
|
|
$ipv4_addr{$eth} = $fields[0];
|
|
}
|
|
|
|
# get ethernet mtu
|
|
if ($line =~ /mtu /) {
|
|
$line =~ s/^.*mtu //g;
|
|
@fields = split(/\s+/, $line);
|
|
$eth_mtu{$eth} = $fields[0];
|
|
}
|
|
}
|
|
}
|
|
|
|
# get rxe mtu
|
|
foreach my $rxe (@rxe_array) {
|
|
|
|
@lines = `ibv_devinfo -d $rxe`;
|
|
foreach $line (@lines) {
|
|
if ($line =~ "active_mtu") {
|
|
$line =~ s/^\s+active_mtu:\s+//g;
|
|
chomp($line);
|
|
|
|
$rxe_mtu{$rxe} = $line;
|
|
}
|
|
}
|
|
$rxe_mtu{$rxe} = "(?)" if (!$rxe_mtu{$rxe});
|
|
}
|
|
}
|
|
|
|
# return string or the string "###" if string is all whitespace
|
|
sub set_field {
|
|
my $fld = $_[0];
|
|
|
|
if (defined($fld) && $fld =~ /\S/) {
|
|
return $fld;
|
|
} else {
|
|
return "###";
|
|
}
|
|
}
|
|
|
|
# format status output into fixed width columns
|
|
sub status_print {
|
|
my @fields;
|
|
my $field;
|
|
my @flen = ();
|
|
my $num_fields = 0;
|
|
my $i;
|
|
my $pad;
|
|
my $line;
|
|
|
|
# one pass to size the columns
|
|
foreach $line (@_) {
|
|
@fields = split(/\s+/, $line);
|
|
$i = 0;
|
|
foreach $field (@fields) {
|
|
if (!defined($flen[$i])) {
|
|
$flen[$i] = length($field);
|
|
}
|
|
else {
|
|
$flen[$i] = max($flen[$i], length($field));
|
|
}
|
|
$i++;
|
|
}
|
|
|
|
if ($i > $num_fields) {
|
|
$num_fields = $i;
|
|
}
|
|
}
|
|
|
|
# one pass to print
|
|
foreach $line (@_) {
|
|
print " ";
|
|
@fields = split(/\s+/, $line);
|
|
for ($i = 0; $i < $num_fields; $i++) {
|
|
if (defined($fields[$i])) {
|
|
$pad = $flen[$i] - length($fields[$i]) + 2;
|
|
}
|
|
else {
|
|
$pad = $flen[$i] + 2;
|
|
}
|
|
if (defined($fields[$i]) && ($fields[$i] ne "###")) {
|
|
print "$fields[$i]";
|
|
}
|
|
else {
|
|
print " ";
|
|
}
|
|
printf("%*s", $pad, "");
|
|
}
|
|
print "\n";
|
|
}
|
|
}
|
|
|
|
# check driver load status
|
|
sub check_module_status {
|
|
if (-e $sys) {
|
|
return 0;
|
|
} else {
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
# print driver load status and ethertype for rdma_rxe and rdma_rxe_net
|
|
sub show_module_status {
|
|
print "rdma_rxe module not loaded\n" if (!(-e $sys));
|
|
}
|
|
|
|
# print rxe status
|
|
sub do_status {
|
|
my $instance = $_[0];
|
|
my $ln = 0;
|
|
my @outp;
|
|
my $rxe;
|
|
my $rmtu;
|
|
|
|
get_names();
|
|
get_dev_info();
|
|
show_module_status();
|
|
|
|
$outp[$ln++] = "Name\tLink\tDriver\t\tSpeed\tNMTU\tIPv4_addr\tRDEV\tRMTU";
|
|
|
|
foreach my $eth (@eth_list) {
|
|
|
|
# handle case where rxe_drivers are not loaded
|
|
if (defined($rxe_names{$eth})) {
|
|
$rxe = $rxe_names{$eth};
|
|
$rmtu = $rxe_mtu{$rxe};
|
|
}
|
|
else {
|
|
$rxe = "";
|
|
$rmtu = "";
|
|
}
|
|
|
|
if ((!defined($instance)
|
|
&& (($linkonly == 0) || ($link_state{$eth} =~ "yes")))
|
|
|| (defined($instance) && ($rxe =~ "$instance"))) {
|
|
$outp[$ln] = set_field("$eth");
|
|
$outp[$ln] .= "\t";
|
|
$outp[$ln] .= set_field("$link_state{$eth}");
|
|
$outp[$ln] .= "\t";
|
|
$outp[$ln] .= set_field(exists($eth_driver{$eth}) ? $eth_driver{$eth} : "");
|
|
$outp[$ln] .= "\t";
|
|
$outp[$ln] .= set_field("$link_speed{$eth}");
|
|
$outp[$ln] .= "\t";
|
|
$outp[$ln] .= set_field("$eth_mtu{$eth}");
|
|
$outp[$ln] .= "\t";
|
|
$outp[$ln] .= set_field("$ipv4_addr{$eth}");
|
|
$outp[$ln] .= "\t";
|
|
$outp[$ln] .= set_field("$rxe");
|
|
$outp[$ln] .= "\t";
|
|
$outp[$ln] .= set_field("$rmtu");
|
|
$ln++;
|
|
}
|
|
}
|
|
|
|
status_print(@outp);
|
|
}
|
|
|
|
# read file containing list of ethernet devices into a list
|
|
sub populate_persistence {
|
|
my $i = 0;
|
|
|
|
open FILE, $persistence_file;
|
|
while(<FILE>) {
|
|
my $line = $_;
|
|
chomp($line);
|
|
$line =~ s/^\s+//g;
|
|
if ($line =~ /[\w]+[\d]/) {
|
|
# in case we add fields later
|
|
my ($eth, $cruft) = split(/\s+/, $line, 2);
|
|
if ($eth =~ /^[\w]+[\d]/) {
|
|
$persistence_array[$i] = $eth;
|
|
$persistence_hash{$eth} = $i++;
|
|
}
|
|
}
|
|
}
|
|
close FILE;
|
|
|
|
$num_persistent = $i;
|
|
}
|
|
|
|
# print out list of ethernet devices to file
|
|
sub commit_persistent {
|
|
my $i;
|
|
my $eth;
|
|
|
|
open(PF, ">$persistence_file");
|
|
|
|
for ($i = 0; $i < $num_persistent; $i++) {
|
|
$eth = $persistence_array[$i];
|
|
if ($eth =~ /[\w]+[\d]/) {
|
|
print(PF "$persistence_array[$i]\n");
|
|
}
|
|
}
|
|
|
|
close(PF);
|
|
}
|
|
|
|
sub delete_persistent {
|
|
my $eth = $_[0];
|
|
|
|
if (defined($persistence_hash{$eth})) {
|
|
$persistence_array[$persistence_hash{$eth}] = "";
|
|
}
|
|
}
|
|
|
|
sub add_persistent {
|
|
my $eth = $_[0];
|
|
|
|
# Is this one already in the persistence list?
|
|
if (!defined($persistence_hash{$eth})) {
|
|
$persistence_array[$num_persistent] = $eth;
|
|
$persistence_hash{$eth} = $num_persistent;
|
|
$num_persistent++;
|
|
}
|
|
}
|
|
|
|
# add new rxe device to eth if not already up
|
|
sub rxe_add {
|
|
my $eth = $_[0];
|
|
|
|
if (!($eth =~ /[\w]+[\d]/)) {
|
|
print "eth_name ($eth) looks bogus\n";
|
|
return;
|
|
}
|
|
|
|
if (!defined($rxe_names{$eth})) {
|
|
system("echo '$eth' > $parms/add");
|
|
}
|
|
if (!$no_persist) {
|
|
add_persistent($eth);
|
|
commit_persistent();
|
|
}
|
|
}
|
|
|
|
sub rxe_remove {
|
|
my $arg2 = $_[0];
|
|
my $rxe;
|
|
my $eth;
|
|
|
|
print "remove $arg2\n" if ($debug > 0);
|
|
|
|
if ($arg2 =~ /[\w]+[\d]/) {
|
|
$eth = $arg2;
|
|
$rxe = $rxe_names{$eth};
|
|
}
|
|
elsif ($arg2 =~ /rxe[0123456789]/) {
|
|
$rxe = $arg2;
|
|
$eth = $eth_names{$rxe};
|
|
}
|
|
elsif ($arg2 eq "all") {
|
|
$rxe = "all";
|
|
}
|
|
|
|
if (($rxe eq "all") || ($rxe =~ /^rxe[0123456789]/)) {
|
|
my $cmd = "echo '$rxe' > $parms/remove";
|
|
#print "$cmd\n";
|
|
system($cmd);
|
|
if (!$no_persist) {
|
|
if ($rxe eq "all") {
|
|
unlink($persistence_file);
|
|
}
|
|
elsif ($eth =~/[\w]+[\d]/) {
|
|
delete_persistent($eth);
|
|
commit_persistent();
|
|
}
|
|
else {
|
|
print "Warning: Unable to resolve ethname; "
|
|
. "instance may persist on restart\n";
|
|
}
|
|
}
|
|
}
|
|
else {
|
|
print "rxe instance $rxe not found\n";
|
|
}
|
|
}
|
|
|
|
sub get_devinfo {
|
|
my $rxe = $_[0];
|
|
|
|
my $cmd = "ibv_devinfo -d $rxe";
|
|
return `$cmd`;
|
|
}
|
|
|
|
# allow unsupported modules to load in SLES11 if allowed
|
|
sub modprobe {
|
|
my $module = $_[0];
|
|
my $opts = $_[1];
|
|
my @lines;
|
|
my $line;
|
|
|
|
if ($modprobe_checked == "0") {
|
|
@lines = `modprobe -c`;
|
|
foreach $line (@lines) {
|
|
if ($line =~ /^allow_unsupported_modules *0/) {
|
|
$modprobe_opt = " --allow-unsupported-modules ";
|
|
last;
|
|
}
|
|
}
|
|
$modprobe_checked = "1";
|
|
}
|
|
|
|
if (!defined($opts)) {
|
|
$opts = "";
|
|
}
|
|
|
|
system("modprobe $modprobe_opt $module $opts");
|
|
}
|
|
|
|
# bring up rxe
|
|
sub do_start {
|
|
my $proto_str = "";
|
|
|
|
system("mkdir -p $persistence_path");
|
|
system("touch $persistence_file");
|
|
|
|
modprobe("ib_core");
|
|
modprobe("ib_uverbs");
|
|
modprobe("rdma_ucm");
|
|
modprobe("rdma_rxe");
|
|
|
|
populate_persistence();
|
|
system("udevadm control --reload");
|
|
|
|
foreach my $eth (@persistence_array) {
|
|
rxe_add($eth);
|
|
}
|
|
|
|
get_names();
|
|
|
|
foreach my $rxe (@rxe_array) {
|
|
my $stat = get_devinfo($rxe);
|
|
if ($stat =~ "PORT_DOWN") {
|
|
my $cmd = "ip link set $eth_names{$rxe} up";
|
|
system($cmd);
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
# check if argument is an integer
|
|
sub is_integer {
|
|
defined $_[0] && $_[0] =~ /^[+-]?\d+$/;
|
|
}
|
|
|
|
# remove all rxe devices and unload drivers
|
|
sub do_stop {
|
|
my $rxe;
|
|
|
|
foreach $rxe (@rxe_array) {
|
|
system("echo '$rxe' > $sys/remove");
|
|
}
|
|
|
|
if (-e $sys) {
|
|
system("rmmod rdma_rxe");
|
|
}
|
|
|
|
if (-e $sys) {
|
|
print "unable to unload drivers, reboot required\n";
|
|
}
|
|
}
|
|
|
|
sub do_debug {
|
|
my $arg2 = $_[0];
|
|
my $debugfile = "$parms/debug";
|
|
chomp($arg2);
|
|
|
|
if (!(-e "$debugfile")) {
|
|
print "Error: debug is compiled out of this rxe driver\n";
|
|
return;
|
|
}
|
|
|
|
if ($arg2 eq "on") { system("echo '31' > $debugfile"); }
|
|
elsif ($arg2 eq "off") { system("echo '0' > $debugfile"); }
|
|
elsif ($arg2 eq "0") { system("echo '0' > $debugfile"); }
|
|
elsif ($arg2 eq "") { }
|
|
elsif ($arg2 ge "0" && $arg2 le "31") {
|
|
system("echo '$arg2' > $debugfile");
|
|
}
|
|
else {
|
|
print "unrecognized debug cmd ($arg2)\n";
|
|
}
|
|
|
|
my $current = read_file($debugfile);
|
|
chomp($current);
|
|
if ($current > 0) {
|
|
print "Debug is ON ($current)\n";
|
|
}
|
|
elsif ($current == 0) {
|
|
print "Debug is OFF\n";
|
|
}
|
|
else {
|
|
print "Unrecognized debug value\n";
|
|
}
|
|
}
|
|
|
|
sub max {
|
|
my $a = $_[0];
|
|
my $b = $_[1];
|
|
return $a if ($a > $b);
|
|
return $b;
|
|
}
|
|
|
|
# show usage for rxe_cfg
|
|
sub usage {
|
|
print " Usage:\n";
|
|
print " rxe_cfg [options] start|stop|status|persistent\n";
|
|
print " rxe_cfg debug on|off|<num>\n";
|
|
print " rxe_cfg [-n] add <ndev>\n";
|
|
print " rxe_cfg [-n] remove <ndev>|<rdev>\n";
|
|
print "\n";
|
|
print " <ndev> = network device e.g. eth3\n";
|
|
print " <rdev> = rdma device e.g. rxe1\n";
|
|
print "\n";
|
|
print " Options:\n";
|
|
print " -h: print this usage information\n";
|
|
print " -n: do not make the configuration action persistent\n";
|
|
print " -v: print additional debug output\n";
|
|
print " -l: show status for interfaces with link up\n";
|
|
print " -p <num>: (start command only) - set ethertype\n";
|
|
}
|
|
|
|
sub main {
|
|
GetOptions(
|
|
"-h" => \$help,
|
|
"--help" => \$help,
|
|
"-n" => \$no_persist,
|
|
"-v:+" => \$debug,
|
|
"-f" => \$force,
|
|
"-l" => \$linkonly,
|
|
);
|
|
|
|
my $arg1 = $ARGV[0];
|
|
my $arg2 = $ARGV[1];
|
|
my $arg3 = $ARGV[2];
|
|
|
|
# status is the default
|
|
if (!defined($arg1) || ($arg1 =~ /status/)) {
|
|
do_status($arg2);
|
|
exit;
|
|
}
|
|
|
|
if ($help) {
|
|
usage();
|
|
exit;
|
|
}
|
|
|
|
# stuff that does not require modules to be loaded
|
|
if ($arg1 eq "help") { usage(); exit; }
|
|
elsif ($arg1 eq "start") { do_start(); do_status(); exit; }
|
|
elsif ($arg1 eq "persistent") { system("cat $persistence_file"); exit; }
|
|
|
|
|
|
# can't do much else, bail if modules aren't loaded
|
|
if (check_module_status()) {
|
|
exit;
|
|
}
|
|
|
|
# create persistence file if necessary
|
|
make_path($persistence_path);
|
|
if (!(-e $persistence_file)) {
|
|
`touch $persistence_file`;
|
|
}
|
|
|
|
# Get full context of the configuration
|
|
populate_persistence();
|
|
get_names();
|
|
get_dev_info();
|
|
|
|
# Stuff that requires the rdma_rxe module to be loaded
|
|
if ($arg1 eq "stop") { do_stop(); exit; }
|
|
elsif ($arg1 eq "debug") { do_debug($arg2); exit; }
|
|
elsif ($arg1 eq "add") { rxe_add($arg2); exit; }
|
|
elsif ($arg1 eq "remove") { rxe_remove($arg2); exit; }
|
|
elsif ($arg1 eq "help") { usage(); exit; }
|
|
}
|
|
|
|
main();
|
|
|
|
exit;
|