rdma-core/rxe_cfg

#!/usr/bin/perl

# * Copyright (c) 2009-2011 Mellanox Technologies Ltd. All rights reserved.
# * Copyright (c) 2009-2011 System Fabric Works, Inc. All rights reserved.
# *
# * This software is available to you under a choice of one of two
# * licenses.  You may choose to be licensed under the terms of the GNU
# * General Public License (GPL) Version 2, available from the file
# * COPYING in the main directory of this source tree, or the
# * OpenIB.org BSD license below:
# *
# *     Redistribution and use in source and binary forms, with or
# *     without modification, are permitted provided that the following
# *     conditions are met:
# *
# *	- Redistributions of source code must retain the above
# *	  copyright notice, this list of conditions and the following
# *	  disclaimer.
# *
# *	- Redistributions in binary form must reproduce the above
# *	  copyright notice, this list of conditions and the following
# *	  disclaimer in the documentation and/or other materials
# *	  provided with the distribution.
# *
# * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
# * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
# * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
# * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# * SOFTWARE.
#

use warnings;
use strict;

use File::Basename;
use File::Path qw(make_path);
use Getopt::Long;

my $help = 0;
my $no_persist = 0;
my $debug = 0;
my $force = 0;
my $linkonly = 0;
my $parms = "/sys/module/rdma_rxe/parameters";
my $modprobe_opt = "";
my $modprobe_checked = "0";
my $persistence_path = "/var/lib/rxe";
my $persistence_file = "${persistence_path}/rxe";
my $num_persistent = 0;
my $sys = "/sys/module/rdma_rxe/parameters";
my %rxe_names;
my @rxe_array;
my %eth_names;
my @eth_list;
my %eth_driver;
my %link_state;
my %link_speed;
my %eth_mtu;
my %ipv4_addr;
my %rxe_mtu;
my @persistence_array;
my %persistence_hash;
my @mlx4_port;
my @mlx4_ether;
my @roce_list;

# Read a file and return its contents as a string.
sub read_file {
    my $filename = shift;
    my $result = "";

    if (open(FILE, $filename)) {
	$result = <FILE>;
	close FILE;
    }
    return $result;
}

#get mapping between rxe and eth devices
sub get_names {
    my $i = 0;
    
    foreach my $rxe (glob("/sys/class/infiniband/rxe*")) {
	$rxe = basename($rxe);
	my $eth = read_file("/sys/class/infiniband/$rxe/parent");
	chomp($eth);
	
	if (($eth =~ /[\w]+[\d]/)
	    && ($rxe =~ /rxe[0123456789]/)) {
	    
	    # hash ethername to rxename
	    $rxe_names{$eth} = $rxe;
	    $rxe_array[$i++] = $rxe;
	    
	    # hash rxename to ethername
	    $eth_names{$rxe} = $eth;
	}
    }
}

# get list of Mellanox RoCE ports
sub get_mlx4_list {
    my $i = 0;

    foreach my $mlx4 (glob("/sys/class/infiniband/mlx4_*")) {
	$mlx4 = basename($mlx4);
	foreach my $port (glob("/sys/class/infiniband/$mlx4/ports/*")) {
	    $port = basename($port);
	    my $link = read_file("$port/link_layer");
	    chomp($link);

	    if ($link =~ "Ethernet") {
		$roce_list[$i++] = "$mlx4:$port";
	    }
	}
    }
}

#collect per device information
sub get_dev_info {
    my @list;
    my @fields;
    my @lines;
    my $line;
    my $eth;
    my $drv;
    my $np;
    my $i = 0;
    my $j = 0;

    get_mlx4_list();

    my @my_eth_list = ();
    foreach my $my_eth_dev (glob("/sys/class/net/*")) {
       $my_eth_dev = basename($my_eth_dev);
          if ($my_eth_dev ne "bonding_masters"){
             my $my_dev_type = read_file("/sys/class/net/${my_eth_dev}/type");
             chomp($my_dev_type);
             if ($my_dev_type == "1") {
                push(@my_eth_list, "$my_eth_dev");
             }
          }
    }

    @list = @my_eth_list;
    foreach $eth (@list) {
	chomp($eth);

	$eth_list[$i++] = $eth;

	@lines = `ethtool -i $eth`;
	foreach $line (@lines) {
	    chomp($line);

	    @fields = split(/\s+/, $line);
	    chomp($fields[0]);

	    if ($fields[0] =~ /driver:/) {
		$drv = $fields[1];
		$eth_driver{$eth} = $drv;

		if ($drv =~ /mlx4_en/ && scalar(@roce_list) > 0 ) {
		    $eth_names{$roce_list[$j++]} = $eth;
		}
	    }
	}

	# get link status
	$link_state{$eth} = "";
	$link_speed{$eth} = "";

	@lines = `ethtool $eth`;
	foreach $line (@lines) {
	    chomp($line);

	    @fields = split(/:/, $line);
	    if (defined($fields[1])) {
		    $fields[1] =~ s/^\s+//g;
		    if ($fields[0] =~ "Link detected") {
			$link_state{$eth} = $fields[1];
		    }
	    }
	    elsif ($line =~ "10000baseT") {
		$link_speed{$eth} = "10GigE";
	    }
	}

	$ipv4_addr{$eth} = "            ";
	$eth_mtu{$eth} = "";

	@lines = `ip addr show $eth`;
	foreach $line (@lines) {
		# get IP address
		if ($line =~ /inet /) {
			$line =~ s/^\s+inet ([0-9.]+)\//$1 /g;
			@fields = split(/\s+/, $line);
			$ipv4_addr{$eth} = $fields[0];
		}

		# get ethernet mtu
		if ($line =~ /mtu /) {
			$line =~ s/^.*mtu //g;
			@fields = split(/\s+/, $line);
			$eth_mtu{$eth} = $fields[0];
		}
    }
    }

    # get rxe mtu
    foreach my $rxe (@rxe_array) {
	
	@lines = `ibv_devinfo -d $rxe`;
	foreach $line (@lines) {
	    if ($line =~ "active_mtu") {
		$line =~ s/^\s+active_mtu:\s+//g;
		chomp($line);

		$rxe_mtu{$rxe} = $line;
	    }
	}
	$rxe_mtu{$rxe} = "(?)" if (!$rxe_mtu{$rxe});
    }
}

# return string or the string "###" if string is all whitespace
sub set_field {
    my $fld = $_[0];

    if (defined($fld) && $fld =~ /\S/) {
        return $fld;
    } else {
        return "###";
    }
}

# format status output into fixed width columns
sub status_print {
    my @fields;
    my $field;
    my @flen = ();
    my $num_fields = 0;
    my $i;
    my $pad;
    my $line;

    # one pass to size the columns
    foreach $line (@_) {
	@fields = split(/\s+/, $line);
	$i = 0;
	foreach $field (@fields) {
	    if (!defined($flen[$i])) {
		$flen[$i] = length($field);
	    }
	    else {
		$flen[$i] = max($flen[$i], length($field));
	    }
	    $i++;
	}

	if ($i > $num_fields) {
	    $num_fields = $i;
	}
    }

    # one pass to print
    foreach $line (@_) {
	print "  ";
	@fields = split(/\s+/, $line);
	for ($i = 0; $i < $num_fields; $i++) {
	    if (defined($fields[$i])) {
	        $pad = $flen[$i] - length($fields[$i]) + 2;
	    }
	    else {
	        $pad = $flen[$i] + 2;
	    }
	    if (defined($fields[$i]) && ($fields[$i] ne "###")) {
		print "$fields[$i]";
	    }
	    else {
		print "   ";
	    }
	    printf("%*s", $pad, "");
	}
	print "\n";
    }
}

# check driver load status
sub check_module_status {
    if (-e $sys) {
	return 0;
    } else {
	return 1;
    }
}

# print driver load status and ethertype for rdma_rxe and rdma_rxe_net
sub show_module_status {
    print "rdma_rxe module not loaded\n" if (!(-e $sys));
}

# print rxe status
sub do_status {
    my $instance = $_[0];
    my $ln = 0;
    my @outp;
    my $rxe;
    my $rmtu;

    get_names();
    get_dev_info();
    show_module_status();

    $outp[$ln++] = "Name\tLink\tDriver\t\tSpeed\tNMTU\tIPv4_addr\tRDEV\tRMTU";

    foreach my $eth (@eth_list) {

	# handle case where rxe_drivers are not loaded
	if (defined($rxe_names{$eth})) {
		$rxe = $rxe_names{$eth};
		$rmtu = $rxe_mtu{$rxe};
	}
	else {
		$rxe = "";
		$rmtu = "";
	}

	if ((!defined($instance) 
	     && (($linkonly == 0) || ($link_state{$eth} =~ "yes")))
	    || (defined($instance) && ($rxe =~ "$instance"))) {
	    $outp[$ln] =  set_field("$eth");
	    $outp[$ln] .= "\t";
	    $outp[$ln] .= set_field("$link_state{$eth}");
	    $outp[$ln] .= "\t";
	    $outp[$ln] .= set_field(exists($eth_driver{$eth}) ? $eth_driver{$eth} : "");
	    $outp[$ln] .= "\t";
	    $outp[$ln] .= set_field("$link_speed{$eth}");
	    $outp[$ln] .= "\t";
	    $outp[$ln] .= set_field("$eth_mtu{$eth}");
	    $outp[$ln] .= "\t";
	    $outp[$ln] .= set_field("$ipv4_addr{$eth}");
	    $outp[$ln] .= "\t";
	    $outp[$ln] .= set_field("$rxe");
	    $outp[$ln] .= "\t";
	    $outp[$ln] .= set_field("$rmtu");
	    $ln++;
	}
    }

    status_print(@outp);
}

# read file containing list of ethernet devices into a list
sub populate_persistence {
    my $i = 0;
    
    open FILE, $persistence_file;
    while(<FILE>) {
	my $line = $_;
	chomp($line);
	$line =~ s/^\s+//g;
	if ($line =~ /[\w]+[\d]/) {
	    # in case we add fields later
	    my ($eth, $cruft) = split(/\s+/, $line, 2);
	    if ($eth =~ /^[\w]+[\d]/) {
		$persistence_array[$i] = $eth;
		$persistence_hash{$eth} = $i++;
	    }
	}
    }
    close FILE;

    $num_persistent = $i;
}

# print out list of ethernet devices to file
sub commit_persistent {
    my $i;
    my $eth;

    open(PF, ">$persistence_file");
    
    for ($i = 0; $i < $num_persistent; $i++) {
	$eth = $persistence_array[$i];
	if ($eth =~ /[\w]+[\d]/) {
	    print(PF "$persistence_array[$i]\n");
	}
    }

    close(PF);
}

sub delete_persistent {
    my $eth = $_[0];
    
    if (defined($persistence_hash{$eth})) {
	$persistence_array[$persistence_hash{$eth}] = "";
    }
}

sub add_persistent {
    my $eth = $_[0];

    # Is this one already in the persistence list?
    if (!defined($persistence_hash{$eth})) {
	$persistence_array[$num_persistent] = $eth;
	$persistence_hash{$eth} = $num_persistent;
	$num_persistent++;
    }
}

# add new rxe device to eth if not already up
sub rxe_add {
    my $eth = $_[0];

    if (!($eth =~ /[\w]+[\d]/)) {
	print "eth_name ($eth) looks bogus\n";
	return;
    }

    if (!defined($rxe_names{$eth})) {
	system("echo '$eth' > $parms/add");
    }
    if (!$no_persist) {
	add_persistent($eth);
	commit_persistent();
    }
}

sub rxe_remove {
    my $arg2 = $_[0];
    my $rxe;
    my $eth;

    print "remove $arg2\n"  if ($debug > 0);

    if ($arg2 =~ /[\w]+[\d]/) {
	$eth = $arg2;
	$rxe = $rxe_names{$eth};
    }
    elsif ($arg2 =~ /rxe[0123456789]/) {
	$rxe = $arg2;
	$eth = $eth_names{$rxe};
    }
    elsif ($arg2 eq "all") {
	$rxe = "all";
    }

    if (($rxe eq "all") || ($rxe =~ /^rxe[0123456789]/)) {
	my $cmd = "echo '$rxe' > $parms/remove";
	#print "$cmd\n";
	system($cmd);
	if (!$no_persist) {
	    if ($rxe eq "all") {
		unlink($persistence_file);
	    }
	    elsif ($eth =~/[\w]+[\d]/) {
		delete_persistent($eth);
		commit_persistent();
	    }
	    else {
		print "Warning: Unable to resolve ethname; "
		    . "instance may persist on restart\n";
	    }
	}
    }
    else {
	print "rxe instance $rxe not found\n";
    }
}

sub get_devinfo {
    my $rxe = $_[0];

    my $cmd = "ibv_devinfo -d $rxe";
    return `$cmd`;
}

# allow unsupported modules to load in SLES11 if allowed
sub modprobe {
    my $module = $_[0];
    my $opts = $_[1];
    my @lines;
    my $line;

    if ($modprobe_checked == "0") {
	@lines = `modprobe -c`;
	foreach $line (@lines) {
	    if ($line =~ /^allow_unsupported_modules  *0/) {
		$modprobe_opt = " --allow-unsupported-modules ";
		last;
	    }
	}
	$modprobe_checked = "1";
    }

    if (!defined($opts)) {
	$opts = "";
    }

    system("modprobe $modprobe_opt $module $opts");
}

# bring up rxe
sub do_start {
    my $proto_str = "";

    system("mkdir -p $persistence_path");
    system("touch $persistence_file");

    modprobe("ib_core");
    modprobe("ib_uverbs");
    modprobe("rdma_ucm");
    modprobe("rdma_rxe");

    populate_persistence();
    system("udevadm control --reload");

    foreach my $eth (@persistence_array) {
	rxe_add($eth);
    }

    get_names();

    foreach my $rxe (@rxe_array) {
	my $stat = get_devinfo($rxe);
	if ($stat =~ "PORT_DOWN") {
		my $cmd = "ip link set $eth_names{$rxe} up";
		system($cmd);
	}
    }

}

# check if argument is an integer
sub is_integer {
    defined $_[0] && $_[0] =~ /^[+-]?\d+$/;
}

# remove all rxe devices and unload drivers
sub do_stop {
    my $rxe;

    foreach $rxe (@rxe_array) {
	system("echo '$rxe' > $sys/remove");
    }

    if (-e $sys) {
	system("rmmod rdma_rxe");
    }

    if (-e $sys) {
	print "unable to unload drivers, reboot required\n";
    }
}

sub do_debug {
    my $arg2 = $_[0];
    my $debugfile = "$parms/debug";
    chomp($arg2);

    if (!(-e "$debugfile")) {
	print "Error: debug is compiled out of this rxe driver\n";
	return;
    }

    if    ($arg2 eq "on")  { system("echo '31' > $debugfile"); }
    elsif ($arg2 eq "off") { system("echo '0'  > $debugfile"); }
    elsif ($arg2 eq "0")   { system("echo '0'  > $debugfile"); }
    elsif ($arg2 eq "")    { }
	elsif ($arg2 ge "0" && $arg2 le "31") {
	    system("echo '$arg2' > $debugfile");
	}
	else {
	    print "unrecognized debug cmd ($arg2)\n";
	}

    my $current = read_file($debugfile);
    chomp($current);
    if ($current > 0) {
	print "Debug is ON ($current)\n";
    }
    elsif ($current == 0) {
	print "Debug is OFF\n";
    }
    else {
	print "Unrecognized debug value\n";
    }
}

sub max {
    my $a = $_[0];
    my $b = $_[1];
    return $a if ($a > $b);
    return $b;
}

# show usage for rxe_cfg
sub usage {
    print "  Usage:\n";
    print "    rxe_cfg [options] start|stop|status|persistent\n";
    print "    rxe_cfg debug on|off|<num>\n";
    print "    rxe_cfg [-n] add <ndev>\n";
    print "    rxe_cfg [-n] remove <ndev>|<rdev>\n";
    print "\n";
    print "    <ndev> = network device e.g. eth3\n";
    print "    <rdev> = rdma device e.g. rxe1\n";
    print "\n";
    print "  Options:\n";
    print "    -h: print this usage information\n";
    print "    -n: do not make the configuration action persistent\n";
    print "    -v: print additional debug output\n";
    print "    -l: show status for interfaces with link up\n";
    print "    -p <num>: (start command only) - set ethertype\n";
}

sub main {
    GetOptions(
	   "-h"          => \$help,
	   "--help"      => \$help,
	   "-n"          => \$no_persist,
	   "-v:+"        => \$debug,
	   "-f"          => \$force,
	   "-l"          => \$linkonly,
	   );

    my $arg1 = $ARGV[0];
    my $arg2 = $ARGV[1];
    my $arg3 = $ARGV[2];

    # status is the default
    if (!defined($arg1) || ($arg1 =~ /status/)) {
        do_status($arg2);
        exit;
    }

    if ($help) {
        usage();
        exit;
    }

    # stuff that does not require modules to be loaded
    if    ($arg1 eq "help")       { usage(); exit; }
    elsif ($arg1 eq "start")      { do_start(); do_status(); exit; }
    elsif ($arg1 eq "persistent") { system("cat $persistence_file"); exit; }


    # can't do much else, bail if modules aren't loaded
    if (check_module_status()) {
	exit;
    }

    # create persistence file if necessary
    make_path($persistence_path);
    if (!(-e $persistence_file)) {
        `touch $persistence_file`;
    }

    # Get full context of the configuration
    populate_persistence();
    get_names();
    get_dev_info();

    # Stuff that requires the rdma_rxe module to be loaded
    if    ($arg1 eq "stop")   { do_stop(); 	   exit; }
    elsif ($arg1 eq "debug")  { do_debug($arg2);   exit; }
    elsif ($arg1 eq "add")    { rxe_add($arg2);    exit; }
    elsif ($arg1 eq "remove") { rxe_remove($arg2); exit; }
    elsif ($arg1 eq "help")   { usage();	   exit; }
}

main();

exit;