diff --git a/.gitignore b/.gitignore index a8d57cf..e49ab0e 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1 @@ -SOURCES/perftest-4.4-0.29.g817ec38.tar.gz +SOURCES/perftest-4.4-0.32.g6fc89a9.tar.gz diff --git a/.perftest.metadata b/.perftest.metadata index 3e8bcdf..f106aff 100644 --- a/.perftest.metadata +++ b/.perftest.metadata @@ -1 +1 @@ -26e318b80b7e2464a139a056332f2400864d4f61 SOURCES/perftest-4.4-0.29.g817ec38.tar.gz +11371a27413a0371e36fe6a27cacd5a72e27fc05 SOURCES/perftest-4.4-0.32.g6fc89a9.tar.gz diff --git a/SOURCES/0001-Check-PCIe-relaxed-ordering-compliant.patch b/SOURCES/0001-Check-PCIe-relaxed-ordering-compliant.patch new file mode 100644 index 0000000..0a39c55 --- /dev/null +++ b/SOURCES/0001-Check-PCIe-relaxed-ordering-compliant.patch @@ -0,0 +1,131 @@ +From 0eeba58a2a5273b5f9dd95d3e219a2d4ad003244 Mon Sep 17 00:00:00 2001 +From: Honggang Li +Date: Mon, 11 Jan 2021 16:59:08 +0800 +Subject: [PATCH] Check PCIe relaxed ordering compliant + +Significantly performance degradation may be observed when PICe +relaxed ordering enabled over CPU which is not PCIe RO compliant. +Emit a warning message for such scenario. + +https://github.com/linux-rdma/perftest/issues/116 + +Signed-off-by: Honggang Li +--- + README | 3 ++- + configure.ac | 3 +++ + perftest.spec | 1 + + src/perftest_parameters.c | 37 +++++++++++++++++++++++++++++++++++++ + 4 files changed, 43 insertions(+), 1 deletion(-) + +diff --git a/README b/README +index 23f81cb..707d9df 100644 +--- a/README ++++ b/README +@@ -108,7 +108,8 @@ Prerequisites: + (kernel module) matches libibverbs + (kernel module) matches librdmacm + (kernel module) matches libibumad +- (kernel module) matches libmath (lm). ++ (kernel module) matches libmath (lm) ++ (linux kernel module) matches pciutils (lpci). + + + Server: ./ +diff --git a/configure.ac b/configure.ac +index 0a1cc41..66ec909 100755 +--- a/configure.ac ++++ b/configure.ac +@@ -174,6 +174,8 @@ if [test $HAVE_SNIFFER = yes]; then + fi + + if [test $IS_FREEBSD = no]; then ++ AC_CHECK_HEADERS([pci/pci.h],,[AC_MSG_ERROR([pciutils header files not found])]) ++ AC_CHECK_LIB([pci], [pci_init], [LIBPCI=-lpci], AC_MSG_ERROR([libpci not found])) + CPU_IS_RO_COMPLIANT=yes + # Actually this is check for being affected by a known issue + # with Intel CPUs: +@@ -193,6 +195,7 @@ AC_TRY_LINK([ + AM_CONDITIONAL([HAVE_RO],[test "x$HAVE_RO" = "xyes"]) + if [test $HAVE_RO = yes] && [test "x$CPU_IS_RO_COMPLIANT" = "xyes"]; then + AC_DEFINE([HAVE_RO], [1], [Enable Relaxed Ordering]) ++ LIBS=$LIBS" -lpci" + fi + + AC_TRY_LINK([#include ], +diff --git a/perftest.spec b/perftest.spec +index 630b00b..b0b50da 100644 +--- a/perftest.spec ++++ b/perftest.spec +@@ -8,6 +8,7 @@ Source: http://www.openfabrics.org/downloads/%{name}-%{version}.tar.gz + Url: http://www.openfabrics.org + BuildRoot: %{_tmppath}/%{name}-%{version}-build + BuildRequires: libibverbs-devel librdmacm-devel libibumad-devel ++BuildRequires: pciutils-devel + + %description + gen3 uverbs microbenchmarks +diff --git a/src/perftest_parameters.c b/src/perftest_parameters.c +index 185d7e3..77fe835 100755 +--- a/src/perftest_parameters.c ++++ b/src/perftest_parameters.c +@@ -11,6 +11,10 @@ + #include "perftest_parameters.h" + #include "raw_ethernet_resources.h" + #include ++#ifdef HAVE_RO ++#include ++#include ++#endif + #define MAC_LEN (17) + #define ETHERTYPE_LEN (6) + #define MAC_ARR_LEN (6) +@@ -163,6 +167,34 @@ static int get_cache_line_size() + + return size; + } ++#ifdef HAVE_RO ++/****************************************************************************** ++ Check PCIe Relaxed Ordering ++ ++ Stolen from https://github.com/pciutils/pciutils/blob/master/example.c ++ ******************************************************************************/ ++static bool check_pcie_relaxed_ordering_compliant(void) { ++ struct pci_access *pacc; ++ struct pci_dev *dev; ++ bool cpu_is_RO_compliant = true; ++ ++ pacc = pci_alloc(); ++ pci_init(pacc); ++ pci_scan_bus(pacc); ++ for (dev = pacc->devices; dev && cpu_is_RO_compliant; ++ dev = dev->next) { ++ pci_fill_info(dev, ++ PCI_FILL_IDENT | PCI_FILL_BASES | PCI_FILL_CLASS); ++ /* https://lore.kernel.org/patchwork/patch/820922/ */ ++ if ((dev->vendor_id == 0x8086) && ++ (((dev->device_id >= 0x6f01 && dev->device_id <= 0x6f0e) || ++ (dev->device_id >= 0x2f01 && dev->device_id <= 0x2f01)))) ++ cpu_is_RO_compliant = false; ++ } ++ pci_cleanup(pacc); ++ return cpu_is_RO_compliant; ++} ++#endif + /****************************************************************************** + * + ******************************************************************************/ +@@ -2873,6 +2905,11 @@ void ctx_print_test_info(struct perftest_parameters *user_param) + printf(" Connection type : %s\t\tUsing SRQ : %s\n", connStr[user_param->connection_type], user_param->use_srq ? "ON" : "OFF"); + #ifdef HAVE_RO + printf(" PCIe relax order: %s\n", user_param->disable_pcir ? "OFF" : "ON"); ++ if ((check_pcie_relaxed_ordering_compliant() == false) && ++ (user_param->disable_pcir == 0)) { ++ printf(" WARNING: CPU is not PCIe relaxed ordering compliant.\n"); ++ printf(" WARNING: You should disable PCIe RO with `--disable_pcie_relaxed` for both server and clinet.\n"); ++ } + #else + printf(" PCIe relax order: %s\n", "Unsupported"); + #endif +-- +2.29.2 + diff --git a/SOURCES/0001-Don-t-call-ibv_qp_to_qp_ex-when-hardware-does-not-su.patch b/SOURCES/0001-Don-t-call-ibv_qp_to_qp_ex-when-hardware-does-not-su.patch deleted file mode 100644 index 3751384..0000000 --- a/SOURCES/0001-Don-t-call-ibv_qp_to_qp_ex-when-hardware-does-not-su.patch +++ /dev/null @@ -1,73 +0,0 @@ -From d30e91efc80d058ddad80913f40dffb1e68c841b Mon Sep 17 00:00:00 2001 -From: Honggang Li -Date: Fri, 24 Jul 2020 09:09:00 +0800 -Subject: [PATCH] Don't call ibv_qp_to_qp_ex when hardware does not support - extended QP - -The sizeof 'struct ibv_qp' allocated by ibv_create_qp is 160. -If the memory holds the 'struct ibv_qp' was allocated at the -upper boundary of a memory page, cast it to 'struct verbs_qp', -whose size is 360, may across the memory page boundary. It will -trigger invalid memory access to the next memory page. - -The issue can be reproduced with OPA and QIB HCA. - -For example run over OPA: - Server Node: $ ib_read_bw -F -N -n 1000 -u 20 -q 257 -s 4194304 - Client Node: $ ib_read_bw -F -N -n 1000 -u 20 -q 257 -s 4194304 - - Program received signal SIGSEGV, Segmentation fault. - ibv_qp_to_qp_ex (qp=0x5555557a5f10) at libibverbs/verbs.c:624 - 624 if (vqp->comp_mask & VERBS_QP_EX) - (gdb) bt - #0 ibv_qp_to_qp_ex (qp=0x5555557a5f10) at libibverbs/verbs.c:624 - #1 0x000055555556af4a in create_reg_qp_main (ctx=ctx@entry=0x7fffffffd500, user_param=user_param@entry=0x7fffffffd670, i=i@entry=21, num_of_qps=num_of_qps@entry=128) at src/perftest_resources.c:1597 - #2 0x000055555556b6d7 in create_qp_main (num_of_qps=, i=21, user_param=0x7fffffffd670, ctx=0x7fffffffd500) at src/perftest_resources.c:1613 - #3 ctx_init (ctx=0x7fffffffd500, user_param=0x7fffffffd670) at src/perftest_resources.c:1552 - #4 0x0000555555558e9c in main (argc=, argv=) at src/read_bw.c:149 - -624 if (vqp->comp_mask & VERBS_QP_EX) -(gdb) p qp -$1 = (struct ibv_qp *) 0x5555557a5f10 -(gdb) p vqp -$2 = (struct verbs_qp *) 0x5555557a5f10 -(gdb) p *qp -$3 = {context = 0x55555578ad00, qp_context = 0x0, .... -(gdb) p *vqp -Cannot access memory at address 0x5555557a6000 - -Signed-off-by: Honggang Li ---- - src/perftest_resources.c | 14 ++++++++------ - 1 file changed, 8 insertions(+), 6 deletions(-) - -diff --git a/src/perftest_resources.c b/src/perftest_resources.c -index 56b78bba2c4b..499820c93185 100755 ---- a/src/perftest_resources.c -+++ b/src/perftest_resources.c -@@ -1649,14 +1649,16 @@ int create_reg_qp_main(struct pingpong_context *ctx, - return FAILURE; - } - #ifdef HAVE_IBV_WR_API -- ctx->qpx[i] = ibv_qp_to_qp_ex(ctx->qp[i]); -- #ifdef HAVE_MLX5DV -- if (user_param->connection_type == DC) -- { -- ctx->dv_qp[i] = mlx5dv_qp_ex_from_ibv_qp_ex(ctx->qpx[i]); -+ if (!user_param->use_old_post_send) { -+ ctx->qpx[i] = ibv_qp_to_qp_ex(ctx->qp[i]); -+ #ifdef HAVE_MLX5DV -+ if (user_param->connection_type == DC) -+ { -+ ctx->dv_qp[i] = mlx5dv_qp_ex_from_ibv_qp_ex(ctx->qpx[i]); -+ } -+ #endif - } - #endif -- #endif - - return SUCCESS; - } --- -2.25.4 - diff --git a/SPECS/perftest.spec b/SPECS/perftest.spec index 83ff363..462c413 100644 --- a/SPECS/perftest.spec +++ b/SPECS/perftest.spec @@ -1,16 +1,17 @@ Name: perftest Summary: IB Performance Tests Version: 4.4 -%define minor_release 0.29 -%define git_hash g817ec38 -Release: 3%{?dist} +%define minor_release 0.32 +%define git_hash g6fc89a9 +Release: 8%{?dist} License: GPLv2 or BSD Source0: https://github.com/linux-rdma/perftest/releases/download/V%{version}-%{minor_release}/perftest-%{version}-%{minor_release}.%{git_hash}.tar.gz Source1: ib_atomic_bw.1 -Patch1: 0001-Don-t-call-ibv_qp_to_qp_ex-when-hardware-does-not-su.patch -Url: http://www.openfabrics.org +Patch1: 0001-Check-PCIe-relaxed-ordering-compliant.patch +Url: https://github.com/linux-rdma/perftest BuildRequires: libibverbs-devel > 1.1.4, librdmacm-devel > 1.0.14 BuildRequires: libibumad-devel > 1.3.6 +BuildRequires: pciutils-devel BuildRequires: autoconf, automake, libtool Obsoletes: openib-perftest < 1.3 @@ -27,7 +28,7 @@ autoreconf --force --install %build %configure -make V=1 %{?_smp_mflags} +%make_build %install for file in ib_{atomic,read,send,write}_{lat,bw}; do @@ -45,11 +46,20 @@ done popd %files -%doc README COPYING +%doc README %{_bindir}/* %{_mandir}/man1/* +%license COPYING %changelog +* Sat Jan 30 2021 Honggang Li - 4.4-8 +- Check PCIe relaxed ordering compliant +- Resolves: rhbz#1902855 + +* Thu Nov 05 2020 Honggang Li - 4.4-7 +- Rebase to upstream release perftest-4.4-0.32 +- Resolves: bz1888570 + * Fri Jul 24 2020 Honggang Li - 4.4-3 - Fix segment fault with large QP numbers - Resolves: rhbz#1859358