rpms/nbdkit
7
0
Fork 0
Code Issues Pull Requests Releases Activity

Improve test times.

Browse Source
This commit is contained in:
Richard W.M. Jones 2019-12-14 12:40:02 +00:00
parent 74c42afbe8
commit 103ba5091c
3 changed files with 553 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

167
0001-tests-Reduce-excessive-time-taken-by-some-tests.patch Normal file
View File

@ -0,0 +1,167 @@
From fdbd0ef69f0c222d0cdc35d3c48297aff549abce Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Fri, 13 Dec 2019 15:38:09 +0000
Subject: [PATCH 1/2] tests: Reduce excessive time taken by some tests.
For context see:
https://www.redhat.com/archives/libguestfs/2019-December/msg00074.html
This is mainly about changing tests to use libnbd/nbdsh instead of
libguestfs where libguestfs isn't really necessary for the test.
---
tests/test-cache-on-read.sh | 37 ++++++++++++++++---------------------
tests/test-cache.sh | 37 ++++++++++++++++++++-----------------
tests/test-cow.sh | 9 ++++++---
3 files changed, 42 insertions(+), 41 deletions(-)
diff --git a/tests/test-cache-on-read.sh b/tests/test-cache-on-read.sh
index 4ecf3af..6196373 100755
--- a/tests/test-cache-on-read.sh
+++ b/tests/test-cache-on-read.sh
@@ -30,15 +30,11 @@
# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
-# XXX Suggestion to improve this test: Use the delay filter below the
-# cache filter, and time reads to prove that the second read is faster
-# because it isn't going through the delay filter and plugin.
-
source ./functions.sh
set -e
set -x
-requires guestfish --version
+requires nbdsh --version
sock=`mktemp -u`
files="cache-on-read.img $sock cache-on-read.pid"
@@ -46,7 +42,7 @@ rm -f $files
cleanup_fn rm -f $files
# Create an empty base image.
-truncate -s 1G cache-on-read.img
+truncate -s 128K cache-on-read.img
# Run nbdkit with the caching filter and cache-on-read set.
start_nbdkit -P cache-on-read.pid -U $sock \
@@ -54,19 +50,18 @@ start_nbdkit -P cache-on-read.pid -U $sock \
file cache-on-read.img \
cache-on-read=true
-# Open the overlay and perform some operations.
-guestfish --format=raw -a "nbd://?socket=$sock" <<'EOF'
- run
- part-disk /dev/sda gpt
- mkfs ext4 /dev/sda1
- mount /dev/sda1 /
- fill-dir / 10000
- fill-pattern "abcde" 5M /large
- write /hello "hello, world"
-EOF
+nbdsh --connect "nbd+unix://?socket=$sock" \
+ -c '
+# Write some pattern data to the overlay and check it reads back OK.
+buf = b"abcd" * 16384
+h.pwrite (buf, 32768)
+zero = h.pread (32768, 0)
+assert zero == bytearray (32768)
+buf2 = h.pread (65536, 32768)
+assert buf == buf2
-# Check the last files we created exist.
-guestfish --ro -a cache-on-read.img -m /dev/sda1 <<'EOF'
- cat /hello
- cat /large | cat >/dev/null
-EOF
+# XXX Suggestion to improve this test: Use the delay filter below the
+# cache filter, and time reads to prove that the second read is faster
+# because it is not going through the delay filter and plugin.
+# XXX second h.pread here ...
+'
diff --git a/tests/test-cache.sh b/tests/test-cache.sh
index bafada2..eead7c8 100755
--- a/tests/test-cache.sh
+++ b/tests/test-cache.sh
@@ -34,7 +34,7 @@ source ./functions.sh
set -e
set -x
-requires guestfish --version
+requires nbdsh --version
sock=`mktemp -u`
files="cache.img $sock cache.pid"
@@ -42,24 +42,27 @@ rm -f $files
cleanup_fn rm -f $files
# Create an empty base image.
-truncate -s 1G cache.img
+truncate -s 128K cache.img
# Run nbdkit with the caching filter.
start_nbdkit -P cache.pid -U $sock --filter=cache file cache.img
-# Open the overlay and perform some operations.
-guestfish --format=raw -a "nbd://?socket=$sock" <<'EOF'
- run
- part-disk /dev/sda gpt
- mkfs ext4 /dev/sda1
- mount /dev/sda1 /
- fill-dir / 10000
- fill-pattern "abcde" 5M /large
- write /hello "hello, world"
-EOF
+nbdsh --connect "nbd+unix://?socket=$sock" \
+ -c '
+# Write some pattern data to the overlay and check it reads back OK.
+buf = b"abcd" * 16384
+h.pwrite (buf, 32768)
+zero = h.pread (32768, 0)
+assert zero == bytearray (32768)
+buf2 = h.pread (65536, 32768)
+assert buf == buf2
-# Check the last files we created exist.
-guestfish --ro -a cache.img -m /dev/sda1 <<'EOF'
- cat /hello
- cat /large | cat >/dev/null
-EOF
+# Flushing should write through to the underlying file.
+h.flush ()
+
+with open ("cache.img", "rb") as file:
+ zero = file.read (32768)
+ assert zero == bytearray (32768)
+ buf2 = file.read (65536)
+ assert buf == buf2
+'
diff --git a/tests/test-cow.sh b/tests/test-cow.sh
index 5567b77..dea518c 100755
--- a/tests/test-cow.sh
+++ b/tests/test-cow.sh
@@ -42,7 +42,11 @@ rm -f $files
cleanup_fn rm -f $files
# Create a base image which is partitioned with an empty filesystem.
-guestfish -N cow-base.img=fs exit
+rm -rf cow.d
+mkdir cow.d
+cleanup_fn rm -rf cow.d
+nbdkit -fv -U - linuxdisk cow.d size=100M \
+ --run 'qemu-img convert $nbd cow-base.img'
lastmod="$(stat -c "%y" cow-base.img)"
# Run nbdkit with a COW overlay.
@@ -50,8 +54,7 @@ start_nbdkit -P cow.pid -U $sock --filter=cow file cow-base.img
# Write some data into the overlay.
guestfish --format=raw -a "nbd://?socket=$sock" -m /dev/sda1 <<EOF
- fill-dir / 10000
- fill-pattern "abcde" 5M /large
+ fill-pattern "abcde" 128K /large
write /hello "hello, world"
EOF
--
2.23.0

377
0002-tests-Improve-test-readahead-test.patch Normal file
View File

@ -0,0 +1,377 @@
From ce8357a883167b75ced16ea9f1488f7933b3e813 Mon Sep 17 00:00:00 2001
From: "Richard W.M. Jones" <rjones@redhat.com>
Date: Sat, 14 Dec 2019 12:20:55 +0000
Subject: [PATCH 2/2] tests: Improve test-readahead test.
See
https://www.redhat.com/archives/libguestfs/2019-December/msg00088.html
---
tests/Makefile.am | 13 +--
tests/test-readahead-test-plugin.sh | 48 +++++++++
tests/test-readahead-test-request.py | 44 +++++++++
tests/test-readahead.c | 140 ---------------------------
tests/test-readahead.sh | 67 +++++++++++++
5 files changed, 166 insertions(+), 146 deletions(-)
create mode 100755 tests/test-readahead-test-plugin.sh
create mode 100755 tests/test-readahead-test-request.py
delete mode 100644 tests/test-readahead.c
create mode 100755 tests/test-readahead.sh
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 1932eea..a5e9897 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -168,7 +168,10 @@ EXTRA_DIST = \
test-rate.sh \
test-rate-dynamic.sh \
test.rb \
+ test-readahead.sh \
test-readahead-copy.sh \
+ test-readahead-test-plugin.sh \
+ test-readahead-test-request.py \
test-retry.sh \
test-retry-extents.sh \
test-retry-size.sh \
@@ -1050,12 +1053,10 @@ TESTS += \
$(NULL)
# readahead filter test.
-LIBGUESTFS_TESTS += test-readahead
-TESTS += test-readahead-copy.sh
-
-test_readahead_SOURCES = test-readahead.c test.h
-test_readahead_CFLAGS = $(WARNINGS_CFLAGS) $(LIBGUESTFS_CFLAGS)
-test_readahead_LDADD = libtest.la $(LIBGUESTFS_LIBS)
+TESTS += \
+ test-readahead.sh \
+ test-readahead-copy.sh \
+ $(NULL)
# retry filter test.
TESTS += \
diff --git a/tests/test-readahead-test-plugin.sh b/tests/test-readahead-test-plugin.sh
new file mode 100755
index 0000000..3c068cb
--- /dev/null
+++ b/tests/test-readahead-test-plugin.sh
@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+# nbdkit
+# Copyright (C) 2018-2019 Red Hat Inc.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# * Neither the name of Red Hat nor the names of its contributors may be
+# used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+
+# Test plugin used by test-readahead.sh.
+
+case "$1" in
+ get_size)
+ # test-readahead-test-request.py will make 10 requests of 512
+ # bytes each, so this just has to be >= 512*10.
+ echo 1M
+ ;;
+ pread)
+ sleep 5
+ dd if=/dev/zero count=$3 iflag=count_bytes
+ ;;
+ *)
+ exit 2
+ ;;
+esac
diff --git a/tests/test-readahead-test-request.py b/tests/test-readahead-test-request.py
new file mode 100755
index 0000000..fd615bd
--- /dev/null
+++ b/tests/test-readahead-test-request.py
@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+# -*- python -*-
+# nbdkit
+# Copyright (C) 2018-2019 Red Hat Inc.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# * Neither the name of Red Hat nor the names of its contributors may be
+# used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+
+# Used by test-readahead.sh to make a linear series of requests
+# with a constant, known size.
+
+import nbd
+import sys
+
+h = nbd.NBD ()
+h.connect_unix (sys.argv[1])
+
+for i in range (0, 512*10, 512):
+ h.pread (512, i)
diff --git a/tests/test-readahead.c b/tests/test-readahead.c
deleted file mode 100644
index 1772228..0000000
--- a/tests/test-readahead.c
+++ /dev/null
@@ -1,140 +0,0 @@
-/* nbdkit
- * Copyright (C) 2017-2019 Red Hat Inc.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- *
- * * Neither the name of Red Hat nor the names of its contributors may be
- * used to endorse or promote products derived from this software without
- * specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
- * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
- * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
- * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
- * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
- * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
- * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include <config.h>
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <stdint.h>
-#include <inttypes.h>
-#include <string.h>
-#include <unistd.h>
-
-#include <guestfs.h>
-
-#include "test.h"
-
-int
-main (int argc, char *argv[])
-{
- guestfs_h *g;
- int r;
- char *data;
- char **files;
- size_t i;
-
- /* The idea behind the test is that the readahead filter should not
- * interfere with normal, complex, non-sequential read and write
- * operations (although it will slow things down a lot).
- */
- if (test_start_nbdkit ("--filter", "readahead",
- "memory", "1G", NULL) == -1)
- exit (EXIT_FAILURE);
-
- g = guestfs_create ();
- if (g == NULL) {
- perror ("guestfs_create");
- exit (EXIT_FAILURE);
- }
-
- r = guestfs_add_drive_opts (g, "",
- GUESTFS_ADD_DRIVE_OPTS_FORMAT, "raw",
- GUESTFS_ADD_DRIVE_OPTS_PROTOCOL, "nbd",
- GUESTFS_ADD_DRIVE_OPTS_SERVER, server,
- -1);
- if (r == -1)
- exit (EXIT_FAILURE);
-
- if (guestfs_launch (g) == -1)
- exit (EXIT_FAILURE);
-
- if (guestfs_part_disk (g, "/dev/sda", "mbr") == -1)
- exit (EXIT_FAILURE);
- if (guestfs_mkfs (g, "ext2", "/dev/sda1") == -1)
- exit (EXIT_FAILURE);
-
- if (guestfs_mount_options (g, "discard", "/dev/sda1", "/") == -1)
- exit (EXIT_FAILURE);
-
-#define filename "/hello.txt"
-#define content "hello, people of the world"
-
- if (guestfs_write (g, filename, content, strlen (content)) == -1)
- exit (EXIT_FAILURE);
-
- if (guestfs_mkdir (g, "/fill") == -1)
- exit (EXIT_FAILURE);
-
- if (guestfs_fill_dir (g, "/fill", 10000) == -1)
- exit (EXIT_FAILURE);
-
- /* Force write through to and read back from disk. */
- if (guestfs_sync (g) == -1 || guestfs_drop_caches (g, 3) == -1)
- exit (EXIT_FAILURE);
-
- data = guestfs_cat (g, filename);
- if (!data)
- exit (EXIT_FAILURE);
-
- if (strcmp (data, content) != 0) {
- fprintf (stderr,
- "%s FAILED: unexpected content of %s file "
- "(actual: %s, expected: %s)\n",
- program_name, filename, data, content);
- exit (EXIT_FAILURE);
- }
-
- files = guestfs_find (g, "/");
- if (!files)
- exit (EXIT_FAILURE);
- for (i = 0; files[i] != NULL; ++i)
- free (files[i]);
- free (files);
-
-#ifdef GUESTFS_HAVE_FSTRIM
- /* Delete the files and fstrim to test zeroing/trimming. */
- if (guestfs_rm (g, filename) == -1)
- exit (EXIT_FAILURE);
-
- if (guestfs_rm_rf (g, "/fill") == -1)
- exit (EXIT_FAILURE);
-
- if (guestfs_fstrim (g, "/", -1) == -1)
- exit (EXIT_FAILURE);
-#endif
-
- if (guestfs_shutdown (g) == -1)
- exit (EXIT_FAILURE);
-
- guestfs_close (g);
- exit (EXIT_SUCCESS);
-}
diff --git a/tests/test-readahead.sh b/tests/test-readahead.sh
new file mode 100755
index 0000000..182aa00
--- /dev/null
+++ b/tests/test-readahead.sh
@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+# nbdkit
+# Copyright (C) 2018-2019 Red Hat Inc.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# * Neither the name of Red Hat nor the names of its contributors may be
+# used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+
+# Is the readahead filter faster? Copy a blank disk with a custom
+# plugin that sleeps on every request. Because the readahead filter
+# should result in fewer requests it should run faster.
+
+source ./functions.sh
+set -e
+set -x
+
+requires python3 --version
+requires python3 -c 'import nbd'
+
+files="readahead.img"
+rm -f $files
+cleanup_fn rm -f $files
+
+test ()
+{
+ start_t=$SECONDS
+ nbdkit -fv -U - "$@" sh ./test-readahead-test-plugin.sh \
+ --run './test-readahead-test-request.py $unixsocket'
+ end_t=$SECONDS
+ echo $((end_t - start_t))
+}
+
+t1=$(test --filter=readahead)
+t2=$(test)
+
+# In the t1 case we should make only 1 request into the plugin,
+# resulting in around 1 sleep period (5 seconds). In the t2 case we
+# make 10 requests so sleep for around 50 seconds. t1 should be < t2
+# is every reasonable scenario.
+if [ $t1 -ge $t2 ]; then
+ echo "$0: readahead filter took longer, should be shorter"
+ exit 1
+fi
--
2.23.0

11
nbdkit.spec
View File

@ -31,14 +31,14 @@ ExclusiveArch: x86_64
%global verify_tarball_signature 1
# If there are patches which touch autotools files, set this to 1.
%global patches_touch_autotools %{nil}
%global patches_touch_autotools 1
# The source directory.
%global source_directory 1.17-development
Name: nbdkit
Version: 1.17.4
Release: 1%{?dist}
Release: 2%{?dist}
Summary: NBD server
License: BSD
@ -51,6 +51,10 @@ Source1: http://libguestfs.org/download/nbdkit/%{source_directory}/%{name
Source2: libguestfs.keyring
%endif
# Upstream patches to improve test times.
Patch1: 0001-tests-Reduce-excessive-time-taken-by-some-tests.patch
Patch2: 0002-tests-Improve-test-readahead-test.patch
%if 0%{patches_touch_autotools}
BuildRequires: autoconf, automake, libtool
%endif
@ -915,6 +919,9 @@ make %{?_smp_mflags} check || {
%changelog
* Sat Dec 14 2019 Richard W.M. Jones <rjones@redhat.com> - 1.17.4-2
- Improve test times.
* Fri Dec 13 2019 Richard W.M. Jones <rjones@redhat.com> - 1.17.4-1
- New upstream development version 1.17.4.
- New filter: nofilter.