From fa93f5d5ded6a242a3f3355f9250bbf1c875d975 Mon Sep 17 00:00:00 2001 From: "Richard W.M. Jones" Date: Thu, 7 Mar 2019 06:39:35 +0000 Subject: [PATCH] New upstream version 1.11.7. Add nbdkit ssh plugin. Remove patches already upstream. --- ...-include-directory-to-get-nbdkit-plu.patch | 34 - ...filter-for-rate-limiting-connections.patch | 844 ------------------ nbdkit.spec | 32 +- sources | 4 +- 4 files changed, 24 insertions(+), 890 deletions(-) delete mode 100644 0001-common-utils-Use-include-directory-to-get-nbdkit-plu.patch delete mode 100644 0002-Add-new-filter-for-rate-limiting-connections.patch diff --git a/0001-common-utils-Use-include-directory-to-get-nbdkit-plu.patch b/0001-common-utils-Use-include-directory-to-get-nbdkit-plu.patch deleted file mode 100644 index a268b4b..0000000 --- a/0001-common-utils-Use-include-directory-to-get-nbdkit-plu.patch +++ /dev/null @@ -1,34 +0,0 @@ -From d77c93b1cc42fc6cbda1b2abf941b001ad741cba Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Sat, 2 Mar 2019 08:28:04 +0000 -Subject: [PATCH 1/2] common: utils: Use include/ directory to get - - -This worked before because nbdkit was installed by the system package -manager, but if not: - - utils.c:42:10: fatal error: nbdkit-plugin.h: No such file or directory - 42 | #include - | ^~~~~~~~~~~~~~~~~ - compilation terminated. - -Fixes commit 96b468d2e4214d15d722bd44db289d95e137280a. ---- - common/utils/Makefile.am | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/common/utils/Makefile.am b/common/utils/Makefile.am -index e002586..d1d95ed 100644 ---- a/common/utils/Makefile.am -+++ b/common/utils/Makefile.am -@@ -37,5 +37,7 @@ noinst_LTLIBRARIES = libutils.la - libutils_la_SOURCES = \ - utils.c \ - utils.h -+libutils_la_CPPFLAGS = \ -+ -I$(top_srcdir)/include - libutils_la_CFLAGS = \ - $(WARNINGS_CFLAGS) --- -2.20.1 - diff --git a/0002-Add-new-filter-for-rate-limiting-connections.patch b/0002-Add-new-filter-for-rate-limiting-connections.patch deleted file mode 100644 index 512685b..0000000 --- a/0002-Add-new-filter-for-rate-limiting-connections.patch +++ /dev/null @@ -1,844 +0,0 @@ -From 53d75b09d9873c69a2d0e7f514a24af746f14292 Mon Sep 17 00:00:00 2001 -From: "Richard W.M. Jones" -Date: Tue, 5 Mar 2019 06:24:29 +0000 -Subject: [PATCH 2/2] Add new filter for rate-limiting connections. - -As an example this creates a RAM disk with the rate filter on top, -then copies the data out to a local file: - - $ time ./nbdkit --filter=rate memory size=50M rate=10M \ - --run 'qemu-img convert $nbd /var/tmp/disk.img' - - real 0m40.023s - user 0m0.032s - sys 0m0.081s - -The size of the RAM disk is 8*50 = 400 Mbits, and we copy out at a -rate of 10 Mbps, so it takes exactly 40 seconds. ---- - filters/delay/nbdkit-delay-filter.pod | 4 +- - filters/rate/nbdkit-rate-filter.pod | 86 ++++++++++ - configure.ac | 2 + - filters/rate/bucket.h | 71 ++++++++ - filters/rate/bucket.c | 166 ++++++++++++++++++ - filters/rate/rate.c | 231 ++++++++++++++++++++++++++ - TODO | 9 + - filters/rate/Makefile.am | 64 +++++++ - tests/Makefile.am | 6 +- - tests/test-rate.sh | 60 +++++++ - 10 files changed, 697 insertions(+), 2 deletions(-) - create mode 100644 filters/rate/nbdkit-rate-filter.pod - create mode 100644 filters/rate/bucket.h - create mode 100644 filters/rate/bucket.c - create mode 100644 filters/rate/rate.c - create mode 100644 filters/rate/Makefile.am - create mode 100755 tests/test-rate.sh - -diff --git a/filters/delay/nbdkit-delay-filter.pod b/filters/delay/nbdkit-delay-filter.pod -index 7009a8c..c2eb172 100644 ---- a/filters/delay/nbdkit-delay-filter.pod -+++ b/filters/delay/nbdkit-delay-filter.pod -@@ -17,6 +17,7 @@ nbdkit-delay-filter - nbdkit delay filter - C is a filter that delays read and write requests - by some seconds or milliseconds. This is used to simulate a slow or - remote server, or to test certain kinds of race conditions in Linux. -+To limit server bandwidth use L instead. - - =head1 EXAMPLES - -@@ -74,7 +75,8 @@ milliseconds. - =head1 SEE ALSO - - L, --L. -+L, -+L. - - =head1 AUTHORS - -diff --git a/filters/rate/nbdkit-rate-filter.pod b/filters/rate/nbdkit-rate-filter.pod -new file mode 100644 -index 0000000..99a0373 ---- /dev/null -+++ b/filters/rate/nbdkit-rate-filter.pod -@@ -0,0 +1,86 @@ -+=head1 NAME -+ -+nbdkit-rate-filter - limit bandwidth by connection or server -+ -+=head1 SYNOPSIS -+ -+ nbdkit --filter=rate PLUGIN [PLUGIN-ARGS...] -+ [rate=BITSPERSEC] -+ [connection-rate=BITSPERSEC] -+ -+=head1 DESCRIPTION -+ -+C is a filter that limits the bandwidth that can -+be used by the server. Limits can be applied per connection and/or -+for the server as a whole. -+ -+=head1 EXAMPLES -+ -+=over 4 -+ -+=item nbdkit --filter=rate memory size=64M rate=1M -+ -+Create a 64M RAM disk and limit server bandwidth as a whole to a -+maximum of S<1 Mbps> (megabit per second). -+ -+=item nbdkit --filter=rate memory size=64M connection-rate=50K -+ -+Limit each connection to S<50 Kbps> (kilobits per second). However as -+there is no limit to the number of simultaneous connections this does -+not limit overall server bandwidth. -+ -+=item nbdkit --filter=rate memory size=64M connection-rate=50K rate=1M -+ -+Limit each connection to S<50 Kbps>. Additionally the total bandwidth -+across all connections to the server is limited to S<1 Mbps>. -+ -+=back -+ -+=head1 PARAMETERS -+ -+=over 4 -+ -+=item BBITSPERSEC -+ -+Limit each connection to C. -+ -+=item BBITSPERSEC -+ -+Limit total bandwidth across all connections to C. -+ -+=back -+ -+C can be specified as a simple number, or you can use a -+number followed by C, C etc to mean kilobits, megabits and so -+on. -+ -+=head1 NOTES -+ -+You can specify C and C on their own or -+together. If you specify neither, the filter is turned off. -+ -+The rate filter approximates the bandwidth used by the NBD protocol on -+the wire. Some operations such as zeroing and trimming are -+effectively free (because only a tiny NBD message is sent over the -+network) and so do not count against the bandwidth limit. NBD and TCP -+protocol overhead is not included, so you may find that other tools -+such as L and L give more accurate results. -+ -+here are separate bandwidth limits for read and write (ie. download -+and upload to the server). -+ -+=head1 SEE ALSO -+ -+L, -+L, -+L, -+L, -+L. -+ -+=head1 AUTHORS -+ -+Richard W.M. Jones -+ -+=head1 COPYRIGHT -+ -+Copyright (C) 2019 Red Hat Inc. -diff --git a/configure.ac b/configure.ac -index 9e7e5ca..467d48f 100644 ---- a/configure.ac -+++ b/configure.ac -@@ -819,6 +819,7 @@ filters="\ - nozero \ - offset \ - partition \ -+ rate \ - truncate \ - xz \ - " -@@ -889,6 +890,7 @@ AC_CONFIG_FILES([Makefile - filters/nozero/Makefile - filters/offset/Makefile - filters/partition/Makefile -+ filters/rate/Makefile - filters/truncate/Makefile - filters/xz/Makefile - fuzzing/Makefile -diff --git a/filters/rate/bucket.h b/filters/rate/bucket.h -new file mode 100644 -index 0000000..bc36fc1 ---- /dev/null -+++ b/filters/rate/bucket.h -@@ -0,0 +1,71 @@ -+/* nbdkit -+ * Copyright (C) 2018-2019 Red Hat Inc. -+ * All rights reserved. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are -+ * met: -+ * -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * -+ * * Neither the name of Red Hat nor the names of its contributors may be -+ * used to endorse or promote products derived from this software without -+ * specific prior written permission. -+ * -+ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND -+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR -+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF -+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT -+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -+ * SUCH DAMAGE. -+ */ -+ -+#ifndef NBDKIT_BUCKET_H -+#define NBDKIT_BUCKET_H -+ -+#include -+#include -+#include -+ -+/* A token bucket. */ -+struct bucket { -+ uint64_t rate; /* Fill rate. 0 = no limit set. */ -+ uint64_t capacity; /* Maximum capacity of the bucket. */ -+ uint64_t level; /* How full is the bucket now? */ -+ struct timeval tv; /* Last time we updated the level. */ -+}; -+ -+/* Initialize the bucket structure. Capacity is expressed in -+ * rate-equivalent seconds. -+ */ -+extern void bucket_init (struct bucket *bucket, -+ uint64_t rate, double capacity); -+ -+/* Take up to N tokens from the bucket. Returns the number -+ * of tokens remaining (that could not be taken from the bucket), -+ * or 0 if we were able to take all N tokens from the bucket. -+ * -+ * In the case that the return value > 0, *TS is initialized with the -+ * estimated length of time you should sleep. Note that *TS is _NOT_ -+ * initialized if the return value == 0, because the caller should not -+ * sleep in that case. -+ * -+ * In the case where the caller needs to sleep, it must make a further -+ * call to bucket_run before proceeding, since another thread may have -+ * "stolen" the tokens while you were sleeping. -+ */ -+extern uint64_t bucket_run (struct bucket *bucket, uint64_t n, -+ struct timespec *ts); -+ -+#endif /* NBDKIT_BUCKET_H */ -diff --git a/filters/rate/bucket.c b/filters/rate/bucket.c -new file mode 100644 -index 0000000..b68d7b3 ---- /dev/null -+++ b/filters/rate/bucket.c -@@ -0,0 +1,166 @@ -+/* nbdkit -+ * Copyright (C) 2018-2019 Red Hat Inc. -+ * All rights reserved. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are -+ * met: -+ * -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * -+ * * Neither the name of Red Hat nor the names of its contributors may be -+ * used to endorse or promote products derived from this software without -+ * specific prior written permission. -+ * -+ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND -+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR -+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF -+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT -+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -+ * SUCH DAMAGE. -+ */ -+ -+/* This filter is implemented using a Token Bucket -+ * (https://en.wikipedia.org/wiki/Token_bucket). There are two -+ * buckets per connection (one each for reading and writing) and two -+ * global buckets (also for reading and writing). -+ * -+ * We add tokens at the desired rate (the per-connection rate for the -+ * connection buckets, and the global rate for the global buckets). -+ * Note that we don't actually keep the buckets updated in real time -+ * because as a filter we are called asynchronously. Instead for each -+ * bucket we store the last time we were called and add the -+ * appropriate number of tokens when we are called next. -+ * -+ * The bucket capacity controls the burstiness allowed. This is -+ * hard-coded at the moment but could be configurable. All buckets -+ * start off full. -+ * -+ * When a packet is to be read or written, if there are sufficient -+ * tokens in the bucket then the packet may be immediately passed -+ * through to the underlying plugin. The number of bits used is -+ * deducted from the appropriate per-connection and global bucket. -+ * -+ * If there are insufficient tokens then the packet must be delayed. -+ * This is done by inserting a sleep which has an estimated length -+ * that is long enough based on the rate at which enough tokens will -+ * replenish the bucket to allow the packet to be sent next time. -+ */ -+ -+#include -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include -+ -+#include "minmax.h" -+ -+#include "bucket.h" -+ -+int rate_debug_bucket; /* -D rate.bucket=1 */ -+ -+void -+bucket_init (struct bucket *bucket, uint64_t rate, double capacity) -+{ -+ bucket->rate = rate; -+ -+ /* Capacity is expressed in seconds, but we want to know the -+ * capacity in tokens, so multiply by the rate to get this. -+ */ -+ bucket->capacity = rate * capacity; -+ -+ /* Buckets start off full. */ -+ bucket->level = capacity; -+ -+ gettimeofday (&bucket->tv, NULL); -+} -+ -+/* Return the number of microseconds in y - x. */ -+static int64_t -+tvdiff (const struct timeval *x, const struct timeval *y) -+{ -+ int64_t usec; -+ -+ usec = (y->tv_sec - x->tv_sec) * 1000000; -+ usec += y->tv_usec - x->tv_usec; -+ return usec; -+} -+ -+uint64_t -+bucket_run (struct bucket *bucket, uint64_t n, struct timespec *ts) -+{ -+ struct timeval now; -+ int64_t usec; -+ uint64_t add, nsec; -+ -+ /* rate == 0 is a special case meaning that there is no limit being -+ * enforced. -+ */ -+ if (bucket->rate == 0) -+ return 0; -+ -+ gettimeofday (&now, NULL); -+ -+ /* Work out how much time has elapsed since we last added tokens to -+ * the bucket, and add the correct number of tokens. -+ */ -+ usec = tvdiff (&bucket->tv, &now); -+ if (usec < 0) /* Maybe happens if system time not monotonic? */ -+ usec = 0; -+ -+ add = bucket->rate * usec / 1000000; -+ add = MIN (add, bucket->capacity - bucket->level); -+ if (rate_debug_bucket) -+ nbdkit_debug ("bucket %p: adding %" PRIu64 " tokens, new level %" PRIu64, -+ bucket, add, bucket->level + add); -+ bucket->level += add; -+ bucket->tv = now; -+ -+ /* Can we deduct N tokens from the bucket? If yes then we're good, -+ * and we can return 0 which means the caller won't sleep. -+ */ -+ if (bucket->level >= n) { -+ if (rate_debug_bucket) -+ nbdkit_debug ("bucket %p: deducting %" PRIu64 " tokens", bucket, n); -+ bucket->level -= n; -+ return 0; -+ } -+ -+ if (rate_debug_bucket) -+ nbdkit_debug ("bucket %p: deducting %" PRIu64 " tokens, bucket empty, " -+ "need another %" PRIu64 " tokens", -+ bucket, bucket->level, n - bucket->level); -+ -+ n -= bucket->level; -+ bucket->level = 0; -+ -+ /* Now we need to estimate how long it will take to add N tokens to -+ * the bucket, which is how long the caller must sleep for. -+ */ -+ nsec = 1000000000 * n / bucket->rate; -+ ts->tv_sec = nsec / 1000000000; -+ ts->tv_nsec = nsec % 1000000000; -+ -+ if (rate_debug_bucket) -+ nbdkit_debug ("bucket %p: sleeping for %.1f seconds", bucket, -+ nsec / 1000000000.); -+ -+ return n; -+} -diff --git a/filters/rate/rate.c b/filters/rate/rate.c -new file mode 100644 -index 0000000..a0c6ae3 ---- /dev/null -+++ b/filters/rate/rate.c -@@ -0,0 +1,231 @@ -+/* nbdkit -+ * Copyright (C) 2018-2019 Red Hat Inc. -+ * All rights reserved. -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are -+ * met: -+ * -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * -+ * * Neither the name of Red Hat nor the names of its contributors may be -+ * used to endorse or promote products derived from this software without -+ * specific prior written permission. -+ * -+ * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND -+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR -+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF -+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT -+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -+ * SUCH DAMAGE. -+ */ -+ -+/* For a note on the implementation of this filter, see bucket.c. */ -+ -+#include -+ -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include -+ -+#include -+ -+#include "bucket.h" -+ -+#define THREAD_MODEL NBDKIT_THREAD_MODEL_PARALLEL -+ -+/* Per-connection and global limit, both in bits per second, with zero -+ * meaning not set / not enforced. -+ */ -+static uint64_t connection_rate = 0; -+static uint64_t rate = 0; -+ -+/* Bucket capacity controls the burst rate. It is expressed as the -+ * length of time in "rate-equivalent seconds" that the client can -+ * burst for after a period of inactivity. This could be adjustable -+ * in future. -+ */ -+#define BUCKET_CAPACITY 2.0 -+ -+/* Global read and write buckets. */ -+static struct bucket read_bucket; -+static pthread_mutex_t read_bucket_lock = PTHREAD_MUTEX_INITIALIZER; -+static struct bucket write_bucket; -+static pthread_mutex_t write_bucket_lock = PTHREAD_MUTEX_INITIALIZER; -+ -+/* Per-connection handle. */ -+struct rate_handle { -+ /* Per-connection read and write buckets. */ -+ struct bucket read_bucket; -+ pthread_mutex_t read_bucket_lock; -+ struct bucket write_bucket; -+ pthread_mutex_t write_bucket_lock; -+}; -+ -+/* Called for each key=value passed on the command line. */ -+static int -+rate_config (nbdkit_next_config *next, void *nxdata, -+ const char *key, const char *value) -+{ -+ if (strcmp (key, "rate") == 0) { -+ if (rate > 0) { -+ nbdkit_error ("rate set twice on the command line"); -+ return -1; -+ } -+ rate = nbdkit_parse_size (value); -+ if (rate == -1) -+ return -1; -+ if (rate == 0) { -+ nbdkit_error ("rate cannot be set to 0"); -+ return -1; -+ } -+ return 0; -+ } -+ else if (strcmp (key, "connection-rate") == 0) { -+ if (connection_rate > 0) { -+ nbdkit_error ("connection-rate set twice on the command line"); -+ return -1; -+ } -+ connection_rate = nbdkit_parse_size (value); -+ if (connection_rate == -1) -+ return -1; -+ if (connection_rate == 0) { -+ nbdkit_error ("connection-rate cannot be set to 0"); -+ return -1; -+ } -+ return 0; -+ } -+ else -+ return next (nxdata, key, value); -+} -+ -+static int -+rate_config_complete (nbdkit_next_config_complete *next, void *nxdata) -+{ -+ /* Initialize the global buckets. */ -+ bucket_init (&read_bucket, rate, BUCKET_CAPACITY); -+ bucket_init (&write_bucket, rate, BUCKET_CAPACITY); -+ -+ return next (nxdata); -+} -+ -+#define rate_config_help \ -+ "rate=BITSPERSEC Limit total bandwidth.\n" \ -+ "connection-rate=BITSPERSEC Limit per-connection bandwidth." -+ -+/* Create the per-connection handle. */ -+static void * -+rate_open (nbdkit_next_open *next, void *nxdata, int readonly) -+{ -+ struct rate_handle *h; -+ -+ if (next (nxdata, readonly) == -1) -+ return NULL; -+ -+ h = malloc (sizeof *h); -+ if (h == NULL) { -+ nbdkit_error ("malloc: %m"); -+ return NULL; -+ } -+ -+ bucket_init (&h->read_bucket, connection_rate, BUCKET_CAPACITY); -+ bucket_init (&h->write_bucket, connection_rate, BUCKET_CAPACITY); -+ pthread_mutex_init (&h->read_bucket_lock, NULL); -+ pthread_mutex_init (&h->write_bucket_lock, NULL); -+ -+ return h; -+} -+ -+/* Free up the per-connection handle. */ -+static void -+rate_close (void *handle) -+{ -+ struct rate_handle *h = handle; -+ -+ pthread_mutex_destroy (&h->read_bucket_lock); -+ pthread_mutex_destroy (&h->write_bucket_lock); -+ free (h); -+} -+ -+static inline void -+maybe_sleep (struct bucket *bucket, pthread_mutex_t *lock, uint32_t count) -+{ -+ struct timespec ts; -+ uint64_t bits; -+ -+ /* Count is in bytes, but we rate limit using bits. We could -+ * multiply this by 10 to include start/stop but let's not -+ * second-guess the transport layers underneath. -+ */ -+ bits = count * UINT64_C(8); -+ -+ while (bits > 0) { -+ /* Run the token bucket algorithm. */ -+ pthread_mutex_lock (lock); -+ bits = bucket_run (bucket, bits, &ts); -+ pthread_mutex_unlock (lock); -+ -+ if (bits > 0) -+ nanosleep (&ts, NULL); -+ } -+} -+ -+/* Read data. */ -+static int -+rate_pread (struct nbdkit_next_ops *next_ops, void *nxdata, -+ void *handle, void *buf, uint32_t count, uint64_t offset, -+ uint32_t flags, int *err) -+{ -+ struct rate_handle *h = handle; -+ -+ maybe_sleep (&read_bucket, &read_bucket_lock, count); -+ maybe_sleep (&h->read_bucket, &h->read_bucket_lock, count); -+ -+ return next_ops->pread (nxdata, buf, count, offset, flags, err); -+} -+ -+/* Write data. */ -+static int -+rate_pwrite (struct nbdkit_next_ops *next_ops, void *nxdata, -+ void *handle, -+ const void *buf, uint32_t count, uint64_t offset, uint32_t flags, -+ int *err) -+{ -+ struct rate_handle *h = handle; -+ -+ maybe_sleep (&write_bucket, &write_bucket_lock, count); -+ maybe_sleep (&h->write_bucket, &h->write_bucket_lock, count); -+ -+ return next_ops->pwrite (nxdata, buf, count, offset, flags, err); -+} -+ -+static struct nbdkit_filter filter = { -+ .name = "rate", -+ .longname = "nbdkit rate filter", -+ .version = PACKAGE_VERSION, -+ .config = rate_config, -+ .config_complete = rate_config_complete, -+ .config_help = rate_config_help, -+ .open = rate_open, -+ .close = rate_close, -+ .pread = rate_pread, -+ .pwrite = rate_pwrite, -+}; -+ -+NBDKIT_REGISTER_FILTER(filter) -diff --git a/TODO b/TODO -index 59590a1..b589127 100644 ---- a/TODO -+++ b/TODO -@@ -129,6 +129,15 @@ Suggestions for filters - * nbdkit-cache-filter should handle ENOSPC errors automatically by - reclaiming blocks from the cache - -+nbdkit-rate-filter: -+ -+* allow other kinds of traffic shaping such as VBR -+ -+* limit traffic per client (ie. per IP address) -+ -+* split large requests to avoid long, lumpy sleeps when request size -+ is much larger than rate limit -+ - Filters for security - -------------------- - -diff --git a/filters/rate/Makefile.am b/filters/rate/Makefile.am -new file mode 100644 -index 0000000..c39aa01 ---- /dev/null -+++ b/filters/rate/Makefile.am -@@ -0,0 +1,64 @@ -+# nbdkit -+# Copyright (C) 2018-2019 Red Hat Inc. -+# All rights reserved. -+# -+# Redistribution and use in source and binary forms, with or without -+# modification, are permitted provided that the following conditions are -+# met: -+# -+# * Redistributions of source code must retain the above copyright -+# notice, this list of conditions and the following disclaimer. -+# -+# * Redistributions in binary form must reproduce the above copyright -+# notice, this list of conditions and the following disclaimer in the -+# documentation and/or other materials provided with the distribution. -+# -+# * Neither the name of Red Hat nor the names of its contributors may be -+# used to endorse or promote products derived from this software without -+# specific prior written permission. -+# -+# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND -+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR -+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF -+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT -+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -+# SUCH DAMAGE. -+ -+include $(top_srcdir)/common-rules.mk -+ -+EXTRA_DIST = nbdkit-rate-filter.pod -+ -+filter_LTLIBRARIES = nbdkit-rate-filter.la -+ -+nbdkit_rate_filter_la_SOURCES = \ -+ bucket.c \ -+ bucket.h \ -+ rate.c \ -+ $(top_srcdir)/include/nbdkit-filter.h -+ -+nbdkit_rate_filter_la_CPPFLAGS = \ -+ -I$(top_srcdir)/include \ -+ -I$(top_srcdir)/common/include -+nbdkit_rate_filter_la_CFLAGS = \ -+ $(WARNINGS_CFLAGS) -+nbdkit_rate_filter_la_LDFLAGS = \ -+ -module -avoid-version -shared \ -+ -Wl,--version-script=$(top_srcdir)/filters/filters.syms -+ -+if HAVE_POD -+ -+man_MANS = nbdkit-rate-filter.1 -+CLEANFILES += $(man_MANS) -+ -+nbdkit-rate-filter.1: nbdkit-rate-filter.pod -+ $(PODWRAPPER) --section=1 --man $@ \ -+ --html $(top_builddir)/html/$@.html \ -+ $< -+ -+endif HAVE_POD -diff --git a/tests/Makefile.am b/tests/Makefile.am -index 3992d9b..d1e6f0e 100644 ---- a/tests/Makefile.am -+++ b/tests/Makefile.am -@@ -1,5 +1,5 @@ - # nbdkit --# Copyright (C) 2013-2018 Red Hat Inc. -+# Copyright (C) 2013-2019 Red Hat Inc. - # All rights reserved. - # - # Redistribution and use in source and binary forms, with or without -@@ -97,6 +97,7 @@ EXTRA_DIST = \ - test-python-exception.sh \ - test.pl \ - test.py \ -+ test-rate.sh \ - test.rb \ - test.tcl \ - test-shebang-perl.sh \ -@@ -810,6 +811,9 @@ if HAVE_GUESTFISH - TESTS += test-partition2.sh - endif HAVE_GUESTFISH - -+# rate filter test. -+TESTS += test-rate.sh -+ - # truncate filter tests. - TESTS += \ - test-truncate1.sh \ -diff --git a/tests/test-rate.sh b/tests/test-rate.sh -new file mode 100755 -index 0000000..010ef19 ---- /dev/null -+++ b/tests/test-rate.sh -@@ -0,0 +1,60 @@ -+#!/usr/bin/env bash -+# nbdkit -+# Copyright (C) 2018-2019 Red Hat Inc. -+# All rights reserved. -+# -+# Redistribution and use in source and binary forms, with or without -+# modification, are permitted provided that the following conditions are -+# met: -+# -+# * Redistributions of source code must retain the above copyright -+# notice, this list of conditions and the following disclaimer. -+# -+# * Redistributions in binary form must reproduce the above copyright -+# notice, this list of conditions and the following disclaimer in the -+# documentation and/or other materials provided with the distribution. -+# -+# * Neither the name of Red Hat nor the names of its contributors may be -+# used to endorse or promote products derived from this software without -+# specific prior written permission. -+# -+# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND -+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, -+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -+# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR -+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF -+# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT -+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -+# SUCH DAMAGE. -+ -+source ./functions.sh -+set -e -+set -x -+ -+requires qemu-img --version -+ -+files="rate.img rate.time rate.err" -+rm -f $files -+cleanup_fn rm -f $files -+ -+# This should take no less than 20 seconds to run: -+# (8 * 25 * 1024 * 1024) / (10 * 1024 * 1024) = 20 -+ -+# We are using the bash time builtin, so setting TIMEFORMAT will -+# control the output format of the time builtin. For strange use of -+# { ; } here, see: https://stackoverflow.com/a/13356654 -+set +x -+{ TIMEFORMAT="%0R" ; time nbdkit --filter=rate memory size=25M rate=10M --run 'qemu-img convert -p $nbd rate.img' 2>rate.err ; } 2>rate.time -+set -x -+ -+cat rate.err ||: -+ -+seconds="$( cat rate.time )" -+if [ "$seconds" -lt 20 ]; then -+ echo "$0: rate filter failed: command took $seconds seconds, expected > 20" -+ exit 1 -+fi --- -2.20.1 - diff --git a/nbdkit.spec b/nbdkit.spec index ff4d5fa..8e9652c 100644 --- a/nbdkit.spec +++ b/nbdkit.spec @@ -25,14 +25,14 @@ %global verify_tarball_signature 1 # If there are patches which touch autotools files, set this to 1. -%global patches_touch_autotools 1 +%global patches_touch_autotools %{nil} # The source directory. %global source_directory 1.11-development Name: nbdkit -Version: 1.11.6 -Release: 2%{?dist} +Version: 1.11.7 +Release: 1%{?dist} Summary: NBD server License: BSD @@ -45,11 +45,6 @@ Source1: http://libguestfs.org/download/nbdkit/%{source_directory}/%{name Source2: libguestfs.keyring %endif -# Upstream fix to include directory. -Patch1: 0001-common-utils-Use-include-directory-to-get-nbdkit-plu.patch -# Posted but not upstream patch to add nbdkit-rate-filter. -Patch2: 0002-Add-new-filter-for-rate-limiting-connections.patch - %if 0%{patches_touch_autotools} BuildRequires: autoconf, automake, libtool %endif @@ -77,6 +72,7 @@ BuildRequires: libvirt-devel BuildRequires: xz-devel BuildRequires: zlib-devel BuildRequires: libcurl-devel +BuildRequires: libssh-devel BuildRequires: e2fsprogs, e2fsprogs-devel BuildRequires: genisoimage BuildRequires: bash-completion @@ -209,7 +205,7 @@ This package contains example plugins for %{name}. # packaged separately. %package curl-plugin -Summary: HTTP/FTP/SSH (cURL) plugin for %{name} +Summary: HTTP/FTP (cURL) plugin for %{name} License: BSD Requires: %{name}-server%{?_isa} = %{version}-%{release} @@ -220,7 +216,7 @@ Obsoletes: %{name}-plugin-curl <= %{version}-%{release} %description curl-plugin -This package contains cURL (HTTP/FTP/SSH) support for %{name}. +This package contains cURL (HTTP/FTP) support for %{name}. %package ext2-plugin @@ -449,6 +445,17 @@ Obsoletes: %{name}-plugin-ruby <= %{version}-%{release} This package lets you write Ruby plugins for %{name}. +%package ssh-plugin +Summary: SSH plugin for %{name} +License: BSD + +Requires: %{name}-server%{?_isa} = %{version}-%{release} + + +%description ssh-plugin +This package contains SSH support for %{name}. + + %package tar-plugin Summary: Tar archive plugin for %{name} License: BSD @@ -992,6 +999,11 @@ popd %changelog +* Thu Mar 07 2019 Richard W.M. Jones - 1.11.7-1 +- New upstream version 1.11.7. +- Add nbdkit ssh plugin. +- Remove patches already upstream. + * Tue Mar 05 2019 Richard W.M. Jones - 1.11.6-2 - Add nbdkit rate filter. diff --git a/sources b/sources index 914e80c..cddc0c3 100644 --- a/sources +++ b/sources @@ -1,2 +1,2 @@ -SHA512 (nbdkit-1.11.6.tar.gz) = 2262533796a13f3097f2e892ef06122505810f8f7fb7b19d6e9b0aed0b612228181ea31048f6f818195b7cded4669948ca3ef74fca3e245661eee4cccfeb5276 -SHA512 (nbdkit-1.11.6.tar.gz.sig) = ec33f8cc713144309a9f0a517a278122f58daac7c43051c4d6f8fc73eac2f8aa12cc29f52e5886650e5c51c603bde0a8d3eab241d986845921144568f550c65f +SHA512 (nbdkit-1.11.7.tar.gz) = c86161122d4f1060eab0b2aba319e29944649b453c44d9a1f7d660cc0d01b953f3808bfe41fcbf8c932c0a7d73011a692445a18cd48e6f4f7bf83a66bd1c6abd +SHA512 (nbdkit-1.11.7.tar.gz.sig) = 89a0f77db0964219883802dc4060d6e645d4c826e35bfd1a3ba9412c291d7802c194b700105e2db35d876657099bb5d26990ad78e6b67f16ddfed3464e931d7b