From 17c24027cea812d4a3517c48bdf8f79bbb0045c2 Mon Sep 17 00:00:00 2001 From: "Richard W.M. Jones" Date: Sat, 26 Jun 2021 16:10:40 +0100 Subject: [PATCH] info: Add --map --totals sub-mode to display summary of map This is similar to "qemu-img measure". Some examples: $ nbdkit -r file fedora-33.img --run 'nbdinfo --map --totals $uri' 1226113024 19.0 0 data 5216337920 81.0 3 hole,zero $ nbdkit -r file fedora-33.img --run 'nbdinfo --map --totals --json $uri | jq' [ { "size": 1226113024, "percent": 19.0318, "type": 0, "description": "data" }, { "size": 5216337920, "percent": 80.9682, "type": 3, "description": "hole,zero" } ] $ nbdkit sparse-random 6G --run 'nbdinfo --map --totals $uri' 941551616 14.6 0 data 5500899328 85.4 3 hole,zero (cherry picked from commit 7968bfe328e86574276283b159a594eeebeaf32a) --- info/Makefile.am | 2 + info/info-map-totals-json.sh | 48 +++++++++++++++++++++ info/info-map-totals.sh | 43 +++++++++++++++++++ info/main.c | 15 ++++++- info/map.c | 82 +++++++++++++++++++++++++++++++++++- info/nbdinfo.h | 1 + info/nbdinfo.pod | 38 ++++++++++++++++- 7 files changed, 225 insertions(+), 4 deletions(-) create mode 100755 info/info-map-totals-json.sh create mode 100755 info/info-map-totals.sh diff --git a/info/Makefile.am b/info/Makefile.am index 5c717c7..75c6a75 100644 --- a/info/Makefile.am +++ b/info/Makefile.am @@ -38,6 +38,8 @@ info_sh_files = \ info-map-base-allocation-zero.sh \ info-map-qemu-dirty-bitmap.sh \ info-map-qemu-allocation-depth.sh \ + info-map-totals.sh \ + info-map-totals-json.sh \ info-atomic-output.sh \ $(NULL) diff --git a/info/info-map-totals-json.sh b/info/info-map-totals-json.sh new file mode 100755 index 0000000..dc386ef --- /dev/null +++ b/info/info-map-totals-json.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# nbd client library in userspace +# Copyright (C) 2020-2021 Red Hat Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +. ../tests/functions.sh + +set -e +set -x + +requires nbdkit --version +requires nbdkit -U - null --run 'test "$uri" != ""' +requires jq --version + +out=info-map-totals-json.out +cleanup_fn rm -f $out +rm -f $out + +# The sparse allocator used by nbdkit-data-plugin uses a 32K page +# size, and extents are always aligned with this. +nbdkit -U - data data='1 @131072 2' size=1M \ + --run '$VG nbdinfo --map --totals --json "$uri"' > $out + +cat $out +jq . < $out + +test $( jq -r '.[0].size' < $out ) -eq 65536 +test $( jq -r '.[0].percent' < $out ) = "6.25" +test $( jq -r '.[0].type' < $out ) -eq 0 +test $( jq -r '.[0].description' < $out ) = "data" + +test $( jq -r '.[1].size' < $out ) -eq 983040 +test $( jq -r '.[1].percent' < $out ) = "93.75" +test $( jq -r '.[1].type' < $out ) -eq 3 +test $( jq -r '.[1].description' < $out ) = "hole,zero" diff --git a/info/info-map-totals.sh b/info/info-map-totals.sh new file mode 100755 index 0000000..12c1263 --- /dev/null +++ b/info/info-map-totals.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# nbd client library in userspace +# Copyright (C) 2020-2021 Red Hat Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +. ../tests/functions.sh + +set -e +set -x + +requires nbdkit --version +requires nbdkit -U - null --run 'test "$uri" != ""' +requires tr --version + +out=info-map-totals.out +cleanup_fn rm -f $out +rm -f $out + +# The sparse allocator used by nbdkit-data-plugin uses a 32K page +# size, and extents are always aligned with this. +nbdkit -U - data data='1 @131072 2' size=1M \ + --run '$VG nbdinfo --map --totals "$uri"' > $out + +cat $out + +if [ "$(tr -s ' ' < $out)" != " 65536 6.2 0 data + 983040 93.8 3 hole,zero" ]; then + echo "$0: unexpected output from nbdinfo --map" + exit 1 +fi diff --git a/info/main.c b/info/main.c index 8be52b7..fcda25c 100644 --- a/info/main.c +++ b/info/main.c @@ -42,6 +42,7 @@ bool probe_content = false; /* --content / --no-content option */ bool json_output = false; /* --json option */ const char *map = NULL; /* --map option */ bool size_only = false; /* --size option */ +bool totals = false; /* --totals option */ static void __attribute__((noreturn)) usage (FILE *fp, int exitcode) @@ -52,7 +53,7 @@ usage (FILE *fp, int exitcode) "\n" " nbdinfo [--json] NBD-URI\n" " nbdinfo --size [--json] NBD-URI\n" -" nbdinfo --map [--json] NBD-URI\n" +" nbdinfo --map [--totals] [--json] NBD-URI\n" " nbdinfo -L|--list [--json] NBD-URI\n" "\n" "Other options:\n" @@ -87,6 +88,7 @@ main (int argc, char *argv[]) JSON_OPTION, MAP_OPTION, SIZE_OPTION, + TOTALS_OPTION, }; const char *short_options = "LV"; const struct option long_options[] = { @@ -99,6 +101,8 @@ main (int argc, char *argv[]) { "map", optional_argument, NULL, MAP_OPTION }, { "short-options", no_argument, NULL, SHORT_OPTIONS }, { "size", no_argument, NULL, SIZE_OPTION }, + { "total", no_argument, NULL, TOTALS_OPTION }, + { "totals", no_argument, NULL, TOTALS_OPTION }, { "version", no_argument, NULL, 'V' }, { NULL } }; @@ -155,6 +159,10 @@ main (int argc, char *argv[]) size_only = true; break; + case TOTALS_OPTION: + totals = true; + break; + case 'L': list_all = true; break; @@ -184,6 +192,11 @@ main (int argc, char *argv[]) progname, "--content", "--no-content"); exit (EXIT_FAILURE); } + if (totals && !map) { + fprintf (stderr, "%s: you must use --totals only with --map option.\n", + progname); + exit (EXIT_FAILURE); + } /* Work out if we should probe content. */ probe_content = !list_all; diff --git a/info/map.c b/info/map.c index 82c9507..628033c 100644 --- a/info/map.c +++ b/info/map.c @@ -38,6 +38,7 @@ DEFINE_VECTOR_TYPE (uint32_vector, uint32_t) static void print_extents (uint32_vector *entries); +static void print_totals (uint32_vector *entries, int64_t size); static int extent_callback (void *user_data, const char *metacontext, uint64_t offset, uint32_t *entries, size_t nr_entries, @@ -89,7 +90,10 @@ do_map (void) offset += entries.ptr[i]; } - print_extents (&entries); + if (!totals) + print_extents (&entries); + else + print_totals (&entries, size); free (entries.ptr); } @@ -195,6 +199,82 @@ print_one_extent (uint64_t offset, uint64_t len, uint32_t type) free (descr); } +/* --map --totals suboption */ +static void +print_totals (uint32_vector *entries, int64_t size) +{ + uint32_t type; + bool comma = false; + + /* This is necessary to avoid a divide by zero below, but if the + * size of the export is zero then we know we will not print any + * information below so return quickly. + */ + if (size == 0) { + if (json_output) fprintf (fp, "[]\n"); + return; + } + + if (json_output) fprintf (fp, "[\n"); + + /* In the outer loop assume we have already printed all entries with + * entry type < type. Count all instances of type and at the same + * time find the next type that exists > type. + */ + type = 0; + for (;;) { + uint64_t next_type = (uint64_t)UINT32_MAX + 1; + uint64_t c = 0; + size_t i; + + for (i = 0; i < entries->size; i += 2) { + uint32_t t = entries->ptr[i+1]; + + if (t == type) + c += entries->ptr[i]; + else if (type < t && t < next_type) + next_type = t; + } + + if (c > 0) { + char *descr = extent_description (map, type); + double percent = 100.0 * c / size; + + if (!json_output) { + fprintf (fp, "%10" PRIu64 " %5.1f %3" PRIu32, + c, percent, type); + if (descr) + fprintf (fp, " %s", descr); + fprintf (fp, "\n"); + } + else { + if (comma) + fprintf (fp, ",\n"); + + fprintf (fp, + "{ \"size\": %" PRIu64 ", " + "\"percent\": %g, " + "\"type\": %" PRIu32, + c, percent, type); + if (descr) { + fprintf (fp, ", \"description\": "); + print_json_string (descr); + } + fprintf (fp, " }"); + comma = true; + } + + free (descr); + } + + if (next_type == (uint64_t)UINT32_MAX + 1) + break; + type = next_type; + } + + if (json_output) fprintf (fp, "\n]\n"); +} + static char * extent_description (const char *metacontext, uint32_t type) { diff --git a/info/nbdinfo.h b/info/nbdinfo.h index ff13e37..6ff73af 100644 --- a/info/nbdinfo.h +++ b/info/nbdinfo.h @@ -32,6 +32,7 @@ extern bool probe_content; extern bool json_output; extern const char *map; extern bool size_only; +extern bool totals; /* list.c */ extern void collect_exports (void); diff --git a/info/nbdinfo.pod b/info/nbdinfo.pod index f1344d4..0c03bcd 100644 --- a/info/nbdinfo.pod +++ b/info/nbdinfo.pod @@ -8,7 +8,7 @@ nbdinfo - display information and metadata about NBD servers and exports nbdinfo --size [--json] NBD-URI - nbdinfo --map [--json] NBD-URI + nbdinfo --map [--totals] [--json] NBD-URI nbdinfo -L|--list [--json] NBD-URI @@ -119,6 +119,35 @@ other maps too: For more information on NBD maps, see I in the NBD protocol. +=head2 Map totals + +Using S> performs the same operation as I<--map> but +displays a summary of the total size of each type of allocation, in +bytes and as a percentage (of the virtual size of the export). This +is useful for estimating how much real storage is used on the server, +or might be required when copying a sparse image with L. + +In the example below, half (50.0%) of the disk is allocated data and +half is unallocated: + + $ nbdinfo --map --totals nbd://localhost/ + 1048576 50.0 0 data + 1048576 50.0 3 hole,zero + +The fields are: total size in bytes, percentage of the virtual size, +type, description (optional). + +You can also get the same information in parseable form using I<--json>: + + $ nbdinfo --map --totals --json nbd://localhost/ + [{ "size": 1048576, "percent": 50, + "type": 0, "description": "data" }, + { "size": 1048576, "percent": 50, + "type": 3, "description": "hole,zero" }] + +As with the I<--map> option, by default this shows the +C<"base:allocation"> map, but you can show the summary for other maps. + =head2 List all exports To list all the exports available on an NBD server use the I<--list> @@ -179,7 +208,7 @@ The output is displayed in JSON format. Display the map (usually whether parts of the disk are allocated or sparse) of the given export. This displays the C<"base:allocation"> map by default, you can choose a different map with the optional -parameter. +parameter. Using S> displays a summary. =item B<-L> @@ -188,6 +217,11 @@ parameter. List all the exports on an NBD server. The export name in the NBD URI is ignored. +=item B<--totals> + +Use S> to display a summary. See L +above. + =item B<--size> Display only the size in bytes of the export. -- 2.31.1