diff --git a/0001-tests-test-tar-info-Remove-bogus-test-requires-of-gu.patch b/0001-tests-test-tar-info-Remove-bogus-test-requires-of-gu.patch new file mode 100644 index 0000000..3047bcf --- /dev/null +++ b/0001-tests-test-tar-info-Remove-bogus-test-requires-of-gu.patch @@ -0,0 +1,41 @@ +From d2e335b2d05e68c1fc7ed9b507fd5cafc172a112 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Tue, 13 Jun 2023 20:11:34 +0100 +Subject: [PATCH 01/15] tests/test-tar-info*: Remove bogus test requires of + guestfish + +These tests don't need guestfish so don't test for it. + +Fixes: commit 05b49f89dccb0ca3fd0919e9502c54dbcadf0b73 +--- + tests/test-tar-info-xz.sh | 1 - + tests/test-tar-info.sh | 1 - + 2 files changed, 2 deletions(-) + +diff --git a/tests/test-tar-info-xz.sh b/tests/test-tar-info-xz.sh +index 1f968e9b8..abd854974 100755 +--- a/tests/test-tar-info-xz.sh ++++ b/tests/test-tar-info-xz.sh +@@ -37,7 +37,6 @@ set -e + set -x + + requires test -f disk +-requires guestfish --version + requires tar --version + requires qemu-img --version + requires qemu-img info --output=json /dev/null +diff --git a/tests/test-tar-info.sh b/tests/test-tar-info.sh +index 572768b85..5c015d88e 100755 +--- a/tests/test-tar-info.sh ++++ b/tests/test-tar-info.sh +@@ -37,7 +37,6 @@ set -e + set -x + + requires test -f disk +-requires guestfish --version + requires tar --version + requires qemu-img --version + requires qemu-img info --output=json /dev/null +-- +2.41.0 + diff --git a/0002-tar-Document-the-optional-tar-option-in-help-output-.patch b/0002-tar-Document-the-optional-tar-option-in-help-output-.patch new file mode 100644 index 0000000..84be7bc --- /dev/null +++ b/0002-tar-Document-the-optional-tar-option-in-help-output-.patch @@ -0,0 +1,43 @@ +From eb44ca26a84c369a7908f19517c0a2f8f621d57f Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Tue, 13 Jun 2023 20:25:40 +0100 +Subject: [PATCH 02/15] tar: Document the optional tar option in --help output + and synopsis + +Fixes: commit 05b49f89dccb0ca3fd0919e9502c54dbcadf0b73 +--- + filters/tar/nbdkit-tar-filter.pod | 1 + + filters/tar/tar.c | 5 +++-- + 2 files changed, 4 insertions(+), 2 deletions(-) + +diff --git a/filters/tar/nbdkit-tar-filter.pod b/filters/tar/nbdkit-tar-filter.pod +index 96aba16d9..eb0cfd4ab 100644 +--- a/filters/tar/nbdkit-tar-filter.pod ++++ b/filters/tar/nbdkit-tar-filter.pod +@@ -5,6 +5,7 @@ nbdkit-tar-filter - read and write files inside tar files without unpacking + =head1 SYNOPSIS + + nbdkit file FILENAME.tar --filter=tar tar-entry=PATH_INSIDE_TAR ++ [tar=TAR_COMMAND] + + =head1 EXAMPLES + +diff --git a/filters/tar/tar.c b/filters/tar/tar.c +index c650a3499..f3adb2c46 100644 +--- a/filters/tar/tar.c ++++ b/filters/tar/tar.c +@@ -96,8 +96,9 @@ tar_config_complete (nbdkit_next_config_complete *next, + return next (nxdata); + } + +-#define tar_config_help \ +- "tar-entry= (required) The path inside the tar file to serve." ++#define tar_config_help \ ++ "tar-entry= (required) The path inside the tar file to serve.\n" \ ++ "tar= Path of the tar binary." + + static int + tar_thread_model (void) +-- +2.41.0 + diff --git a/0003-tar-Implement-tar-limit.patch b/0003-tar-Implement-tar-limit.patch new file mode 100644 index 0000000..cd54c95 --- /dev/null +++ b/0003-tar-Implement-tar-limit.patch @@ -0,0 +1,193 @@ +From ae3fb4def811f5bc42563128e8fae0a4288584d1 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Tue, 13 Jun 2023 20:01:05 +0100 +Subject: [PATCH 03/15] tar: Implement tar-limit + +This can be used to ensure that the tar filter does not read +indefinite amounts of input when opening the tar file. + +See: https://github.com/kubevirt/containerized-data-importer/pull/2748#issuecomment-1589852102 +--- + filters/tar/nbdkit-tar-filter.pod | 24 +++++++++++- + tests/Makefile.am | 2 + + filters/tar/tar.c | 10 +++++ + tests/test-tar-limit.sh | 65 +++++++++++++++++++++++++++++++ + 4 files changed, 99 insertions(+), 2 deletions(-) + create mode 100755 tests/test-tar-limit.sh + +diff --git a/filters/tar/nbdkit-tar-filter.pod b/filters/tar/nbdkit-tar-filter.pod +index eb0cfd4ab..a3c2d1328 100644 +--- a/filters/tar/nbdkit-tar-filter.pod ++++ b/filters/tar/nbdkit-tar-filter.pod +@@ -4,8 +4,9 @@ nbdkit-tar-filter - read and write files inside tar files without unpacking + + =head1 SYNOPSIS + +- nbdkit file FILENAME.tar --filter=tar tar-entry=PATH_INSIDE_TAR +- [tar=TAR_COMMAND] ++ nbdkit file FILENAME.tar ++ --filter=tar tar-entry=PATH_INSIDE_TAR ++ [tar=TAR_COMMAND] [tar-limit=SIZE] + + =head1 EXAMPLES + +@@ -83,6 +84,25 @@ The path of the file inside the tarball to serve. This parameter is + required. It must exactly match the name stored in the tarball, so + use S> + ++=item [B]SIZE ++ ++When opening the tar file we have to locate the file (C) ++inside the tarball. Because tar files do not have a central index we ++must iterate over the tar file to find the entry, and that may be ++costly (especially with untrusted tar files). In the worst case where ++C starts near the end of the file we may have to iterate ++over the whole tar file. If this is a problem you may set ++C to some smaller value, eg: ++ ++ nbdkit -r curl https://example.com/file.tar \ ++ --filter=tar tar-entry=disk.img \ ++ tar-limit=10M ++ ++which ensures no more than 10 megabytes are read before we give up and ++reject the tar file (sending an error back to the NBD client). ++ ++The default is 0 meaning no limit. ++ + =item B + + =item B/PATH/TO/GTAR +diff --git a/tests/Makefile.am b/tests/Makefile.am +index 6694e409e..f2912aa93 100644 +--- a/tests/Makefile.am ++++ b/tests/Makefile.am +@@ -1957,11 +1957,13 @@ TESTS += \ + test-tar.sh \ + test-tar-info.sh \ + test-tar-info-xz.sh \ ++ test-tar-limit.sh \ + $(NULL) + EXTRA_DIST += \ + test-tar.sh \ + test-tar-info.sh \ + test-tar-info-xz.sh \ ++ test-tar-limit.sh \ + $(NULL) + + # truncate filter tests. +diff --git a/filters/tar/tar.c b/filters/tar/tar.c +index f3adb2c46..efe47684d 100644 +--- a/filters/tar/tar.c ++++ b/filters/tar/tar.c +@@ -53,6 +53,7 @@ + #include "utils.h" + + static const char *entry; /* File within tar (tar-entry=...) */ ++static int64_t tar_limit = 0; + static const char *tar_program = "tar"; + + /* Offset and size within tarball. +@@ -76,6 +77,12 @@ tar_config (nbdkit_next_config *next, nbdkit_backend *nxdata, + entry = value; + return 0; + } ++ else if (strcmp (key, "tar-limit") == 0) { ++ tar_limit = nbdkit_parse_size (value); ++ if (tar_limit == -1) ++ return -1; ++ return 0; ++ } + else if (strcmp (key, "tar") == 0) { + tar_program = value; + return 0; +@@ -98,6 +105,7 @@ tar_config_complete (nbdkit_next_config_complete *next, + + #define tar_config_help \ + "tar-entry= (required) The path inside the tar file to serve.\n" \ ++ "tar-limit=SIZE Limit on reading to find entry.\n" \ + "tar= Path of the tar binary." + + static int +@@ -197,6 +205,8 @@ calculate_offset_of_entry (nbdkit_next *next) + copysize = next->get_size (next); + if (copysize == -1) + return -1; ++ if (tar_limit > 0 && copysize > tar_limit) ++ copysize = tar_limit; + + /* Run the tar command. */ + nbdkit_debug ("%s", cmd); +diff --git a/tests/test-tar-limit.sh b/tests/test-tar-limit.sh +new file mode 100755 +index 000000000..168f2c497 +--- /dev/null ++++ b/tests/test-tar-limit.sh +@@ -0,0 +1,65 @@ ++#!/usr/bin/env bash ++# nbdkit ++# Copyright Red Hat ++# ++# Redistribution and use in source and binary forms, with or without ++# modification, are permitted provided that the following conditions are ++# met: ++# ++# * Redistributions of source code must retain the above copyright ++# notice, this list of conditions and the following disclaimer. ++# ++# * Redistributions in binary form must reproduce the above copyright ++# notice, this list of conditions and the following disclaimer in the ++# documentation and/or other materials provided with the distribution. ++# ++# * Neither the name of Red Hat nor the names of its contributors may be ++# used to endorse or promote products derived from this software without ++# specific prior written permission. ++# ++# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND ++# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ++# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A ++# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR ++# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF ++# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ++# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, ++# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT ++# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF ++# SUCH DAMAGE. ++ ++# Test the tar filter and tar-limit filter. ++ ++source ./functions.sh ++set -e ++set -x ++ ++requires test -f disk ++requires tar --version ++requires test -f disk ++requires_nbdinfo ++ ++tar_bad=tar-limit-bad.tar ++tar_good=tar-limit-good.tar ++tar_filler=tar-limit-filler.img ++files="$tar_bad $tar_good $tar_filler" ++rm -f $files ++cleanup_fn rm -f $files ++ ++# Create two tar files, one where the disk is located before an ++# arbitrary boundary and one after. ++truncate -s 1M $tar_filler ++tar cf $tar_good disk ++tar cf $tar_bad $tar_filler disk ++ ++# Check we can read the good disk and reject the bad disk. ++cmd="nbdkit -U - file --filter=tar tar-entry=disk tar-limit=131072" ++ ++$cmd $tar_good --run 'nbdinfo "$uri"' ++ ++if $cmd $tar_bad --run 'nbdinfo "$uri"' ; then ++ echo "ERROR: $0: expected $tar_bad to fail" ++ exit 1 ++fi +-- +2.41.0 + diff --git a/0004-xz-Fix-error-message-to-refer-to-xz-max-block-parame.patch b/0004-xz-Fix-error-message-to-refer-to-xz-max-block-parame.patch new file mode 100644 index 0000000..ed32847 --- /dev/null +++ b/0004-xz-Fix-error-message-to-refer-to-xz-max-block-parame.patch @@ -0,0 +1,40 @@ +From 35db543e38293808e910b291ff26b17284640ec9 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Mon, 19 Jun 2023 13:53:09 +0100 +Subject: [PATCH 04/15] xz: Fix error message to refer to xz-max-block + parameter + +In the original xz plugin the parameter was called "maxblock", but +when we converted the plugin to a filter (commit c879d31047 "Rewrite +xz plugin as a filter.") we changed the parameter name to +"xz-max-block" but retained the old name in the error message. + +Fixes: commit c879d31047ae149cedabd5abbdb0a95e610f01c5 +--- + filters/xz/xz.c | 8 ++++---- + 1 file changed, 4 insertions(+), 4 deletions(-) + +diff --git a/filters/xz/xz.c b/filters/xz/xz.c +index 0c92cfff4..21e8993b5 100644 +--- a/filters/xz/xz.c ++++ b/filters/xz/xz.c +@@ -143,12 +143,12 @@ xz_prepare (nbdkit_next *next, void *handle, + return -1; + + if (maxblock < xzfile_max_uncompressed_block_size (h->xz)) { +- nbdkit_error ("xz file largest block is bigger than maxblock\n" ++ nbdkit_error ("xz file largest block is bigger than xz-max-block\n" + "Either recompress the xz file with smaller blocks " + "(see nbdkit-xz-filter(1))\n" +- "or make maxblock parameter bigger.\n" +- "maxblock = %" PRIu64 " (bytes)\n" +- "largest block in xz file = %" PRIu64 " (bytes)", ++ "or make xz-max-block parameter bigger.\n" ++ "Current xz-max-block = %" PRIu64 " (bytes)\n" ++ "Largest block in xz file = %" PRIu64 " (bytes)", + maxblock, + xzfile_max_uncompressed_block_size (h->xz)); + return -1; +-- +2.41.0 + diff --git a/0005-configure-Recommend-using-g-with-OCAMLOPTFLAGS.patch b/0005-configure-Recommend-using-g-with-OCAMLOPTFLAGS.patch new file mode 100644 index 0000000..6d419b0 --- /dev/null +++ b/0005-configure-Recommend-using-g-with-OCAMLOPTFLAGS.patch @@ -0,0 +1,27 @@ +From 4398b1871f8832109fcdbe25c70a1dd341ed5bd4 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Wed, 21 Jun 2023 13:05:22 +0100 +Subject: [PATCH 05/15] configure: Recommend using -g with OCAMLOPTFLAGS + +--- + configure.ac | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/configure.ac b/configure.ac +index c1d83a568..39b58b44e 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -935,8 +935,8 @@ AC_SUBST([PYTHON_LDFLAGS]) + + dnl For the OCaml plugin, you can set OCAMLOPTFLAGS before running + dnl ./configure to specify any extra flags you want to pass to +-dnl ocamlopt. For example to enable OCaml warnings: +-dnl OCAMLOPTFLAGS="-warn-error +A-3" ++dnl ocamlopt. For example to enable debug symbols & warnings: ++dnl OCAMLOPTFLAGS="-g -warn-error +A-3" + AC_SUBST([OCAMLOPTFLAGS]) + + dnl Check for OCaml, for embedding in the ocaml plugin. +-- +2.41.0 + diff --git a/0006-tests-test_ocaml_plugin.ml-Print-a-message-when-test.patch b/0006-tests-test_ocaml_plugin.ml-Print-a-message-when-test.patch new file mode 100644 index 0000000..25fbbdb --- /dev/null +++ b/0006-tests-test_ocaml_plugin.ml-Print-a-message-when-test.patch @@ -0,0 +1,32 @@ +From b53b78e542fe8e49ce5c670e8ddc9d2be66f73cc Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Wed, 21 Jun 2023 18:47:40 +0100 +Subject: [PATCH 06/15] tests/test_ocaml_plugin.ml: Print a message when test + plugin initializes + +It's useful to have a message which is printed as the top level +statements in the plugin module are being initialized, for debugging +purposes. +--- + tests/test_ocaml_plugin.ml | 5 +++++ + 1 file changed, 5 insertions(+) + +diff --git a/tests/test_ocaml_plugin.ml b/tests/test_ocaml_plugin.ml +index 561a8690f..f8eed4f7f 100644 +--- a/tests/test_ocaml_plugin.ml ++++ b/tests/test_ocaml_plugin.ml +@@ -30,6 +30,11 @@ + * SUCH DAMAGE. + *) + ++(* Print something during module initialization, useful for debugging ++ * obscure OCaml startup issues. ++ *) ++let () = Printf.eprintf "test_ocaml_plugin.ml: module initializing\n%!" ++ + let sector_size = 512 + let nr_sectors = 2048 + +-- +2.41.0 + diff --git a/0007-ocaml-Add-I-unix-before-using-unix.cmxa.patch b/0007-ocaml-Add-I-unix-before-using-unix.cmxa.patch new file mode 100644 index 0000000..cb40bc2 --- /dev/null +++ b/0007-ocaml-Add-I-unix-before-using-unix.cmxa.patch @@ -0,0 +1,124 @@ +From b778c56443e11817d4974dcab5356e2d4065980b Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Wed, 21 Jun 2023 18:56:58 +0100 +Subject: [PATCH 07/15] ocaml: Add -I +unix before using unix.cmxa + +In OCaml 5.0 you will see this warning: + + Alert ocaml_deprecated_auto_include: + OCaml's lib directory layout changed in 5.0. The unix subdirectory has been + automatically added to the search path, but you should add -I +unix to the + command-line to silence this alert (e.g. by adding unix to the list of + libraries in your dune file, or adding use_unix to your _tags file for + ocamlbuild, or using -package unix for ocamlfind). + +Using -I +unix doesn't hurt earlier versions, as the directory is just +ignored if it doesn't exist, and anyway findlib adds the directory if +you're using that. +--- + plugins/cc/nbdkit-cc-plugin.pod | 4 ++-- + plugins/ocaml/nbdkit-ocaml-plugin.pod | 2 +- + plugins/ocaml/Makefile.am | 2 +- + tests/Makefile.am | 4 ++-- + tests/test-cc-ocaml.sh | 2 +- + tests/cc_shebang.ml | 2 +- + 6 files changed, 8 insertions(+), 8 deletions(-) + +diff --git a/plugins/cc/nbdkit-cc-plugin.pod b/plugins/cc/nbdkit-cc-plugin.pod +index 412233797..f55f74ab0 100644 +--- a/plugins/cc/nbdkit-cc-plugin.pod ++++ b/plugins/cc/nbdkit-cc-plugin.pod +@@ -89,7 +89,7 @@ C as a parameter to exec nbdkit. + =head2 Using this plugin with OCaml + + nbdkit cc CC=ocamlopt \ +- CFLAGS="-output-obj -runtime-variant _pic unix.cmxa NBDKit.cmx -cclib -lnbdkitocaml" \ ++ CFLAGS="-output-obj -runtime-variant _pic -I +unix unix.cmxa NBDKit.cmx -cclib -lnbdkitocaml" \ + source.ml + + OCaml plugin scripts can be created using this trick: +@@ -97,7 +97,7 @@ OCaml plugin scripts can be created using this trick: + (*/.)>/dev/null 2>&1 + exec nbdkit cc "$0" \ + CC=ocamlopt \ +- CFLAGS="-output-obj -runtime-variant _pic unix.cmxa NBDKit.cmx -cclib -lnbdkitocaml" \ ++ CFLAGS="-output-obj -runtime-variant _pic -I +unix unix.cmxa NBDKit.cmx -cclib -lnbdkitocaml" \ + "$@" + *) + (* followed by OCaml code for the plugin here *) +diff --git a/plugins/ocaml/nbdkit-ocaml-plugin.pod b/plugins/ocaml/nbdkit-ocaml-plugin.pod +index dc8260174..e4a8cf0b0 100644 +--- a/plugins/ocaml/nbdkit-ocaml-plugin.pod ++++ b/plugins/ocaml/nbdkit-ocaml-plugin.pod +@@ -53,7 +53,7 @@ using this command: + + ocamlopt.opt -output-obj -runtime-variant _pic \ + -o nbdkit-myplugin-plugin.so \ +- unix.cmxa NBDKit.cmx myplugin.ml \ ++ -I +unix unix.cmxa NBDKit.cmx myplugin.ml \ + -cclib -lnbdkitocaml + + You can then use C as an nbdkit plugin (see +diff --git a/plugins/ocaml/Makefile.am b/plugins/ocaml/Makefile.am +index da1b1ec96..e7faae506 100644 +--- a/plugins/ocaml/Makefile.am ++++ b/plugins/ocaml/Makefile.am +@@ -84,7 +84,7 @@ noinst_SCRIPTS = nbdkit-ocamlexample-plugin.so + nbdkit-ocamlexample-plugin.so: example.cmx libnbdkitocaml.la NBDKit.cmi NBDKit.cmx + $(OCAMLOPT) $(OCAMLOPTFLAGS) \ + -output-obj -runtime-variant _pic -o $@ \ +- unix.cmxa NBDKit.cmx $< \ ++ -I +unix unix.cmxa NBDKit.cmx $< \ + -cclib -L.libs -cclib -lnbdkitocaml + example.cmx: example.ml NBDKit.cmi NBDKit.cmx + $(OCAMLOPT) $(OCAMLOPTFLAGS) -c $< -o $@ +diff --git a/tests/Makefile.am b/tests/Makefile.am +index f2912aa93..d8a640e1e 100644 +--- a/tests/Makefile.am ++++ b/tests/Makefile.am +@@ -1185,7 +1185,7 @@ OCAML_PLUGIN_DEPS = \ + test-ocaml-plugin.so: test_ocaml_plugin.cmx $(OCAML_PLUGIN_DEPS) + $(OCAMLOPT) $(OCAMLOPTFLAGS) -I ../plugins/ocaml \ + -output-obj -runtime-variant _pic -o $@ \ +- unix.cmxa NBDKit.cmx $< \ ++ -I +unix unix.cmxa NBDKit.cmx $< \ + -cclib -L../plugins/ocaml/.libs -cclib -lnbdkitocaml + test_ocaml_plugin.cmx: test_ocaml_plugin.ml $(OCAML_PLUGIN_DEPS) + $(OCAMLOPT) $(OCAMLOPTFLAGS) -I ../plugins/ocaml -c $< -o $@ +@@ -1194,7 +1194,7 @@ test-ocaml-errorcodes-plugin.so: \ + test_ocaml_errorcodes_plugin.cmx $(OCAML_PLUGIN_DEPS) + $(OCAMLOPT) $(OCAMLOPTFLAGS) -I ../plugins/ocaml \ + -output-obj -runtime-variant _pic -o $@ \ +- unix.cmxa NBDKit.cmx $< \ ++ -I +unix unix.cmxa NBDKit.cmx $< \ + -cclib -L../plugins/ocaml/.libs -cclib -lnbdkitocaml + test_ocaml_errorcodes_plugin.cmx: \ + test_ocaml_errorcodes_plugin.ml $(OCAML_PLUGIN_DEPS) +diff --git a/tests/test-cc-ocaml.sh b/tests/test-cc-ocaml.sh +index 659fa195e..3b4f6a553 100755 +--- a/tests/test-cc-ocaml.sh ++++ b/tests/test-cc-ocaml.sh +@@ -61,6 +61,6 @@ cleanup_fn rm -f $out + rm -f $out + + nbdkit -U - cc $script a=1 b=2 c=3 d=4 \ +- CC="$OCAMLOPT" CFLAGS="-output-obj -runtime-variant _pic -I $SRCDIR/../plugins/ocaml unix.cmxa NBDKit.cmx -cclib -lnbdkitocaml" \ ++ CC="$OCAMLOPT" CFLAGS="-output-obj -runtime-variant _pic -I $SRCDIR/../plugins/ocaml -I +unix unix.cmxa NBDKit.cmx -cclib -lnbdkitocaml" \ + --run 'nbdinfo --size $uri' > $out + test "$(cat $out)" -eq $((512 * 2048)) +diff --git a/tests/cc_shebang.ml b/tests/cc_shebang.ml +index d78d76618..619b08bb5 100755 +--- a/tests/cc_shebang.ml ++++ b/tests/cc_shebang.ml +@@ -4,7 +4,7 @@ + # shell as an impossible command which is ignored. The line below is + # run by the shell and ignored by OCaml. + +-exec nbdkit cc "$0" CC=ocamlopt CFLAGS="-output-obj -runtime-variant _pic unix.cmxa NBDKit.cmx -cclib -lnbdkitocaml" "$@" ++exec nbdkit cc "$0" CC=ocamlopt CFLAGS="-output-obj -runtime-variant _pic -I +unix unix.cmxa NBDKit.cmx -cclib -lnbdkitocaml" "$@" + *) + + open Printf +-- +2.41.0 + diff --git a/0008-ocaml-Replace-caml_leave_blocking_section-with-caml_.patch b/0008-ocaml-Replace-caml_leave_blocking_section-with-caml_.patch new file mode 100644 index 0000000..f7ba249 --- /dev/null +++ b/0008-ocaml-Replace-caml_leave_blocking_section-with-caml_.patch @@ -0,0 +1,468 @@ +From f8b761c62f177176dfe154a95824576eef740827 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Wed, 21 Jun 2023 19:07:00 +0100 +Subject: [PATCH 08/15] ocaml: Replace caml_leave_blocking_section with + caml_acquire_runtime_system + +Replace: + + caml_leave_blocking_section + ... + caml_enter_blocking_section + +with the more sensibly named (and equivalent): + + caml_acquire_runtime_system + ... + caml_release_runtime_system + +In addition we must release the runtime system just after caml_startup +and only acquire it around callbacks into OCaml code. (The reason for +this is only apparent in a later commit.) + +OCaml 5 is more strict than earlier versions of OCaml and actually +implements the locks using pthread_mutex_t. So in OCaml 5 you will +see a deadlock error: + + Fatal error: Fatal error during lock: Resource deadlock avoided + +which was caused by attempting to double lock (ie. trying to acquire +the lock, when we already held it from caml_startup). + +Also because of additional strictness we must acquire the lock even +before creating the stack frame with CAMLparam* functions. +--- + plugins/ocaml/plugin.h | 16 ++++----- + plugins/ocaml/plugin.c | 75 +++++++++++++++++++++++------------------- + 2 files changed, 49 insertions(+), 42 deletions(-) + +diff --git a/plugins/ocaml/plugin.h b/plugins/ocaml/plugin.h +index 572c43c04..3d6d27db1 100644 +--- a/plugins/ocaml/plugin.h ++++ b/plugins/ocaml/plugin.h +@@ -47,18 +47,18 @@ caml_alloc_initialized_string (mlsize_t len, const char *p) + #endif + + /* For functions which call into OCaml code, call +- * caml_leave_blocking_section() to prevent other threads running, +- * then caml_enter_blocking_section() on return to C code. This macro +- * ensures that the calls are paired properly. ++ * caml_acquire_runtime_system ... caml_release_runtime_system around ++ * the code. This prevents other threads in the same domain running. ++ * The macro ensures that the calls are paired properly. + */ +-#define LEAVE_BLOCKING_SECTION_FOR_CURRENT_SCOPE() \ +- __attribute__ ((unused, cleanup (cleanup_enter_blocking_section))) \ ++#define ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE() \ ++ __attribute__ ((unused, cleanup (cleanup_release_runtime_system))) \ + int _unused; \ +- caml_leave_blocking_section () ++ caml_acquire_runtime_system () + static inline void +-cleanup_enter_blocking_section (int *unused) ++cleanup_release_runtime_system (int *unused) + { +- caml_enter_blocking_section (); ++ caml_release_runtime_system (); + } + + #endif /* NBDKIT_OCAML_PLUGIN_H */ +diff --git a/plugins/ocaml/plugin.c b/plugins/ocaml/plugin.c +index 722b95e49..fe781f9af 100644 +--- a/plugins/ocaml/plugin.c ++++ b/plugins/ocaml/plugin.c +@@ -64,6 +64,12 @@ constructor (void) + + /* Initialize OCaml runtime. */ + caml_startup (argv); ++ ++ /* We need to release the runtime system here so other threads may ++ * use it. Before we call any OCaml callbacks we must acquire the ++ * runtime system again. ++ */ ++ caml_release_runtime_system (); + } + + /* Instead of using the NBDKIT_REGISTER_PLUGIN macro, we construct the +@@ -113,7 +119,7 @@ plugin_init (void) + static void + load_wrapper (void) + { +- LEAVE_BLOCKING_SECTION_FOR_CURRENT_SCOPE (); ++ ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + caml_callback (load_fn, Val_unit); + } + +@@ -123,8 +129,9 @@ load_wrapper (void) + static void + unload_wrapper (void) + { ++ ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); ++ + if (unload_fn) { +- LEAVE_BLOCKING_SECTION_FOR_CURRENT_SCOPE (); + caml_callback (unload_fn, Val_unit); + } + +@@ -139,9 +146,9 @@ unload_wrapper (void) + static void + dump_plugin_wrapper (void) + { ++ ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal1 (rv); +- LEAVE_BLOCKING_SECTION_FOR_CURRENT_SCOPE (); + + rv = caml_callback_exn (dump_plugin_fn, Val_unit); + if (Is_exception_result (rv)) +@@ -152,9 +159,9 @@ dump_plugin_wrapper (void) + static int + config_wrapper (const char *key, const char *val) + { ++ ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal3 (keyv, valv, rv); +- LEAVE_BLOCKING_SECTION_FOR_CURRENT_SCOPE (); + + keyv = caml_copy_string (key); + valv = caml_copy_string (val); +@@ -171,9 +178,9 @@ config_wrapper (const char *key, const char *val) + static int + config_complete_wrapper (void) + { ++ ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal1 (rv); +- LEAVE_BLOCKING_SECTION_FOR_CURRENT_SCOPE (); + + rv = caml_callback_exn (config_complete_fn, Val_unit); + if (Is_exception_result (rv)) { +@@ -187,9 +194,9 @@ config_complete_wrapper (void) + static int + thread_model_wrapper (void) + { ++ ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal1 (rv); +- LEAVE_BLOCKING_SECTION_FOR_CURRENT_SCOPE (); + + rv = caml_callback_exn (thread_model_fn, Val_unit); + if (Is_exception_result (rv)) { +@@ -203,9 +210,9 @@ thread_model_wrapper (void) + static int + get_ready_wrapper (void) + { ++ ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal1 (rv); +- LEAVE_BLOCKING_SECTION_FOR_CURRENT_SCOPE (); + + rv = caml_callback_exn (get_ready_fn, Val_unit); + if (Is_exception_result (rv)) { +@@ -219,9 +226,9 @@ get_ready_wrapper (void) + static int + after_fork_wrapper (void) + { ++ ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal1 (rv); +- LEAVE_BLOCKING_SECTION_FOR_CURRENT_SCOPE (); + + rv = caml_callback_exn (after_fork_fn, Val_unit); + if (Is_exception_result (rv)) { +@@ -235,9 +242,9 @@ after_fork_wrapper (void) + static void + cleanup_wrapper (void) + { ++ ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal1 (rv); +- LEAVE_BLOCKING_SECTION_FOR_CURRENT_SCOPE (); + + rv = caml_callback_exn (cleanup_fn, Val_unit); + if (Is_exception_result (rv)) { +@@ -251,9 +258,9 @@ cleanup_wrapper (void) + static int + preconnect_wrapper (int readonly) + { ++ ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal1 (rv); +- LEAVE_BLOCKING_SECTION_FOR_CURRENT_SCOPE (); + + rv = caml_callback_exn (preconnect_fn, Val_bool (readonly)); + if (Is_exception_result (rv)) { +@@ -267,9 +274,9 @@ preconnect_wrapper (int readonly) + static int + list_exports_wrapper (int readonly, int is_tls, struct nbdkit_exports *exports) + { ++ ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal2 (rv, v); +- LEAVE_BLOCKING_SECTION_FOR_CURRENT_SCOPE (); + + rv = caml_callback2_exn (list_exports_fn, Val_bool (readonly), + Val_bool (is_tls)); +@@ -299,10 +306,10 @@ list_exports_wrapper (int readonly, int is_tls, struct nbdkit_exports *exports) + static const char * + default_export_wrapper (int readonly, int is_tls) + { ++ ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal1 (rv); + const char *name; +- LEAVE_BLOCKING_SECTION_FOR_CURRENT_SCOPE (); + + rv = caml_callback2_exn (default_export_fn, Val_bool (readonly), + Val_bool (is_tls)); +@@ -318,10 +325,10 @@ default_export_wrapper (int readonly, int is_tls) + static void * + open_wrapper (int readonly) + { ++ ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal1 (rv); + value *ret; +- LEAVE_BLOCKING_SECTION_FOR_CURRENT_SCOPE (); + + rv = caml_callback_exn (open_fn, Val_bool (readonly)); + if (Is_exception_result (rv)) { +@@ -341,9 +348,9 @@ open_wrapper (int readonly) + static void + close_wrapper (void *h) + { ++ ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal1 (rv); +- LEAVE_BLOCKING_SECTION_FOR_CURRENT_SCOPE (); + + rv = caml_callback_exn (close_fn, *(value *) h); + if (Is_exception_result (rv)) { +@@ -360,10 +367,10 @@ close_wrapper (void *h) + static const char * + export_description_wrapper (void *h) + { ++ ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal1 (rv); + const char *desc; +- LEAVE_BLOCKING_SECTION_FOR_CURRENT_SCOPE (); + + rv = caml_callback_exn (export_description_fn, *(value *) h); + if (Is_exception_result (rv)) { +@@ -378,10 +385,10 @@ export_description_wrapper (void *h) + static int64_t + get_size_wrapper (void *h) + { ++ ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal1 (rv); + int64_t r; +- LEAVE_BLOCKING_SECTION_FOR_CURRENT_SCOPE (); + + rv = caml_callback_exn (get_size_fn, *(value *) h); + if (Is_exception_result (rv)) { +@@ -397,11 +404,11 @@ static int + block_size_wrapper (void *h, + uint32_t *minimum, uint32_t *preferred, uint32_t *maximum) + { ++ ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal1 (rv); + int i; + int64_t i64; +- LEAVE_BLOCKING_SECTION_FOR_CURRENT_SCOPE (); + + rv = caml_callback_exn (block_size_fn, *(value *) h); + if (Is_exception_result (rv)) { +@@ -439,9 +446,9 @@ block_size_wrapper (void *h, + static int + can_write_wrapper (void *h) + { ++ ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal1 (rv); +- LEAVE_BLOCKING_SECTION_FOR_CURRENT_SCOPE (); + + rv = caml_callback_exn (can_write_fn, *(value *) h); + if (Is_exception_result (rv)) { +@@ -455,9 +462,9 @@ can_write_wrapper (void *h) + static int + can_flush_wrapper (void *h) + { ++ ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal1 (rv); +- LEAVE_BLOCKING_SECTION_FOR_CURRENT_SCOPE (); + + rv = caml_callback_exn (can_flush_fn, *(value *) h); + if (Is_exception_result (rv)) { +@@ -471,9 +478,9 @@ can_flush_wrapper (void *h) + static int + is_rotational_wrapper (void *h) + { ++ ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal1 (rv); +- LEAVE_BLOCKING_SECTION_FOR_CURRENT_SCOPE (); + + rv = caml_callback_exn (is_rotational_fn, *(value *) h); + if (Is_exception_result (rv)) { +@@ -487,9 +494,9 @@ is_rotational_wrapper (void *h) + static int + can_trim_wrapper (void *h) + { ++ ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal1 (rv); +- LEAVE_BLOCKING_SECTION_FOR_CURRENT_SCOPE (); + + rv = caml_callback_exn (can_trim_fn, *(value *) h); + if (Is_exception_result (rv)) { +@@ -503,9 +510,9 @@ can_trim_wrapper (void *h) + static int + can_zero_wrapper (void *h) + { ++ ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal1 (rv); +- LEAVE_BLOCKING_SECTION_FOR_CURRENT_SCOPE (); + + rv = caml_callback_exn (can_zero_fn, *(value *) h); + if (Is_exception_result (rv)) { +@@ -519,9 +526,9 @@ can_zero_wrapper (void *h) + static int + can_fua_wrapper (void *h) + { ++ ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal1 (rv); +- LEAVE_BLOCKING_SECTION_FOR_CURRENT_SCOPE (); + + rv = caml_callback_exn (can_fua_fn, *(value *) h); + if (Is_exception_result (rv)) { +@@ -535,9 +542,9 @@ can_fua_wrapper (void *h) + static int + can_fast_zero_wrapper (void *h) + { ++ ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal1 (rv); +- LEAVE_BLOCKING_SECTION_FOR_CURRENT_SCOPE (); + + rv = caml_callback_exn (can_fast_zero_fn, *(value *) h); + if (Is_exception_result (rv)) { +@@ -551,9 +558,9 @@ can_fast_zero_wrapper (void *h) + static int + can_cache_wrapper (void *h) + { ++ ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal1 (rv); +- LEAVE_BLOCKING_SECTION_FOR_CURRENT_SCOPE (); + + rv = caml_callback_exn (can_cache_fn, *(value *) h); + if (Is_exception_result (rv)) { +@@ -567,9 +574,9 @@ can_cache_wrapper (void *h) + static int + can_extents_wrapper (void *h) + { ++ ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal1 (rv); +- LEAVE_BLOCKING_SECTION_FOR_CURRENT_SCOPE (); + + rv = caml_callback_exn (can_extents_fn, *(value *) h); + if (Is_exception_result (rv)) { +@@ -583,9 +590,9 @@ can_extents_wrapper (void *h) + static int + can_multi_conn_wrapper (void *h) + { ++ ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal1 (rv); +- LEAVE_BLOCKING_SECTION_FOR_CURRENT_SCOPE (); + + rv = caml_callback_exn (can_multi_conn_fn, *(value *) h); + if (Is_exception_result (rv)) { +@@ -629,10 +636,10 @@ static int + pread_wrapper (void *h, void *buf, uint32_t count, uint64_t offset, + uint32_t flags) + { ++ ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal4 (rv, countv, offsetv, flagsv); + mlsize_t len; +- LEAVE_BLOCKING_SECTION_FOR_CURRENT_SCOPE (); + + countv = Val_int (count); + offsetv = caml_copy_int64 (offset); +@@ -659,9 +666,9 @@ static int + pwrite_wrapper (void *h, const void *buf, uint32_t count, uint64_t offset, + uint32_t flags) + { ++ ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal4 (rv, strv, offsetv, flagsv); +- LEAVE_BLOCKING_SECTION_FOR_CURRENT_SCOPE (); + + strv = caml_alloc_initialized_string (count, buf); + offsetv = caml_copy_int64 (offset); +@@ -680,9 +687,9 @@ pwrite_wrapper (void *h, const void *buf, uint32_t count, uint64_t offset, + static int + flush_wrapper (void *h, uint32_t flags) + { ++ ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal2 (rv, flagsv); +- LEAVE_BLOCKING_SECTION_FOR_CURRENT_SCOPE (); + + flagsv = Val_flags (flags); + +@@ -698,9 +705,9 @@ flush_wrapper (void *h, uint32_t flags) + static int + trim_wrapper (void *h, uint32_t count, uint64_t offset, uint32_t flags) + { ++ ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal4 (rv, countv, offsetv, flagsv); +- LEAVE_BLOCKING_SECTION_FOR_CURRENT_SCOPE (); + + countv = caml_copy_int64 (count); + offsetv = caml_copy_int64 (offset); +@@ -719,9 +726,9 @@ trim_wrapper (void *h, uint32_t count, uint64_t offset, uint32_t flags) + static int + zero_wrapper (void *h, uint32_t count, uint64_t offset, uint32_t flags) + { ++ ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal4 (rv, countv, offsetv, flagsv); +- LEAVE_BLOCKING_SECTION_FOR_CURRENT_SCOPE (); + + countv = caml_copy_int64 (count); + offsetv = caml_copy_int64 (offset); +@@ -741,9 +748,9 @@ static int + extents_wrapper (void *h, uint32_t count, uint64_t offset, uint32_t flags, + struct nbdkit_extents *extents) + { ++ ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal5 (rv, countv, offsetv, flagsv, v); +- LEAVE_BLOCKING_SECTION_FOR_CURRENT_SCOPE (); + + countv = caml_copy_int64 (count); + offsetv = caml_copy_int64 (offset); +@@ -781,9 +788,9 @@ extents_wrapper (void *h, uint32_t count, uint64_t offset, uint32_t flags, + static int + cache_wrapper (void *h, uint32_t count, uint64_t offset, uint32_t flags) + { ++ ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal4 (rv, countv, offsetv, flagsv); +- LEAVE_BLOCKING_SECTION_FOR_CURRENT_SCOPE (); + + countv = caml_copy_int64 (count); + offsetv = caml_copy_int64 (offset); +-- +2.41.0 + diff --git a/0009-ocaml-Always-unregister-the-global-root-and-free-the.patch b/0009-ocaml-Always-unregister-the-global-root-and-free-the.patch new file mode 100644 index 0000000..2380388 --- /dev/null +++ b/0009-ocaml-Always-unregister-the-global-root-and-free-the.patch @@ -0,0 +1,65 @@ +From f4d5f179c9be6db3c39bc0752e36f262c3f87716 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Wed, 21 Jun 2023 20:15:46 +0100 +Subject: [PATCH 09/15] ocaml: Always unregister the global root and free the + handle + +If the OCaml code did not provide a close method, we would never call +close_wrapper, and then we ended up leaking the global root and handle. +--- + plugins/ocaml/plugin.c | 18 ++++++++++++++---- + 1 file changed, 14 insertions(+), 4 deletions(-) + +diff --git a/plugins/ocaml/plugin.c b/plugins/ocaml/plugin.c +index fe781f9af..a4671d6ed 100644 +--- a/plugins/ocaml/plugin.c ++++ b/plugins/ocaml/plugin.c +@@ -76,6 +76,7 @@ constructor (void) + * nbdkit_plugin struct and return it from our own plugin_init + * function. + */ ++static void close_wrapper (void *h); + static void unload_wrapper (void); + static void free_strings (void); + static void remove_roots (void); +@@ -92,6 +93,10 @@ static struct nbdkit_plugin plugin = { + */ + .name = NULL, + ++ /* We always call these, even if the OCaml code does not provide a ++ * callback. ++ */ ++ .close = close_wrapper, + .unload = unload_wrapper, + }; + +@@ -345,6 +350,9 @@ open_wrapper (int readonly) + CAMLreturnT (void *, ret); + } + ++/* We always have a close function, since we need to unregister the ++ * global root and free the handle. ++ */ + static void + close_wrapper (void *h) + { +@@ -352,10 +360,12 @@ close_wrapper (void *h) + CAMLparam0 (); + CAMLlocal1 (rv); + +- rv = caml_callback_exn (close_fn, *(value *) h); +- if (Is_exception_result (rv)) { +- nbdkit_error ("%s", caml_format_exception (Extract_exception (rv))); +- /*FALLTHROUGH*/ ++ if (close_fn) { ++ rv = caml_callback_exn (close_fn, *(value *) h); ++ if (Is_exception_result (rv)) { ++ nbdkit_error ("%s", caml_format_exception (Extract_exception (rv))); ++ /*FALLTHROUGH*/ ++ } + } + + caml_remove_generational_global_root (h); +-- +2.41.0 + diff --git a/0010-ocaml-Fix-thread-registration-for-OCaml-5.patch b/0010-ocaml-Fix-thread-registration-for-OCaml-5.patch new file mode 100644 index 0000000..05982d0 --- /dev/null +++ b/0010-ocaml-Fix-thread-registration-for-OCaml-5.patch @@ -0,0 +1,375 @@ +From 9d4b87e0338d45456c0506fd2c25b80f7ef6b1e1 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Wed, 21 Jun 2023 20:18:58 +0100 +Subject: [PATCH 10/15] ocaml: Fix thread registration for OCaml 5 + +OCaml 5 is strict about registering threads before calling OCaml heap +functions, and will abort the program with this error if you don't do +this correctly: + + Fatal error: no domain lock held + +Fix this as explained in the comment. + +Note (as it's not explained well in the documentation): Threads +created from C are placed in OCaml thread domain 0. In order to add +them to this domain, the main program must not hold on to the runtime +system lock (for domain 0, because that's what caml_startup gives +you). For this to work we must only hold this lock in the main +program briefly around calls to OCaml code, which means we must +release the runtime system after calling caml_startup as we did in an +earlier commit. +--- + plugins/cc/nbdkit-cc-plugin.pod | 4 +-- + plugins/ocaml/nbdkit-ocaml-plugin.pod | 3 +- + plugins/ocaml/Makefile.am | 2 +- + tests/Makefile.am | 4 +-- + plugins/ocaml/plugin.c | 52 +++++++++++++++++++++++++-- + tests/test-cc-ocaml.sh | 2 +- + tests/cc_shebang.ml | 2 +- + 7 files changed, 59 insertions(+), 10 deletions(-) + +diff --git a/plugins/cc/nbdkit-cc-plugin.pod b/plugins/cc/nbdkit-cc-plugin.pod +index f55f74ab0..e393457c4 100644 +--- a/plugins/cc/nbdkit-cc-plugin.pod ++++ b/plugins/cc/nbdkit-cc-plugin.pod +@@ -89,7 +89,7 @@ C as a parameter to exec nbdkit. + =head2 Using this plugin with OCaml + + nbdkit cc CC=ocamlopt \ +- CFLAGS="-output-obj -runtime-variant _pic -I +unix unix.cmxa NBDKit.cmx -cclib -lnbdkitocaml" \ ++ CFLAGS="-output-obj -runtime-variant _pic -I +unix unix.cmxa -I +threads threads.cmxa NBDKit.cmx -cclib -lnbdkitocaml" \ + source.ml + + OCaml plugin scripts can be created using this trick: +@@ -97,7 +97,7 @@ OCaml plugin scripts can be created using this trick: + (*/.)>/dev/null 2>&1 + exec nbdkit cc "$0" \ + CC=ocamlopt \ +- CFLAGS="-output-obj -runtime-variant _pic -I +unix unix.cmxa NBDKit.cmx -cclib -lnbdkitocaml" \ ++ CFLAGS="-output-obj -runtime-variant _pic -I +unix unix.cmxa -I +threads threads.cmxa NBDKit.cmx -cclib -lnbdkitocaml" \ + "$@" + *) + (* followed by OCaml code for the plugin here *) +diff --git a/plugins/ocaml/nbdkit-ocaml-plugin.pod b/plugins/ocaml/nbdkit-ocaml-plugin.pod +index e4a8cf0b0..f1e06d3e2 100644 +--- a/plugins/ocaml/nbdkit-ocaml-plugin.pod ++++ b/plugins/ocaml/nbdkit-ocaml-plugin.pod +@@ -53,7 +53,8 @@ using this command: + + ocamlopt.opt -output-obj -runtime-variant _pic \ + -o nbdkit-myplugin-plugin.so \ +- -I +unix unix.cmxa NBDKit.cmx myplugin.ml \ ++ -I +unix unix.cmxa -I +threads threads.cmxa \ ++ NBDKit.cmx myplugin.ml \ + -cclib -lnbdkitocaml + + You can then use C as an nbdkit plugin (see +diff --git a/plugins/ocaml/Makefile.am b/plugins/ocaml/Makefile.am +index e7faae506..a61550cb9 100644 +--- a/plugins/ocaml/Makefile.am ++++ b/plugins/ocaml/Makefile.am +@@ -84,7 +84,7 @@ noinst_SCRIPTS = nbdkit-ocamlexample-plugin.so + nbdkit-ocamlexample-plugin.so: example.cmx libnbdkitocaml.la NBDKit.cmi NBDKit.cmx + $(OCAMLOPT) $(OCAMLOPTFLAGS) \ + -output-obj -runtime-variant _pic -o $@ \ +- -I +unix unix.cmxa NBDKit.cmx $< \ ++ -I +unix unix.cmxa -I +threads threads.cmxa NBDKit.cmx $< \ + -cclib -L.libs -cclib -lnbdkitocaml + example.cmx: example.ml NBDKit.cmi NBDKit.cmx + $(OCAMLOPT) $(OCAMLOPTFLAGS) -c $< -o $@ +diff --git a/tests/Makefile.am b/tests/Makefile.am +index d8a640e1e..32ebb7002 100644 +--- a/tests/Makefile.am ++++ b/tests/Makefile.am +@@ -1185,7 +1185,7 @@ OCAML_PLUGIN_DEPS = \ + test-ocaml-plugin.so: test_ocaml_plugin.cmx $(OCAML_PLUGIN_DEPS) + $(OCAMLOPT) $(OCAMLOPTFLAGS) -I ../plugins/ocaml \ + -output-obj -runtime-variant _pic -o $@ \ +- -I +unix unix.cmxa NBDKit.cmx $< \ ++ -I +unix unix.cmxa -I +threads threads.cmxa NBDKit.cmx $< \ + -cclib -L../plugins/ocaml/.libs -cclib -lnbdkitocaml + test_ocaml_plugin.cmx: test_ocaml_plugin.ml $(OCAML_PLUGIN_DEPS) + $(OCAMLOPT) $(OCAMLOPTFLAGS) -I ../plugins/ocaml -c $< -o $@ +@@ -1194,7 +1194,7 @@ test-ocaml-errorcodes-plugin.so: \ + test_ocaml_errorcodes_plugin.cmx $(OCAML_PLUGIN_DEPS) + $(OCAMLOPT) $(OCAMLOPTFLAGS) -I ../plugins/ocaml \ + -output-obj -runtime-variant _pic -o $@ \ +- -I +unix unix.cmxa NBDKit.cmx $< \ ++ -I +unix unix.cmxa -I +threads threads.cmxa NBDKit.cmx $< \ + -cclib -L../plugins/ocaml/.libs -cclib -lnbdkitocaml + test_ocaml_errorcodes_plugin.cmx: \ + test_ocaml_errorcodes_plugin.ml $(OCAML_PLUGIN_DEPS) +diff --git a/plugins/ocaml/plugin.c b/plugins/ocaml/plugin.c +index a4671d6ed..eaa88a925 100644 +--- a/plugins/ocaml/plugin.c ++++ b/plugins/ocaml/plugin.c +@@ -260,9 +260,31 @@ cleanup_wrapper (void) + CAMLreturn0; + } + ++/* A note about nbdkit threads and OCaml: ++ * ++ * OCaml requires that all C threads are registered and unregistered. ++ * ++ * For the main thread callbacks like load, config, get_ready [above ++ * this comment] we don't need to do anything. ++ * ++ * For the connected callbacks [below this comment] nbdkit creates its ++ * own threads but does not provide a way to intercept thread creation ++ * or destruction. However we can register the current thread in ++ * every callback, and unregister the thread only call_wrapper. ++ * ++ * This is safe and cheap: Registering a thread is basically free if ++ * the thread is already registered (the OCaml code checks a ++ * thread-local variable to see if it needs to register). nbdkit will ++ * always call the .close method, which does not necessarily indicate ++ * that the thread is being destroyed, but if the thread is reused we ++ * will register the same thread again when .open or similar is called ++ * next time. ++ */ ++ + static int + preconnect_wrapper (int readonly) + { ++ caml_c_thread_register (); + ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal1 (rv); +@@ -279,6 +301,7 @@ preconnect_wrapper (int readonly) + static int + list_exports_wrapper (int readonly, int is_tls, struct nbdkit_exports *exports) + { ++ caml_c_thread_register (); + ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal2 (rv, v); +@@ -311,6 +334,7 @@ list_exports_wrapper (int readonly, int is_tls, struct nbdkit_exports *exports) + static const char * + default_export_wrapper (int readonly, int is_tls) + { ++ caml_c_thread_register (); + ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal1 (rv); +@@ -330,6 +354,7 @@ default_export_wrapper (int readonly, int is_tls) + static void * + open_wrapper (int readonly) + { ++ caml_c_thread_register (); + ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal1 (rv); +@@ -351,12 +376,13 @@ open_wrapper (int readonly) + } + + /* We always have a close function, since we need to unregister the +- * global root and free the handle. ++ * global root, free the handle and unregister the thread. + */ + static void + close_wrapper (void *h) + { +- ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); ++ caml_c_thread_register (); ++ caml_acquire_runtime_system (); + CAMLparam0 (); + CAMLlocal1 (rv); + +@@ -370,6 +396,8 @@ close_wrapper (void *h) + + caml_remove_generational_global_root (h); + free (h); ++ caml_release_runtime_system (); ++ caml_c_thread_unregister (); + + CAMLreturn0; + } +@@ -377,6 +405,7 @@ close_wrapper (void *h) + static const char * + export_description_wrapper (void *h) + { ++ caml_c_thread_register (); + ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal1 (rv); +@@ -395,6 +424,7 @@ export_description_wrapper (void *h) + static int64_t + get_size_wrapper (void *h) + { ++ caml_c_thread_register (); + ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal1 (rv); +@@ -414,6 +444,7 @@ static int + block_size_wrapper (void *h, + uint32_t *minimum, uint32_t *preferred, uint32_t *maximum) + { ++ caml_c_thread_register (); + ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal1 (rv); +@@ -456,6 +487,7 @@ block_size_wrapper (void *h, + static int + can_write_wrapper (void *h) + { ++ caml_c_thread_register (); + ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal1 (rv); +@@ -472,6 +504,7 @@ can_write_wrapper (void *h) + static int + can_flush_wrapper (void *h) + { ++ caml_c_thread_register (); + ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal1 (rv); +@@ -488,6 +521,7 @@ can_flush_wrapper (void *h) + static int + is_rotational_wrapper (void *h) + { ++ caml_c_thread_register (); + ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal1 (rv); +@@ -504,6 +538,7 @@ is_rotational_wrapper (void *h) + static int + can_trim_wrapper (void *h) + { ++ caml_c_thread_register (); + ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal1 (rv); +@@ -520,6 +555,7 @@ can_trim_wrapper (void *h) + static int + can_zero_wrapper (void *h) + { ++ caml_c_thread_register (); + ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal1 (rv); +@@ -536,6 +572,7 @@ can_zero_wrapper (void *h) + static int + can_fua_wrapper (void *h) + { ++ caml_c_thread_register (); + ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal1 (rv); +@@ -552,6 +589,7 @@ can_fua_wrapper (void *h) + static int + can_fast_zero_wrapper (void *h) + { ++ caml_c_thread_register (); + ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal1 (rv); +@@ -568,6 +606,7 @@ can_fast_zero_wrapper (void *h) + static int + can_cache_wrapper (void *h) + { ++ caml_c_thread_register (); + ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal1 (rv); +@@ -584,6 +623,7 @@ can_cache_wrapper (void *h) + static int + can_extents_wrapper (void *h) + { ++ caml_c_thread_register (); + ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal1 (rv); +@@ -600,6 +640,7 @@ can_extents_wrapper (void *h) + static int + can_multi_conn_wrapper (void *h) + { ++ caml_c_thread_register (); + ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal1 (rv); +@@ -646,6 +687,7 @@ static int + pread_wrapper (void *h, void *buf, uint32_t count, uint64_t offset, + uint32_t flags) + { ++ caml_c_thread_register (); + ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal4 (rv, countv, offsetv, flagsv); +@@ -676,6 +718,7 @@ static int + pwrite_wrapper (void *h, const void *buf, uint32_t count, uint64_t offset, + uint32_t flags) + { ++ caml_c_thread_register (); + ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal4 (rv, strv, offsetv, flagsv); +@@ -697,6 +740,7 @@ pwrite_wrapper (void *h, const void *buf, uint32_t count, uint64_t offset, + static int + flush_wrapper (void *h, uint32_t flags) + { ++ caml_c_thread_register (); + ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal2 (rv, flagsv); +@@ -715,6 +759,7 @@ flush_wrapper (void *h, uint32_t flags) + static int + trim_wrapper (void *h, uint32_t count, uint64_t offset, uint32_t flags) + { ++ caml_c_thread_register (); + ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal4 (rv, countv, offsetv, flagsv); +@@ -736,6 +781,7 @@ trim_wrapper (void *h, uint32_t count, uint64_t offset, uint32_t flags) + static int + zero_wrapper (void *h, uint32_t count, uint64_t offset, uint32_t flags) + { ++ caml_c_thread_register (); + ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal4 (rv, countv, offsetv, flagsv); +@@ -758,6 +804,7 @@ static int + extents_wrapper (void *h, uint32_t count, uint64_t offset, uint32_t flags, + struct nbdkit_extents *extents) + { ++ caml_c_thread_register (); + ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal5 (rv, countv, offsetv, flagsv, v); +@@ -798,6 +845,7 @@ extents_wrapper (void *h, uint32_t count, uint64_t offset, uint32_t flags, + static int + cache_wrapper (void *h, uint32_t count, uint64_t offset, uint32_t flags) + { ++ caml_c_thread_register (); + ACQUIRE_RUNTIME_FOR_CURRENT_SCOPE (); + CAMLparam0 (); + CAMLlocal4 (rv, countv, offsetv, flagsv); +diff --git a/tests/test-cc-ocaml.sh b/tests/test-cc-ocaml.sh +index 3b4f6a553..9458201a4 100755 +--- a/tests/test-cc-ocaml.sh ++++ b/tests/test-cc-ocaml.sh +@@ -61,6 +61,6 @@ cleanup_fn rm -f $out + rm -f $out + + nbdkit -U - cc $script a=1 b=2 c=3 d=4 \ +- CC="$OCAMLOPT" CFLAGS="-output-obj -runtime-variant _pic -I $SRCDIR/../plugins/ocaml -I +unix unix.cmxa NBDKit.cmx -cclib -lnbdkitocaml" \ ++ CC="$OCAMLOPT" CFLAGS="-output-obj -runtime-variant _pic -I $SRCDIR/../plugins/ocaml -I +unix unix.cmxa -I +threads threads.cmxa NBDKit.cmx -cclib -lnbdkitocaml" \ + --run 'nbdinfo --size $uri' > $out + test "$(cat $out)" -eq $((512 * 2048)) +diff --git a/tests/cc_shebang.ml b/tests/cc_shebang.ml +index 619b08bb5..05ca77b64 100755 +--- a/tests/cc_shebang.ml ++++ b/tests/cc_shebang.ml +@@ -4,7 +4,7 @@ + # shell as an impossible command which is ignored. The line below is + # run by the shell and ignored by OCaml. + +-exec nbdkit cc "$0" CC=ocamlopt CFLAGS="-output-obj -runtime-variant _pic -I +unix unix.cmxa NBDKit.cmx -cclib -lnbdkitocaml" "$@" ++exec nbdkit cc "$0" CC=ocamlopt CFLAGS="-output-obj -runtime-variant _pic -I +unix unix.cmxa -I +threads threads.cmxa NBDKit.cmx -cclib -lnbdkitocaml" "$@" + *) + + open Printf +-- +2.41.0 + diff --git a/0011-tests-test-cc-ocaml.sh-Use-nbdkit-v-option-to-help-w.patch b/0011-tests-test-cc-ocaml.sh-Use-nbdkit-v-option-to-help-w.patch new file mode 100644 index 0000000..fc8045c --- /dev/null +++ b/0011-tests-test-cc-ocaml.sh-Use-nbdkit-v-option-to-help-w.patch @@ -0,0 +1,26 @@ +From c78266258709e2cbe6f0e869c866d09ad72c072c Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Thu, 22 Jun 2023 10:33:31 +0100 +Subject: [PATCH 11/15] tests/test-cc-ocaml.sh: Use nbdkit -v option to help + with debugging + +--- + tests/test-cc-ocaml.sh | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/tests/test-cc-ocaml.sh b/tests/test-cc-ocaml.sh +index 9458201a4..fc1335591 100755 +--- a/tests/test-cc-ocaml.sh ++++ b/tests/test-cc-ocaml.sh +@@ -60,7 +60,7 @@ out=test-cc-ocaml.out + cleanup_fn rm -f $out + rm -f $out + +-nbdkit -U - cc $script a=1 b=2 c=3 d=4 \ ++nbdkit -v -U - cc $script a=1 b=2 c=3 d=4 \ + CC="$OCAMLOPT" CFLAGS="-output-obj -runtime-variant _pic -I $SRCDIR/../plugins/ocaml -I +unix unix.cmxa -I +threads threads.cmxa NBDKit.cmx -cclib -lnbdkitocaml" \ + --run 'nbdinfo --size $uri' > $out + test "$(cat $out)" -eq $((512 * 2048)) +-- +2.41.0 + diff --git a/0012-tests-Replace-SRCDIR-with-abs_top_srcdir.patch b/0012-tests-Replace-SRCDIR-with-abs_top_srcdir.patch new file mode 100644 index 0000000..991b3a7 --- /dev/null +++ b/0012-tests-Replace-SRCDIR-with-abs_top_srcdir.patch @@ -0,0 +1,324 @@ +From d968c4858847388e515bc1ef84ff4479c18a414d Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Thu, 22 Jun 2023 10:44:10 +0100 +Subject: [PATCH 12/15] tests: Replace $SRCDIR with $abs_top_srcdir + +functions.sh defines $abs_top_srcdir so there's no need to use $SRCDIR +in any test scripts. +--- + tests/test-cc-cpp.sh | 9 ++------- + tests/test-cc-ocaml.sh | 9 ++------- + tests/test-cc.sh | 9 ++------- + tests/test-python-exception.sh | 12 +++++------- + tests/test-python-export-list.sh | 8 +++----- + tests/test-python-export-name.sh | 8 +++----- + tests/test-python-thread-model.sh | 13 +++++-------- + tests/test-read-password-interactive.sh | 2 +- + tests/test-read-password.sh | 2 +- + tests/test-shebang-cc-ocaml.sh | 3 +-- + tests/test-shebang-cc.sh | 5 ++--- + tests/test-shebang-perl.sh | 4 ++-- + tests/test-shebang-python.sh | 4 ++-- + 13 files changed, 31 insertions(+), 57 deletions(-) + +diff --git a/tests/test-cc-cpp.sh b/tests/test-cc-cpp.sh +index 9a4c16242..7c7bf6353 100755 +--- a/tests/test-cc-cpp.sh ++++ b/tests/test-cc-cpp.sh +@@ -37,12 +37,7 @@ source ./functions.sh + set -e + set -x + +-if test "$SRCDIR" = ""; then +- echo "$0: \$SRCDIR is not set" +- exit 1 +-fi +- +-script=$SRCDIR/test-cc-cpp.cpp ++script=$abs_top_srcdir/tests/test-cc-cpp.cpp + if test ! -f "$script"; then + echo "$0: could not locate test-cc-cpp.cpp" + exit 1 +@@ -59,6 +54,6 @@ rm -f $out + + nbdkit -U - cc $script \ + CC="$CXX" \ +- EXTRA_CFLAGS="-I$SRCDIR/../include" \ ++ EXTRA_CFLAGS="-I$abs_top_srcdir/include" \ + --run 'nbdinfo --size $uri' > $out + test "$(cat $out)" -eq $((100 * 1024 * 1024)) +diff --git a/tests/test-cc-ocaml.sh b/tests/test-cc-ocaml.sh +index fc1335591..db7d252cf 100755 +--- a/tests/test-cc-ocaml.sh ++++ b/tests/test-cc-ocaml.sh +@@ -37,12 +37,7 @@ source ./functions.sh + set -e + set -x + +-if test "$SRCDIR" = ""; then +- echo "$0: \$SRCDIR is not set" +- exit 1 +-fi +- +-script=$SRCDIR/test_ocaml_plugin.ml ++script=$abs_top_srcdir/tests/test_ocaml_plugin.ml + if test ! -f "$script"; then + echo "$0: could not locate test_ocaml_plugin.ml" + exit 1 +@@ -61,6 +56,6 @@ cleanup_fn rm -f $out + rm -f $out + + nbdkit -v -U - cc $script a=1 b=2 c=3 d=4 \ +- CC="$OCAMLOPT" CFLAGS="-output-obj -runtime-variant _pic -I $SRCDIR/../plugins/ocaml -I +unix unix.cmxa -I +threads threads.cmxa NBDKit.cmx -cclib -lnbdkitocaml" \ ++ CC="$OCAMLOPT" CFLAGS="-output-obj -runtime-variant _pic -I $abs_top_srcdir/plugins/ocaml -I +unix unix.cmxa -I +threads threads.cmxa NBDKit.cmx -cclib -lnbdkitocaml" \ + --run 'nbdinfo --size $uri' > $out + test "$(cat $out)" -eq $((512 * 2048)) +diff --git a/tests/test-cc.sh b/tests/test-cc.sh +index 2a719d314..5214327bd 100755 +--- a/tests/test-cc.sh ++++ b/tests/test-cc.sh +@@ -37,12 +37,7 @@ source ./functions.sh + set -e + set -x + +-if test "$SRCDIR" = ""; then +- echo "$0: \$SRCDIR is not set" +- exit 1 +-fi +- +-script=$SRCDIR/cc-shebang.c ++script=$abs_top_srcdir/tests/cc-shebang.c + if test ! -f "$script"; then + echo "$0: could not locate cc-shebang.c" + exit 1 +@@ -57,6 +52,6 @@ cleanup_fn rm -f $out + rm -f $out + + nbdkit -U - cc $script \ +- EXTRA_CFLAGS="-I$SRCDIR/../include" \ ++ EXTRA_CFLAGS="-I$abs_top_srcdir/include" \ + --run 'nbdinfo --size $uri' > $out + test "$(cat $out)" -eq $((100 * 1024 * 1024)) +diff --git a/tests/test-python-exception.sh b/tests/test-python-exception.sh +index a9fb146a6..54c29e66e 100755 +--- a/tests/test-python-exception.sh ++++ b/tests/test-python-exception.sh +@@ -34,23 +34,21 @@ source ./functions.sh + set -e + set -x + +-if test ! -d "$SRCDIR"; then +- echo "$0: could not locate python-exception.py" +- exit 1 +-fi +- + skip_if_valgrind "because Python code leaks memory" + ++script=$abs_top_srcdir/tests/python-exception.py ++test -f "$script" ++ + output=test-python-exception.out + rm -f $output + cleanup_fn rm -f $output + +-nbdkit -f -v python $SRCDIR/python-exception.py test=simple > $output 2>&1 ||: ++nbdkit -f -v python $script test=simple > $output 2>&1 ||: + cat $output + + grep 'this is the test string' $output + +-nbdkit -f -v python $SRCDIR/python-exception.py test=traceback > $output 2>&1 ||: ++nbdkit -f -v python $script test=traceback > $output 2>&1 ||: + cat $output + + grep 'raise_error1' $output +diff --git a/tests/test-python-export-list.sh b/tests/test-python-export-list.sh +index 49ca62e49..08774b9f3 100755 +--- a/tests/test-python-export-list.sh ++++ b/tests/test-python-export-list.sh +@@ -34,10 +34,8 @@ source ./functions.sh + set -e + set -x + +-if test ! -d "$SRCDIR"; then +- echo "$0: could not locate python-export-list.py" +- exit 1 +-fi ++script=$abs_top_srcdir/tests/python-export-list.py ++test -f "$script" + + skip_if_valgrind "because Python code leaks memory" + requires_nbdinfo +@@ -52,7 +50,7 @@ files="$pid $sock $out" + rm -f $files + cleanup_fn rm -f $files + +-start_nbdkit -P $pid -U $sock python $SRCDIR/python-export-list.py ++start_nbdkit -P $pid -U $sock python $script + + nbdinfo --list --json nbd+unix://\?socket=$sock > $out + cat $out +diff --git a/tests/test-python-export-name.sh b/tests/test-python-export-name.sh +index 99f292d2a..7680ae1e0 100755 +--- a/tests/test-python-export-name.sh ++++ b/tests/test-python-export-name.sh +@@ -34,10 +34,8 @@ source ./functions.sh + set -e + set -x + +-if test ! -d "$SRCDIR"; then +- echo "$0: could not locate python-export-name.py" +- exit 1 +-fi ++script=$abs_top_srcdir/tests/python-export-name.py ++test -f "$script" + + skip_if_valgrind "because Python code leaks memory" + requires nbdsh --version +@@ -48,7 +46,7 @@ files="$pid $sock" + rm -f $files + cleanup_fn rm -f $files + +-start_nbdkit -P $pid -U $sock python $SRCDIR/python-export-name.py ++start_nbdkit -P $pid -U $sock python $script + + # Try to read back various export names from the plugin. + for e in "" "test" "/" "//" " " "/ " "?" "ใƒ†ใ‚นใƒˆ" "-n" '\\' $'\n' "%%" \ +diff --git a/tests/test-python-thread-model.sh b/tests/test-python-thread-model.sh +index 69b3d5c76..a0dea2386 100755 +--- a/tests/test-python-thread-model.sh ++++ b/tests/test-python-thread-model.sh +@@ -34,22 +34,19 @@ source ./functions.sh + set -e + set -x + +-SCRIPT="$SRCDIR/python-thread-model.py" +-if ! test -d "$SRCDIR" || ! test -f "$SCRIPT"; then +- echo "$0: could not locate python-thread-model.py" +- exit 1 +-fi ++script="$abs_top_srcdir/tests/python-thread-model.py" ++test -f "$script" + + skip_if_valgrind "because Python code leaks memory" + requires nbdsh --version + + # Check the plugin is loadable. +-nbdkit python $SCRIPT --dump-plugin ++nbdkit python $script --dump-plugin + + # This test only works on modern Linux (with pipe2, accept4 etc) where + # we are able to issue parallel requests. Other platforms have more + # restrictive thread models. +-requires sh -c "nbdkit python $SCRIPT --dump-plugin | ++requires sh -c "nbdkit python $script --dump-plugin | + grep '^thread_model=parallel'" + + pid=test-python-thread-model.pid +@@ -58,7 +55,7 @@ files="$out $pid $sock" + rm -f $files + cleanup_fn rm -f $files + +-start_nbdkit -P $pid -U $sock python $SCRIPT ++start_nbdkit -P $pid -U $sock python $script + + export sock + nbdsh -c ' +diff --git a/tests/test-read-password-interactive.sh b/tests/test-read-password-interactive.sh +index 7af48492b..ee345872e 100755 +--- a/tests/test-read-password-interactive.sh ++++ b/tests/test-read-password-interactive.sh +@@ -36,7 +36,7 @@ set -x + + # This is an executable C script using nbdkit-cc-plugin. + requires_plugin cc +-plugin=$SRCDIR/test-read-password-plugin.c ++plugin=$abs_top_srcdir/tests/test-read-password-plugin.c + requires test -x $plugin + + # expect on macOS does not work for unclear reasons, skip it. +diff --git a/tests/test-read-password.sh b/tests/test-read-password.sh +index 234162bc3..d0033dceb 100755 +--- a/tests/test-read-password.sh ++++ b/tests/test-read-password.sh +@@ -36,7 +36,7 @@ set -x + + # This is an executable C script using nbdkit-cc-plugin. + requires_plugin cc +-plugin=$SRCDIR/test-read-password-plugin.c ++plugin=$abs_top_srcdir/tests/test-read-password-plugin.c + requires test -x $plugin + + # Since we are matching on error messages. +diff --git a/tests/test-shebang-cc-ocaml.sh b/tests/test-shebang-cc-ocaml.sh +index 41e2f3db5..9eebcf07c 100755 +--- a/tests/test-shebang-cc-ocaml.sh ++++ b/tests/test-shebang-cc-ocaml.sh +@@ -32,8 +32,7 @@ + + source ./functions.sh + +-script=$SRCDIR/cc_shebang.ml +- ++script=$abs_top_srcdir/tests/cc_shebang.ml + if test ! -f "$script"; then + echo "$0: could not locate cc_shebang.ml" + exit 1 +diff --git a/tests/test-shebang-cc.sh b/tests/test-shebang-cc.sh +index 6ae75abed..a5e557648 100755 +--- a/tests/test-shebang-cc.sh ++++ b/tests/test-shebang-cc.sh +@@ -35,8 +35,7 @@ + + source ./functions.sh + +-script=$SRCDIR/cc-shebang.c +- ++script=$abs_top_srcdir/tests/cc-shebang.c + if test ! -f "$script"; then + echo "$0: could not locate cc-shebang.c" + exit 1 +@@ -46,7 +45,7 @@ requires_plugin cc + requires guestfish --version + + $script -fv -U - \ +- EXTRA_CFLAGS="-I$SRCDIR/../include" \ ++ EXTRA_CFLAGS="-I$abs_top_srcdir/include" \ + --run ' + guestfish \ + add "" protocol:nbd server:unix:$unixsocket : \ +diff --git a/tests/test-shebang-perl.sh b/tests/test-shebang-perl.sh +index 5f35d75b4..65bb7a403 100755 +--- a/tests/test-shebang-perl.sh ++++ b/tests/test-shebang-perl.sh +@@ -34,9 +34,9 @@ source ./functions.sh + + pidfile=shebang-perl.pid + sock=$(mktemp -u /tmp/nbdkit-test-sock.XXXXXX) +-script=$SRCDIR/shebang.pl ++script=$abs_top_srcdir/tests/shebang.pl + +-if test ! -d "$SRCDIR"; then ++if test ! -f "$script"; then + echo "$0: could not locate shebang.pl" + exit 1 + fi +diff --git a/tests/test-shebang-python.sh b/tests/test-shebang-python.sh +index fb851de94..8674f2fa1 100755 +--- a/tests/test-shebang-python.sh ++++ b/tests/test-shebang-python.sh +@@ -34,9 +34,9 @@ source ./functions.sh + + pidfile=shebang-python.pid + sock=$(mktemp -u /tmp/nbdkit-test-sock.XXXXXX) +-script=$SRCDIR/shebang.py ++script=$abs_top_srcdir/tests/shebang.py + +-if test ! -d "$SRCDIR"; then ++if test ! -f "$script"; then + echo "$0: could not locate shebang.py" + exit 1 + fi +-- +2.41.0 + diff --git a/0013-tests-test-read-password-Remove-use-of-SRCDIR.patch b/0013-tests-test-read-password-Remove-use-of-SRCDIR.patch new file mode 100644 index 0000000..6ef088e --- /dev/null +++ b/0013-tests-test-read-password-Remove-use-of-SRCDIR.patch @@ -0,0 +1,55 @@ +From 92ae65f93d8ebeb09500552d7ebfc1e8149ded6f Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Thu, 22 Jun 2023 10:48:51 +0100 +Subject: [PATCH 13/15] tests/test-read-password*: Remove use of $SRCDIR + +These tests required exporting $abs_top_srcdir so that the plugin +(which is a C script) can use it. +--- + tests/test-read-password-interactive.sh | 3 +++ + tests/test-read-password.sh | 3 +++ + tests/test-read-password-plugin.c | 2 +- + 3 files changed, 7 insertions(+), 1 deletion(-) + +diff --git a/tests/test-read-password-interactive.sh b/tests/test-read-password-interactive.sh +index ee345872e..2738922f4 100755 +--- a/tests/test-read-password-interactive.sh ++++ b/tests/test-read-password-interactive.sh +@@ -39,6 +39,9 @@ requires_plugin cc + plugin=$abs_top_srcdir/tests/test-read-password-plugin.c + requires test -x $plugin + ++# This variable is used by $plugin. ++export abs_top_srcdir ++ + # expect on macOS does not work for unclear reasons, skip it. + requires_not test "$(uname)" = "Darwin" + +diff --git a/tests/test-read-password.sh b/tests/test-read-password.sh +index d0033dceb..8243d5491 100755 +--- a/tests/test-read-password.sh ++++ b/tests/test-read-password.sh +@@ -39,6 +39,9 @@ requires_plugin cc + plugin=$abs_top_srcdir/tests/test-read-password-plugin.c + requires test -x $plugin + ++# This variable is used by $plugin. ++export abs_top_srcdir ++ + # Since we are matching on error messages. + export LANG=C + +diff --git a/tests/test-read-password-plugin.c b/tests/test-read-password-plugin.c +index d8faea4e3..397f69d72 100755 +--- a/tests/test-read-password-plugin.c ++++ b/tests/test-read-password-plugin.c +@@ -1,5 +1,5 @@ + #if 0 +-exec nbdkit cc "$0" "$@" EXTRA_CFLAGS="-I.. -I${SRCDIR:-.}/../include" ++exec nbdkit cc "$0" "$@" EXTRA_CFLAGS="-I.. -I$abs_top_srcdir/include" + #endif + /* nbdkit + * Copyright Red Hat +-- +2.41.0 + diff --git a/0014-tests-Stop-setting-SRCDIR-for-tests.patch b/0014-tests-Stop-setting-SRCDIR-for-tests.patch new file mode 100644 index 0000000..d61725d --- /dev/null +++ b/0014-tests-Stop-setting-SRCDIR-for-tests.patch @@ -0,0 +1,28 @@ +From 2e3dcb2a720e4258b54d04ffd081c5159783df8e Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Thu, 22 Jun 2023 10:50:17 +0100 +Subject: [PATCH 14/15] tests: Stop setting $SRCDIR for tests + +Since no tests use this variable any longer we can stop setting it. + +However note that make-psk.sh still uses the same-named (but not +connected) variable. +--- + tests/Makefile.am | 1 - + 1 file changed, 1 deletion(-) + +diff --git a/tests/Makefile.am b/tests/Makefile.am +index 32ebb7002..2921076de 100644 +--- a/tests/Makefile.am ++++ b/tests/Makefile.am +@@ -63,7 +63,6 @@ EXTRA_PROGRAMS = + # Use the 'direct' backend, and ensure maximum libguestfs debugging. + # Enable libnbd debugging. + TESTS_ENVIRONMENT = \ +- SRCDIR=$(srcdir) \ + LIBGUESTFS_ATTACH_METHOD=appliance \ + LIBGUESTFS_DEBUG=1 \ + LIBGUESTFS_TRACE=1 \ +-- +2.41.0 + diff --git a/0015-tests-Explicitly-add-L.-plugins-ocaml-.libs-to-find-.patch b/0015-tests-Explicitly-add-L.-plugins-ocaml-.libs-to-find-.patch new file mode 100644 index 0000000..ca09a13 --- /dev/null +++ b/0015-tests-Explicitly-add-L.-plugins-ocaml-.libs-to-find-.patch @@ -0,0 +1,44 @@ +From 3e4c1b79a72970c17cb42b21070e61ec634a38bb Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Thu, 22 Jun 2023 11:15:27 +0100 +Subject: [PATCH 15/15] tests: Explicitly add -L../plugins/ocaml/.libs to find + -lnbdkitocaml + +The actual libnbdkitocaml.so library is located in +plugins/ocaml/.libs, but we were only specifying plugins/ocaml as the +search path. How did it ever work? Turns out using LD_LIBRARY_PATH +set in wrapper.c. It seems unreliable to rely on this, so set the +path properly, as we are already doing elsewhere. +--- + tests/test-cc-ocaml.sh | 2 +- + tests/cc_shebang.ml | 2 +- + 2 files changed, 2 insertions(+), 2 deletions(-) + +diff --git a/tests/test-cc-ocaml.sh b/tests/test-cc-ocaml.sh +index db7d252cf..97a1c89e3 100755 +--- a/tests/test-cc-ocaml.sh ++++ b/tests/test-cc-ocaml.sh +@@ -56,6 +56,6 @@ cleanup_fn rm -f $out + rm -f $out + + nbdkit -v -U - cc $script a=1 b=2 c=3 d=4 \ +- CC="$OCAMLOPT" CFLAGS="-output-obj -runtime-variant _pic -I $abs_top_srcdir/plugins/ocaml -I +unix unix.cmxa -I +threads threads.cmxa NBDKit.cmx -cclib -lnbdkitocaml" \ ++ CC="$OCAMLOPT" CFLAGS="-output-obj -runtime-variant _pic -I $abs_top_srcdir/plugins/ocaml -I +unix unix.cmxa -I +threads threads.cmxa NBDKit.cmx -cclib -L../plugins/ocaml/.libs -cclib -lnbdkitocaml" \ + --run 'nbdinfo --size $uri' > $out + test "$(cat $out)" -eq $((512 * 2048)) +diff --git a/tests/cc_shebang.ml b/tests/cc_shebang.ml +index 05ca77b64..57ad85791 100755 +--- a/tests/cc_shebang.ml ++++ b/tests/cc_shebang.ml +@@ -4,7 +4,7 @@ + # shell as an impossible command which is ignored. The line below is + # run by the shell and ignored by OCaml. + +-exec nbdkit cc "$0" CC=ocamlopt CFLAGS="-output-obj -runtime-variant _pic -I +unix unix.cmxa -I +threads threads.cmxa NBDKit.cmx -cclib -lnbdkitocaml" "$@" ++exec nbdkit cc "$0" CC=ocamlopt CFLAGS="-output-obj -runtime-variant _pic -I +unix unix.cmxa -I +threads threads.cmxa NBDKit.cmx -cclib -L../plugins/ocaml/.libs -cclib -lnbdkitocaml" "$@" + *) + + open Printf +-- +2.41.0 + diff --git a/nbdkit.spec b/nbdkit.spec index e7dcb76..9e6c161 100644 --- a/nbdkit.spec +++ b/nbdkit.spec @@ -45,7 +45,7 @@ ExclusiveArch: x86_64 %global verify_tarball_signature 1 # If there are patches which touch autotools files, set this to 1. -%global patches_touch_autotools %{nil} +%global patches_touch_autotools 1 # The source directory. %global source_directory 1.35-development @@ -78,6 +78,24 @@ Source3: copy-patches.sh Source4: nbdkit.attr Source5: nbdkit-find-provides +# All upstream patches since 1.35.5 was released. These are needed +# to fix OCaml 5 builds. +Patch: 0001-tests-test-tar-info-Remove-bogus-test-requires-of-gu.patch +Patch: 0002-tar-Document-the-optional-tar-option-in-help-output-.patch +Patch: 0003-tar-Implement-tar-limit.patch +Patch: 0004-xz-Fix-error-message-to-refer-to-xz-max-block-parame.patch +Patch: 0005-configure-Recommend-using-g-with-OCAMLOPTFLAGS.patch +Patch: 0006-tests-test_ocaml_plugin.ml-Print-a-message-when-test.patch +Patch: 0007-ocaml-Add-I-unix-before-using-unix.cmxa.patch +Patch: 0008-ocaml-Replace-caml_leave_blocking_section-with-caml_.patch +Patch: 0009-ocaml-Always-unregister-the-global-root-and-free-the.patch +Patch: 0010-ocaml-Fix-thread-registration-for-OCaml-5.patch +Patch: 0011-tests-test-cc-ocaml.sh-Use-nbdkit-v-option-to-help-w.patch +Patch: 0012-tests-Replace-SRCDIR-with-abs_top_srcdir.patch +Patch: 0013-tests-test-read-password-Remove-use-of-SRCDIR.patch +Patch: 0014-tests-Stop-setting-SRCDIR-for-tests.patch +Patch: 0015-tests-Explicitly-add-L.-plugins-ocaml-.libs-to-find-.patch + BuildRequires: make %if 0%{patches_touch_autotools} BuildRequires: autoconf, automake, libtool