From 4144ae2cf89b1dcbcec29a8514dd15f2f4d41c93 Mon Sep 17 00:00:00 2001 From: "Richard W.M. Jones" Date: Wed, 6 Jul 2016 14:20:33 +0100 Subject: [PATCH] Add all upstream patches since 5.1.16 was released. --- ...dd-support-for-a-DAX-root-filesystem.patch | 2 +- ...ate-modules-on-the-stack-RHBZ-133969.patch | 42 ++++ 0003-init-Print-size-of-init.patch | 25 +++ ...ramfs-files-before-chrooting-into-th.patch | 100 ++++++++++ ...whole-files-into-memory-when-copying.patch | 186 ++++++++++++++++++ ...lude-packagelist-Use-supermin-tmpdir.patch | 41 ++++ ...-instead-of-tmp-if-TMPDIR-is-not-set.patch | 34 ++++ supermin.spec | 16 +- 8 files changed, 441 insertions(+), 5 deletions(-) create mode 100644 0002-init-Don-t-allocate-modules-on-the-stack-RHBZ-133969.patch create mode 100644 0003-init-Print-size-of-init.patch create mode 100644 0004-init-Delete-initramfs-files-before-chrooting-into-th.patch create mode 100644 0005-ext2-Don-t-load-whole-files-into-memory-when-copying.patch create mode 100644 0006-build-include-packagelist-Use-supermin-tmpdir.patch create mode 100644 0007-Use-var-tmp-instead-of-tmp-if-TMPDIR-is-not-set.patch diff --git a/0001-Add-support-for-a-DAX-root-filesystem.patch b/0001-Add-support-for-a-DAX-root-filesystem.patch index ea41364..e50d3ee 100644 --- a/0001-Add-support-for-a-DAX-root-filesystem.patch +++ b/0001-Add-support-for-a-DAX-root-filesystem.patch @@ -1,7 +1,7 @@ From b4131cefc55ae41862a426bed83aa87c8362866b Mon Sep 17 00:00:00 2001 From: "Richard W.M. Jones" Date: Mon, 16 May 2016 18:55:33 +0100 -Subject: [PATCH] Add support for a DAX root filesystem. +Subject: [PATCH 1/7] Add support for a DAX root filesystem. With these changes you can provide an ext4 root filesystem on a virtual NVDIMM. The filesystem will be mounted using DAX, so files diff --git a/0002-init-Don-t-allocate-modules-on-the-stack-RHBZ-133969.patch b/0002-init-Don-t-allocate-modules-on-the-stack-RHBZ-133969.patch new file mode 100644 index 0000000..af09d11 --- /dev/null +++ b/0002-init-Don-t-allocate-modules-on-the-stack-RHBZ-133969.patch @@ -0,0 +1,42 @@ +From 557952500a10513120e90e5118c53030363a778e Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Wed, 25 May 2016 17:29:16 +0100 +Subject: [PATCH 2/7] init: Don't allocate modules on the stack (RHBZ#1339691). + +If the modules are unstripped and/or especially large, then the stack +can overflow. + +Thanks: Luiz Capitulino for testing. +--- + init/init.c | 8 +++++++- + 1 file changed, 7 insertions(+), 1 deletion(-) + +diff --git a/init/init.c b/init/init.c +index 106be02..733d66e 100644 +--- a/init/init.c ++++ b/init/init.c +@@ -314,7 +314,11 @@ insmod (const char *filename) + exit (EXIT_FAILURE); + } + size = st.st_size; +- char buf[size]; ++ char *buf = malloc (size); ++ if (buf == NULL) { ++ fprintf (stderr, "insmod: malloc (%s, %zu bytes): %m\n", filename, size); ++ exit (EXIT_FAILURE); ++ } + size_t offset = 0; + do { + ssize_t rc = read (fd, buf + offset, size - offset); +@@ -332,6 +336,8 @@ insmod (const char *filename) + * of a missing device. + */ + } ++ ++ free (buf); + } + + /* Mount /proc unless it's mounted already. */ +-- +2.7.4 + diff --git a/0003-init-Print-size-of-init.patch b/0003-init-Print-size-of-init.patch new file mode 100644 index 0000000..6d6d33f --- /dev/null +++ b/0003-init-Print-size-of-init.patch @@ -0,0 +1,25 @@ +From 272c35b435dd6a71c79f90626567bdddebb72b41 Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Tue, 14 Jun 2016 16:06:24 +0100 +Subject: [PATCH 3/7] init: Print size of 'init'. + +Useful for seeing in the build logs if the init program grows in size. +--- + src/Makefile.am | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/src/Makefile.am b/src/Makefile.am +index 443e25d..11adf31 100644 +--- a/src/Makefile.am ++++ b/src/Makefile.am +@@ -143,6 +143,7 @@ ext2init-bin.o: ext2init-bin.S + + ext2init-bin.S: ../init/init $(srcdir)/bin2s.pl + strip --strip-all $< ++ ls -l $< + @file $< | grep -isq static || \ + (echo "*** error: init is not statically linked"; exit 1) + $(srcdir)/bin2s.pl $< $@ +-- +2.7.4 + diff --git a/0004-init-Delete-initramfs-files-before-chrooting-into-th.patch b/0004-init-Delete-initramfs-files-before-chrooting-into-th.patch new file mode 100644 index 0000000..84c5d74 --- /dev/null +++ b/0004-init-Delete-initramfs-files-before-chrooting-into-th.patch @@ -0,0 +1,100 @@ +From a67c0c5632fe6f68c02b0b5623217782b094c00e Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Tue, 14 Jun 2016 16:07:29 +0100 +Subject: [PATCH 4/7] init: Delete initramfs files before chrooting into the + appliance. + +After supermin has finished running, the initramfs files sit around +occupying non-swappable memory but serving no further purpose. + +This saves a little memory, at the cost of about 1ms of extra boot +time. +--- + init/init.c | 50 ++++++++++++++++++++++++++++++++++++++++++++++++-- + 1 file changed, 48 insertions(+), 2 deletions(-) + +diff --git a/init/init.c b/init/init.c +index 733d66e..5ac53e9 100644 +--- a/init/init.c ++++ b/init/init.c +@@ -1,5 +1,5 @@ + /* supermin-helper reimplementation in C. +- * Copyright (C) 2009-2014 Red Hat Inc. ++ * Copyright (C) 2009-2016 Red Hat Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by +@@ -80,6 +80,7 @@ static void mount_proc (void); + static void print_uptime (void); + static void read_cmdline (void); + static void insmod (const char *filename); ++static void delete_initramfs_files (void); + static void show_directory (const char *dir); + + static char cmdline[1024]; +@@ -264,9 +265,12 @@ main () + exit (EXIT_FAILURE); + } + ++ if (!quiet) ++ fprintf (stderr, "supermin: deleting initramfs files\n"); ++ delete_initramfs_files (); ++ + /* Note that pivot_root won't work. See the note in + * Documentation/filesystems/ramfs-rootfs-initramfs.txt +- * We could remove the old initramfs files, but let's not bother. + */ + if (!quiet) + fprintf (stderr, "supermin: chroot\n"); +@@ -396,6 +400,48 @@ read_cmdline (void) + cmdline[len-1] = '\0'; + } + ++/* By deleting the files in the initramfs before we chroot, we save a ++ * little bit of memory (or quite a lot of memory if the user is using ++ * unstripped kmods). ++ * ++ * We only delete files in the root directory. We don't delete ++ * directories because they only take a tiny amount of space and ++ * because we must not delete any mountpoints, especially not /root ++ * where we are about to chroot. ++ * ++ * We don't recursively look for files because that would be too ++ * complex and risky, and the normal supermin initramfs doesn't have ++ * any files except in the root directory. ++ */ ++static void ++delete_initramfs_files (void) ++{ ++ DIR *dir; ++ struct dirent *d; ++ struct stat statbuf; ++ ++ if (chdir ("/") == -1) { ++ perror ("chdir: /"); ++ return; ++ } ++ ++ dir = opendir ("."); ++ if (!dir) { ++ perror ("opendir: /"); ++ return; ++ } ++ ++ while ((d = readdir (dir)) != NULL) { ++ /* "." and ".." are directories, so the S_ISREG test ignores them. */ ++ if (lstat (d->d_name, &statbuf) >= 0 && S_ISREG (statbuf.st_mode)) { ++ if (unlink (d->d_name) == -1) ++ perror (d->d_name); ++ } ++ } ++ ++ closedir (dir); ++} ++ + /* Display a directory on stderr. This is used for debugging only. */ + static char + dirtype (int dt) +-- +2.7.4 + diff --git a/0005-ext2-Don-t-load-whole-files-into-memory-when-copying.patch b/0005-ext2-Don-t-load-whole-files-into-memory-when-copying.patch new file mode 100644 index 0000000..eddfc05 --- /dev/null +++ b/0005-ext2-Don-t-load-whole-files-into-memory-when-copying.patch @@ -0,0 +1,186 @@ +From f5a3deef588759c49dc862427a265aced612002d Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Wed, 6 Jul 2016 12:12:10 +0100 +Subject: [PATCH 5/7] ext2: Don't load whole files into memory when copying to + the appliance (RHBZ#1113065). + +Obviously for very large files this is going to be a problem, as well +as not being very cache efficient. + +libext2fs can handle writes to parts of files just fine so copy files +in blocks. + +Also demote the "Permission denied" error to a warning, and add some +explanatory text telling people not to use sudo. +--- + src/ext2fs-c.c | 127 +++++++++++++++++++++++++++++++++++---------------------- + 1 file changed, 78 insertions(+), 49 deletions(-) + +diff --git a/src/ext2fs-c.c b/src/ext2fs-c.c +index cb9282b..96a3de0 100644 +--- a/src/ext2fs-c.c ++++ b/src/ext2fs-c.c +@@ -185,6 +185,7 @@ supermin_ext2fs_read_bitmaps (value fsv) + static void ext2_mkdir (ext2_filsys fs, ext2_ino_t dir_ino, const char *dirname, const char *basename, mode_t mode, uid_t uid, gid_t gid, time_t ctime, time_t atime, time_t mtime); + static void ext2_empty_inode (ext2_filsys fs, ext2_ino_t dir_ino, const char *dirname, const char *basename, mode_t mode, uid_t uid, gid_t gid, time_t ctime, time_t atime, time_t mtime, int major, int minor, int dir_ft, ext2_ino_t *ino_ret); + static void ext2_write_file (ext2_filsys fs, ext2_ino_t ino, const char *buf, size_t size, const char *filename); ++static void ext2_write_host_file (ext2_filsys fs, ext2_ino_t ino, const char *src, const char *filename); + static void ext2_link (ext2_filsys fs, ext2_ino_t dir_ino, const char *basename, ext2_ino_t ino, int dir_ft); + static void ext2_clean_path (ext2_filsys fs, ext2_ino_t dir_ino, const char *dirname, const char *basename, int isdir); + static void ext2_copy_file (struct ext2_data *data, const char *src, const char *dest); +@@ -500,6 +501,81 @@ ext2_write_file (ext2_filsys fs, + ext2_error_to_exception ("ext2fs_write_inode", err, filename); + } + ++/* Same as ext2_write_file, but it copies the file contents from the ++ * host. You must create the file first with ext2_empty_inode, and ++ * the host file must be a regular file. ++ */ ++static void ++ext2_write_host_file (ext2_filsys fs, ++ ext2_ino_t ino, ++ const char *src, /* source (host) file */ ++ const char *filename) ++{ ++ int fd; ++ char buf[BUFSIZ]; ++ ssize_t r; ++ size_t size = 0; ++ errcode_t err; ++ ext2_file_t file; ++ unsigned int written; ++ ++ fd = open (src, O_RDONLY); ++ if (fd == -1) { ++ static int warned = 0; ++ ++ /* We skip unreadable files. However if the error is -EACCES then ++ * modify the message so as not to frighten the horses. ++ */ ++ fprintf (stderr, "supermin: warning: %s: %m (ignored)\n", filename); ++ if (errno == EACCES && !warned) { ++ fprintf (stderr, ++ "Some distro files are not public readable, so supermin cannot copy them\n" ++ "into the appliance. This is a problem with your Linux distro. Please ask\n" ++ "your distro to stop doing pointless security by obscurity.\n" ++ "You can ignore these warnings. You *do not* need to use sudo.\n"); ++ warned = 1; ++ } ++ return; ++ } ++ ++ err = ext2fs_file_open2 (fs, ino, NULL, EXT2_FILE_WRITE, &file); ++ if (err != 0) ++ ext2_error_to_exception ("ext2fs_file_open2", err, filename); ++ ++ while ((r = read (fd, buf, sizeof buf)) > 0) { ++ err = ext2fs_file_write (file, buf, r, &written); ++ if (err != 0) ++ ext2_error_to_exception ("ext2fs_file_open2", err, filename); ++ if ((ssize_t) written != r) ++ caml_failwith ("ext2fs_file_write: requested write size != bytes written"); ++ size += written; ++ } ++ ++ if (r == -1) ++ unix_error (errno, (char *) "read", caml_copy_string (filename)); ++ ++ if (close (fd) == -1) ++ unix_error (errno, (char *) "close", caml_copy_string (filename)); ++ ++ /* Flush out the ext2 file. */ ++ err = ext2fs_file_flush (file); ++ if (err != 0) ++ ext2_error_to_exception ("ext2fs_file_flush", err, filename); ++ err = ext2fs_file_close (file); ++ if (err != 0) ++ ext2_error_to_exception ("ext2fs_file_close", err, filename); ++ ++ /* Update the true size in the inode. */ ++ struct ext2_inode inode; ++ err = ext2fs_read_inode (fs, ino, &inode); ++ if (err != 0) ++ ext2_error_to_exception ("ext2fs_read_inode", err, filename); ++ inode.i_size = size; ++ err = ext2fs_write_inode (fs, ino, &inode); ++ if (err != 0) ++ ext2_error_to_exception ("ext2fs_write_inode", err, filename); ++} ++ + /* This is just a wrapper around ext2fs_link which calls + * ext2fs_expand_dir as necessary if the directory fills up. See + * definition of expand_dir in the sources of debugfs. +@@ -589,43 +665,6 @@ ext2_clean_path (ext2_filsys fs, ext2_ino_t dir_ino, + /* else it's a directory, what to do? XXX */ + } + +-/* Read in the whole file into memory. Check the size is still 'size'. */ +-static char * +-read_whole_file (const char *filename, size_t size) +-{ +- char *buf = malloc (size); +- if (buf == NULL) +- caml_raise_out_of_memory (); +- +- int fd = open (filename, O_RDONLY); +- if (fd == -1) { +- /* Skip unreadable files. */ +- fprintf (stderr, "supermin: open: %s: %m\n", filename); +- free (buf); +- return NULL; +- } +- +- size_t n = 0; +- char *p = buf; +- +- while (n < size) { +- ssize_t r = read (fd, p, size - n); +- if (r == -1) +- unix_error (errno, (char *) "read", caml_copy_string (filename)); +- if (r == 0) { +- fprintf (stderr, "supermin: end of file reading '%s'\n", filename); +- caml_invalid_argument ("ext2fs: file has changed size unexpectedly"); +- } +- n += r; +- p += r; +- } +- +- if (close (fd) == -1) +- unix_error (errno, (char *) "close", caml_copy_string (filename)); +- +- return buf; +-} +- + /* Copy a file (or directory etc) from the host. */ + static void + ext2_copy_file (struct ext2_data *data, const char *src, const char *dest) +@@ -766,24 +805,14 @@ ext2_copy_file (struct ext2_data *data, const char *src, const char *dest) + if (S_ISREG (statbuf.st_mode)) { + /* XXX Hard links get duplicated here. */ + ext2_ino_t ino; +- char *buf = NULL; +- +- if (statbuf.st_size > 0) { +- buf = read_whole_file (src, statbuf.st_size); +- if (buf == NULL) +- goto skip_unreadable_file; +- } + + ext2_empty_inode (data->fs, dir_ino, dirname, basename, + statbuf.st_mode, statbuf.st_uid, statbuf.st_gid, + statbuf.st_ctime, statbuf.st_atime, statbuf.st_mtime, + 0, 0, EXT2_FT_REG_FILE, &ino); + +- if (statbuf.st_size > 0) { +- ext2_write_file (data->fs, ino, buf, statbuf.st_size, dest); +- free (buf); +- } +- skip_unreadable_file: ; ++ if (statbuf.st_size > 0) ++ ext2_write_host_file (data->fs, ino, src, dest); + } + /* Create a symlink. */ + else if (S_ISLNK (statbuf.st_mode)) { +-- +2.7.4 + diff --git a/0006-build-include-packagelist-Use-supermin-tmpdir.patch b/0006-build-include-packagelist-Use-supermin-tmpdir.patch new file mode 100644 index 0000000..bbee2c8 --- /dev/null +++ b/0006-build-include-packagelist-Use-supermin-tmpdir.patch @@ -0,0 +1,41 @@ +From bc5f01196ebf181b20d43fefe23f7003a5d3aa7c Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Wed, 6 Jul 2016 14:01:16 +0100 +Subject: [PATCH 6/7] build: --include-packagelist: Use supermin tmpdir. + +Fixes commit 535c2cfbf2c8e8cbe5f43dd9e9a0eea8eacb1bed. +--- + src/build.ml | 9 ++------- + 1 file changed, 2 insertions(+), 7 deletions(-) + +diff --git a/src/build.ml b/src/build.ml +index e34ec5f..83a0d98 100644 +--- a/src/build.ml ++++ b/src/build.ml +@@ -207,9 +207,9 @@ let rec build debug + (* Create a temporary file for packagelist, if requested. *) + let packagelist_file = + if include_packagelist then ( +- let filename, chan = Filename.open_temp_file "packagelist." "" in ++ let filename = tmpdir // "packagelist" in ++ let chan = open_out filename in + List.iter (fprintf chan "%s\n") pretty_packages; +- flush chan; + close_out chan; + Some filename + ) else None in +@@ -230,11 +230,6 @@ let rec build debug + Ext2.build_ext2 debug basedir files modpath kernel_version appliance size + packagelist_file; + Ext2_initrd.build_initrd debug tmpdir modpath initrd +- ); +- +- (match packagelist_file with +- | None -> () +- | Some filename -> Sys.remove filename + ) + + and read_appliance debug basedir appliance = function +-- +2.7.4 + diff --git a/0007-Use-var-tmp-instead-of-tmp-if-TMPDIR-is-not-set.patch b/0007-Use-var-tmp-instead-of-tmp-if-TMPDIR-is-not-set.patch new file mode 100644 index 0000000..8a0a701 --- /dev/null +++ b/0007-Use-var-tmp-instead-of-tmp-if-TMPDIR-is-not-set.patch @@ -0,0 +1,34 @@ +From 12fc339efb1d287593b53da1dfddc88f876f8e7d Mon Sep 17 00:00:00 2001 +From: "Richard W.M. Jones" +Date: Wed, 6 Jul 2016 14:12:28 +0100 +Subject: [PATCH 7/7] Use /var/tmp instead of /tmp if TMPDIR is not set. + +See: https://bugzilla.redhat.com/show_bug.cgi?id=1316479#c12 + +Thanks: Junqin Zhou +--- + src/supermin.ml | 7 +++++-- + 1 file changed, 5 insertions(+), 2 deletions(-) + +diff --git a/src/supermin.ml b/src/supermin.ml +index b0532e5..a09b498 100644 +--- a/src/supermin.ml ++++ b/src/supermin.ml +@@ -57,9 +57,12 @@ let main () = + if try Filename.is_relative (getenv "TMPDIR") with Not_found -> false then + error "error: environment variable $TMPDIR must be an absolute path"; + +- (* Create a temporary directory for scratch storage. *) ++ (* Create a temporary directory for scratch storage. Because it's ++ * for large files, use /var/tmp if TMPDIR is not set. ++ *) + let tmpdir = +- let tmpdir = Filename.temp_file "supermin" ".tmpdir" in ++ let temp_dir = try getenv "TMPDIR" with Not_found -> "/var/tmp" in ++ let tmpdir = Filename.temp_file ~temp_dir "supermin" ".tmpdir" in + unlink tmpdir; + mkdir tmpdir 0o700; + at_exit +-- +2.7.4 + diff --git a/supermin.spec b/supermin.spec index 268ca90..948f368 100644 --- a/supermin.spec +++ b/supermin.spec @@ -23,7 +23,7 @@ Summary: Tool for creating supermin appliances Name: supermin Version: 5.1.16 -Release: 3%{?dist} +Release: 4%{?dist} License: GPLv2+ %if 0%{?rhel} >= 7 @@ -33,9 +33,14 @@ ExclusiveArch: x86_64 URL: http://people.redhat.com/~rjones/supermin/ Source0: http://libguestfs.org/download/supermin/%{name}-%{version}.tar.gz -# Add support for DAX root filesystem using vNVDIMM. -# Upstream commit b4131cefc55ae41862a426bed83aa87c8362866b. -Patch0001: 0001-Add-support-for-a-DAX-root-filesystem.patch +# All upstream patches since 5.1.16 was released. +Patch1: 0001-Add-support-for-a-DAX-root-filesystem.patch +Patch2: 0002-init-Don-t-allocate-modules-on-the-stack-RHBZ-133969.patch +Patch3: 0003-init-Print-size-of-init.patch +Patch4: 0004-init-Delete-initramfs-files-before-chrooting-into-th.patch +Patch5: 0005-ext2-Don-t-load-whole-files-into-memory-when-copying.patch +Patch6: 0006-build-include-packagelist-Use-supermin-tmpdir.patch +Patch7: 0007-Use-var-tmp-instead-of-tmp-if-TMPDIR-is-not-set.patch BuildRequires: /usr/bin/pod2man BuildRequires: /usr/bin/pod2html @@ -159,6 +164,9 @@ make check || { %changelog +* Wed Jul 06 2016 Richard W.M. Jones - 5.1.16-4 +- Add all upstream patches since 5.1.16 was released. + * Tue May 17 2016 Richard W.M. Jones - 5.1.16-3 - Add upstream patch for DAX / vNVDIMM support.