None of the old patches are needed any more
This commit is contained in:
parent
2b6677ffdf
commit
aab9aed384
@ -1,413 +0,0 @@
|
||||
From 6113361316d5ce5bfdc118d188e5617a1fcd747c Mon Sep 17 00:00:00 2001
|
||||
From: Sean Purcell <me@seanp.xyz>
|
||||
Date: Mon, 14 Aug 2017 22:46:04 -0700
|
||||
Subject: [PATCH 1/4] squashfs-tools: Add zstd support
|
||||
|
||||
This patch adds zstd support to squashfs-tools. It works with zstd
|
||||
versions >= 1.0.0. It was originally written by Sean Purcell.
|
||||
|
||||
Signed-off-by: Sean Purcell <me@seanp.xyz>
|
||||
Signed-off-by: Nick Terrell <terrelln@fb.com>
|
||||
---
|
||||
squashfs-tools/Makefile | 20 +++
|
||||
squashfs-tools/compressor.c | 8 ++
|
||||
squashfs-tools/squashfs_fs.h | 1 +
|
||||
squashfs-tools/zstd_wrapper.c | 254 ++++++++++++++++++++++++++++++++++
|
||||
squashfs-tools/zstd_wrapper.h | 48 +++++++
|
||||
5 files changed, 331 insertions(+)
|
||||
create mode 100644 squashfs-tools/zstd_wrapper.c
|
||||
create mode 100644 squashfs-tools/zstd_wrapper.h
|
||||
|
||||
diff --git a/squashfs-tools/Makefile b/squashfs-tools/Makefile
|
||||
index 52d2582..22fc559 100644
|
||||
--- a/squashfs-tools/Makefile
|
||||
+++ b/squashfs-tools/Makefile
|
||||
@@ -75,6 +75,18 @@ GZIP_SUPPORT = 1
|
||||
#LZMA_SUPPORT = 1
|
||||
#LZMA_DIR = ../../../../LZMA/lzma465
|
||||
|
||||
+
|
||||
+########### Building ZSTD support ############
|
||||
+#
|
||||
+# The ZSTD library is supported
|
||||
+# ZSTD homepage: http://zstd.net
|
||||
+# ZSTD source repository: https://github.com/facebook/zstd
|
||||
+#
|
||||
+# To build using the ZSTD library - install the library and uncomment the
|
||||
+# ZSTD_SUPPORT line below.
|
||||
+#
|
||||
+#ZSTD_SUPPORT = 1
|
||||
+
|
||||
######## Specifying default compression ########
|
||||
#
|
||||
# The next line specifies which compression algorithm is used by default
|
||||
@@ -177,6 +189,14 @@ LIBS += -llz4
|
||||
COMPRESSORS += lz4
|
||||
endif
|
||||
|
||||
+ifeq ($(ZSTD_SUPPORT),1)
|
||||
+CFLAGS += -DZSTD_SUPPORT
|
||||
+MKSQUASHFS_OBJS += zstd_wrapper.o
|
||||
+UNSQUASHFS_OBJS += zstd_wrapper.o
|
||||
+LIBS += -lzstd
|
||||
+COMPRESSORS += zstd
|
||||
+endif
|
||||
+
|
||||
ifeq ($(XATTR_SUPPORT),1)
|
||||
ifeq ($(XATTR_DEFAULT),1)
|
||||
CFLAGS += -DXATTR_SUPPORT -DXATTR_DEFAULT
|
||||
diff --git a/squashfs-tools/compressor.c b/squashfs-tools/compressor.c
|
||||
index 525e316..02b5e90 100644
|
||||
--- a/squashfs-tools/compressor.c
|
||||
+++ b/squashfs-tools/compressor.c
|
||||
@@ -65,6 +65,13 @@ static struct compressor xz_comp_ops = {
|
||||
extern struct compressor xz_comp_ops;
|
||||
#endif
|
||||
|
||||
+#ifndef ZSTD_SUPPORT
|
||||
+static struct compressor zstd_comp_ops = {
|
||||
+ ZSTD_COMPRESSION, "zstd"
|
||||
+};
|
||||
+#else
|
||||
+extern struct compressor zstd_comp_ops;
|
||||
+#endif
|
||||
|
||||
static struct compressor unknown_comp_ops = {
|
||||
0, "unknown"
|
||||
@@ -77,6 +84,7 @@ struct compressor *compressor[] = {
|
||||
&lzo_comp_ops,
|
||||
&lz4_comp_ops,
|
||||
&xz_comp_ops,
|
||||
+ &zstd_comp_ops,
|
||||
&unknown_comp_ops
|
||||
};
|
||||
|
||||
diff --git a/squashfs-tools/squashfs_fs.h b/squashfs-tools/squashfs_fs.h
|
||||
index 791fe12..afca918 100644
|
||||
--- a/squashfs-tools/squashfs_fs.h
|
||||
+++ b/squashfs-tools/squashfs_fs.h
|
||||
@@ -277,6 +277,7 @@ typedef long long squashfs_inode;
|
||||
#define LZO_COMPRESSION 3
|
||||
#define XZ_COMPRESSION 4
|
||||
#define LZ4_COMPRESSION 5
|
||||
+#define ZSTD_COMPRESSION 6
|
||||
|
||||
struct squashfs_super_block {
|
||||
unsigned int s_magic;
|
||||
diff --git a/squashfs-tools/zstd_wrapper.c b/squashfs-tools/zstd_wrapper.c
|
||||
new file mode 100644
|
||||
index 0000000..dcab75a
|
||||
--- /dev/null
|
||||
+++ b/squashfs-tools/zstd_wrapper.c
|
||||
@@ -0,0 +1,254 @@
|
||||
+/*
|
||||
+ * Copyright (c) 2017
|
||||
+ * Phillip Lougher <phillip@squashfs.org.uk>
|
||||
+ *
|
||||
+ * This program is free software; you can redistribute it and/or
|
||||
+ * modify it under the terms of the GNU General Public License
|
||||
+ * as published by the Free Software Foundation; either version 2,
|
||||
+ * or (at your option) any later version.
|
||||
+ *
|
||||
+ * This program is distributed in the hope that it will be useful,
|
||||
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
+ * GNU General Public License for more details.
|
||||
+ *
|
||||
+ * zstd_wrapper.c
|
||||
+ *
|
||||
+ * Support for ZSTD compression http://zstd.net
|
||||
+ */
|
||||
+
|
||||
+#include <stdio.h>
|
||||
+#include <string.h>
|
||||
+#include <stdlib.h>
|
||||
+#include <zstd.h>
|
||||
+#include <zstd_errors.h>
|
||||
+
|
||||
+#include "squashfs_fs.h"
|
||||
+#include "zstd_wrapper.h"
|
||||
+#include "compressor.h"
|
||||
+
|
||||
+static int compression_level = ZSTD_DEFAULT_COMPRESSION_LEVEL;
|
||||
+
|
||||
+/*
|
||||
+ * This function is called by the options parsing code in mksquashfs.c
|
||||
+ * to parse any -X compressor option.
|
||||
+ *
|
||||
+ * This function returns:
|
||||
+ * >=0 (number of additional args parsed) on success
|
||||
+ * -1 if the option was unrecognised, or
|
||||
+ * -2 if the option was recognised, but otherwise bad in
|
||||
+ * some way (e.g. invalid parameter)
|
||||
+ *
|
||||
+ * Note: this function sets internal compressor state, but does not
|
||||
+ * pass back the results of the parsing other than success/failure.
|
||||
+ * The zstd_dump_options() function is called later to get the options in
|
||||
+ * a format suitable for writing to the filesystem.
|
||||
+ */
|
||||
+static int zstd_options(char *argv[], int argc)
|
||||
+{
|
||||
+ if (strcmp(argv[0], "-Xcompression-level") == 0) {
|
||||
+ if (argc < 2) {
|
||||
+ fprintf(stderr, "zstd: -Xcompression-level missing "
|
||||
+ "compression level\n");
|
||||
+ fprintf(stderr, "zstd: -Xcompression-level it should "
|
||||
+ "be 1 <= n <= %d\n", ZSTD_maxCLevel());
|
||||
+ goto failed;
|
||||
+ }
|
||||
+
|
||||
+ compression_level = atoi(argv[1]);
|
||||
+ if (compression_level < 1 ||
|
||||
+ compression_level > ZSTD_maxCLevel()) {
|
||||
+ fprintf(stderr, "zstd: -Xcompression-level invalid, it "
|
||||
+ "should be 1 <= n <= %d\n", ZSTD_maxCLevel());
|
||||
+ goto failed;
|
||||
+ }
|
||||
+
|
||||
+ return 1;
|
||||
+ }
|
||||
+
|
||||
+ return -1;
|
||||
+failed:
|
||||
+ return -2;
|
||||
+}
|
||||
+
|
||||
+/*
|
||||
+ * This function is called by mksquashfs to dump the parsed
|
||||
+ * compressor options in a format suitable for writing to the
|
||||
+ * compressor options field in the filesystem (stored immediately
|
||||
+ * after the superblock).
|
||||
+ *
|
||||
+ * This function returns a pointer to the compression options structure
|
||||
+ * to be stored (and the size), or NULL if there are no compression
|
||||
+ * options.
|
||||
+ */
|
||||
+static void *zstd_dump_options(int block_size, int *size)
|
||||
+{
|
||||
+ static struct zstd_comp_opts comp_opts;
|
||||
+
|
||||
+ /* don't return anything if the options are all default */
|
||||
+ if (compression_level == ZSTD_DEFAULT_COMPRESSION_LEVEL)
|
||||
+ return NULL;
|
||||
+
|
||||
+ comp_opts.compression_level = compression_level;
|
||||
+
|
||||
+ SQUASHFS_INSWAP_COMP_OPTS(&comp_opts);
|
||||
+
|
||||
+ *size = sizeof(comp_opts);
|
||||
+ return &comp_opts;
|
||||
+}
|
||||
+
|
||||
+/*
|
||||
+ * This function is a helper specifically for the append mode of
|
||||
+ * mksquashfs. Its purpose is to set the internal compressor state
|
||||
+ * to the stored compressor options in the passed compressor options
|
||||
+ * structure.
|
||||
+ *
|
||||
+ * In effect this function sets up the compressor options
|
||||
+ * to the same state they were when the filesystem was originally
|
||||
+ * generated, this is to ensure on appending, the compressor uses
|
||||
+ * the same compression options that were used to generate the
|
||||
+ * original filesystem.
|
||||
+ *
|
||||
+ * Note, even if there are no compressor options, this function is still
|
||||
+ * called with an empty compressor structure (size == 0), to explicitly
|
||||
+ * set the default options, this is to ensure any user supplied
|
||||
+ * -X options on the appending mksquashfs command line are over-ridden.
|
||||
+ *
|
||||
+ * This function returns 0 on sucessful extraction of options, and -1 on error.
|
||||
+ */
|
||||
+static int zstd_extract_options(int block_size, void *buffer, int size)
|
||||
+{
|
||||
+ struct zstd_comp_opts *comp_opts = buffer;
|
||||
+
|
||||
+ if (size == 0) {
|
||||
+ /* Set default values */
|
||||
+ compression_level = ZSTD_DEFAULT_COMPRESSION_LEVEL;
|
||||
+ return 0;
|
||||
+ }
|
||||
+
|
||||
+ /* we expect a comp_opts structure of sufficient size to be present */
|
||||
+ if (size < sizeof(*comp_opts))
|
||||
+ goto failed;
|
||||
+
|
||||
+ SQUASHFS_INSWAP_COMP_OPTS(comp_opts);
|
||||
+
|
||||
+ if (comp_opts->compression_level < 1 ||
|
||||
+ comp_opts->compression_level > ZSTD_maxCLevel()) {
|
||||
+ fprintf(stderr, "zstd: bad compression level in compression "
|
||||
+ "options structure\n");
|
||||
+ goto failed;
|
||||
+ }
|
||||
+
|
||||
+ compression_level = comp_opts->compression_level;
|
||||
+
|
||||
+ return 0;
|
||||
+
|
||||
+failed:
|
||||
+ fprintf(stderr, "zstd: error reading stored compressor options from "
|
||||
+ "filesystem!\n");
|
||||
+
|
||||
+ return -1;
|
||||
+}
|
||||
+
|
||||
+static void zstd_display_options(void *buffer, int size)
|
||||
+{
|
||||
+ struct zstd_comp_opts *comp_opts = buffer;
|
||||
+
|
||||
+ /* we expect a comp_opts structure of sufficient size to be present */
|
||||
+ if (size < sizeof(*comp_opts))
|
||||
+ goto failed;
|
||||
+
|
||||
+ SQUASHFS_INSWAP_COMP_OPTS(comp_opts);
|
||||
+
|
||||
+ if (comp_opts->compression_level < 1 ||
|
||||
+ comp_opts->compression_level > ZSTD_maxCLevel()) {
|
||||
+ fprintf(stderr, "zstd: bad compression level in compression "
|
||||
+ "options structure\n");
|
||||
+ goto failed;
|
||||
+ }
|
||||
+
|
||||
+ printf("\tcompression-level %d\n", comp_opts->compression_level);
|
||||
+
|
||||
+ return;
|
||||
+
|
||||
+failed:
|
||||
+ fprintf(stderr, "zstd: error reading stored compressor options from "
|
||||
+ "filesystem!\n");
|
||||
+}
|
||||
+
|
||||
+/*
|
||||
+ * This function is called by mksquashfs to initialise the
|
||||
+ * compressor, before compress() is called.
|
||||
+ *
|
||||
+ * This function returns 0 on success, and -1 on error.
|
||||
+ */
|
||||
+static int zstd_init(void **strm, int block_size, int datablock)
|
||||
+{
|
||||
+ ZSTD_CCtx *cctx = ZSTD_createCCtx();
|
||||
+
|
||||
+ if (!cctx) {
|
||||
+ fprintf(stderr, "zstd: failed to allocate compression "
|
||||
+ "context!\n");
|
||||
+ return -1;
|
||||
+ }
|
||||
+
|
||||
+ *strm = cctx;
|
||||
+ return 0;
|
||||
+}
|
||||
+
|
||||
+static int zstd_compress(void *strm, void *dest, void *src, int size,
|
||||
+ int block_size, int *error)
|
||||
+{
|
||||
+ const size_t res = ZSTD_compressCCtx((ZSTD_CCtx*)strm, dest, block_size,
|
||||
+ src, size, compression_level);
|
||||
+
|
||||
+ if (ZSTD_isError(res)) {
|
||||
+ /* FIXME:
|
||||
+ * zstd does not expose stable error codes. The error enum may
|
||||
+ * change between versions. Until upstream zstd stablizes the
|
||||
+ * error codes, we have no way of knowing why the error occurs.
|
||||
+ * zstd shouldn't fail to compress any input unless there isn't
|
||||
+ * enough output space. We assume that is the cause and return
|
||||
+ * the special error code for not enough output space.
|
||||
+ */
|
||||
+ return 0;
|
||||
+ }
|
||||
+
|
||||
+ return (int)res;
|
||||
+}
|
||||
+
|
||||
+static int zstd_uncompress(void *dest, void *src, int size, int outsize,
|
||||
+ int *error)
|
||||
+{
|
||||
+ const size_t res = ZSTD_decompress(dest, outsize, src, size);
|
||||
+
|
||||
+ if (ZSTD_isError(res)) {
|
||||
+ fprintf(stderr, "\t%d %d\n", outsize, size);
|
||||
+
|
||||
+ *error = (int)ZSTD_getErrorCode(res);
|
||||
+ return -1;
|
||||
+ }
|
||||
+
|
||||
+ return (int)res;
|
||||
+}
|
||||
+
|
||||
+static void zstd_usage(void)
|
||||
+{
|
||||
+ fprintf(stderr, "\t -Xcompression-level <compression-level>\n");
|
||||
+ fprintf(stderr, "\t\t<compression-level> should be 1 .. %d (default "
|
||||
+ "%d)\n", ZSTD_maxCLevel(), ZSTD_DEFAULT_COMPRESSION_LEVEL);
|
||||
+}
|
||||
+
|
||||
+struct compressor zstd_comp_ops = {
|
||||
+ .init = zstd_init,
|
||||
+ .compress = zstd_compress,
|
||||
+ .uncompress = zstd_uncompress,
|
||||
+ .options = zstd_options,
|
||||
+ .dump_options = zstd_dump_options,
|
||||
+ .extract_options = zstd_extract_options,
|
||||
+ .display_options = zstd_display_options,
|
||||
+ .usage = zstd_usage,
|
||||
+ .id = ZSTD_COMPRESSION,
|
||||
+ .name = "zstd",
|
||||
+ .supported = 1
|
||||
+};
|
||||
diff --git a/squashfs-tools/zstd_wrapper.h b/squashfs-tools/zstd_wrapper.h
|
||||
new file mode 100644
|
||||
index 0000000..4fbef0a
|
||||
--- /dev/null
|
||||
+++ b/squashfs-tools/zstd_wrapper.h
|
||||
@@ -0,0 +1,48 @@
|
||||
+#ifndef ZSTD_WRAPPER_H
|
||||
+#define ZSTD_WRAPPER_H
|
||||
+/*
|
||||
+ * Squashfs
|
||||
+ *
|
||||
+ * Copyright (c) 2017
|
||||
+ * Phillip Lougher <phillip@squashfs.org.uk>
|
||||
+ *
|
||||
+ * This program is free software; you can redistribute it and/or
|
||||
+ * modify it under the terms of the GNU General Public License
|
||||
+ * as published by the Free Software Foundation; either version 2,
|
||||
+ * or (at your option) any later version.
|
||||
+ *
|
||||
+ * This program is distributed in the hope that it will be useful,
|
||||
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
+ * GNU General Public License for more details.
|
||||
+ *
|
||||
+ * zstd_wrapper.h
|
||||
+ *
|
||||
+ */
|
||||
+
|
||||
+#ifndef linux
|
||||
+#define __BYTE_ORDER BYTE_ORDER
|
||||
+#define __BIG_ENDIAN BIG_ENDIAN
|
||||
+#define __LITTLE_ENDIAN LITTLE_ENDIAN
|
||||
+#else
|
||||
+#include <endian.h>
|
||||
+#endif
|
||||
+
|
||||
+#if __BYTE_ORDER == __BIG_ENDIAN
|
||||
+extern unsigned int inswap_le16(unsigned short);
|
||||
+extern unsigned int inswap_le32(unsigned int);
|
||||
+
|
||||
+#define SQUASHFS_INSWAP_COMP_OPTS(s) { \
|
||||
+ (s)->compression_level = inswap_le32((s)->compression_level); \
|
||||
+}
|
||||
+#else
|
||||
+#define SQUASHFS_INSWAP_COMP_OPTS(s)
|
||||
+#endif
|
||||
+
|
||||
+/* Default compression */
|
||||
+#define ZSTD_DEFAULT_COMPRESSION_LEVEL 15
|
||||
+
|
||||
+struct zstd_comp_opts {
|
||||
+ int compression_level;
|
||||
+};
|
||||
+#endif
|
||||
--
|
||||
2.19.1
|
||||
|
11
2gb.patch
11
2gb.patch
@ -1,11 +0,0 @@
|
||||
--- squashfs-tools/mksquashfs.c.orig 2014-09-13 11:08:27.352318167 -0500
|
||||
+++ squashfs-tools/mksquashfs.c 2014-09-13 11:09:36.701132044 -0500
|
||||
@@ -2055,7 +2055,7 @@
|
||||
|
||||
inline int is_fragment(struct inode_info *inode)
|
||||
{
|
||||
- int file_size = inode->buf.st_size;
|
||||
+ off_t file_size = inode->buf.st_size;
|
||||
|
||||
/*
|
||||
* If this block is to be compressed differently to the
|
159
PAE.patch
159
PAE.patch
@ -1,159 +0,0 @@
|
||||
From 55f7ba830d40d438f0b0663a505e0c227fc68b6b Mon Sep 17 00:00:00 2001
|
||||
From: Phillip Lougher <phillip@squashfs.org.uk>
|
||||
Date: Tue, 10 Jun 2014 21:51:52 +0100
|
||||
Subject: mksquashfs: fix phys mem calculation for 32-bit processes on
|
||||
PAE/64-bit kernels
|
||||
|
||||
When adding the code to base default memory usage on physical memory
|
||||
(by default use 25% of physical memory), I made an oversight. I assumed
|
||||
the process would be able to address 25% of physical memory.
|
||||
|
||||
However, for 32-bit processes running on a PAE kernel or 64-bit kernel,
|
||||
25% of physical memory can easily exceed the addressible memory for a
|
||||
32-bit process, e.g. if a machine has 24 GB of physical memory, the
|
||||
code would asume the process could easily use 6 GB.
|
||||
|
||||
A 32-bit process by definition can only address 4 GB (32-bit pointers).
|
||||
But, due to the typical kernel/user-space split (1GB/3GB, or 2GB/2GB)
|
||||
on PAE kernels, a 32-bit process may only be able to address 2 GB.
|
||||
|
||||
So, if Mksquashfs is a 32-bit application running on a PAE/64-bit kernel,
|
||||
the code assumes it can address much more memory than it really can, which
|
||||
means it runs out of memory.
|
||||
|
||||
The fix is to impose a maximum default limit on 32-bit kernels, or
|
||||
otherwise to never use a value more than 25% of the address space. If
|
||||
we assume the maximum address space is 2 GB, then the maximum becomes
|
||||
512 MB. But, given most kernels used the 1GB/3GB split, that may be
|
||||
unduely conservative, and 25% of 3 GB (756 MB) may be better. This
|
||||
patch compromises on 640 MB, which is mid-way between the 512 MB and 756 MB
|
||||
values. It is also the fixed default value previously used by Mksquashfs.
|
||||
|
||||
This patch also alters the code which imposes a maximum size. Previously
|
||||
it was believed limiting to the physical memory size was adequate. But
|
||||
obviously this needs to be updated to take into account a 32-bit process
|
||||
may only be able to address 2 GB. In the process I've also taken the
|
||||
opportunity to limit all requests to no more than 75% of physical memory.
|
||||
|
||||
Signed-off-by: Phillip Lougher <phillip@squashfs.org.uk>
|
||||
|
||||
diff --git a/squashfs-tools/mksquashfs.c b/squashfs-tools/mksquashfs.c
|
||||
index 86f82bb..5370ecf 100644
|
||||
--- a/squashfs-tools/mksquashfs.c
|
||||
+++ b/squashfs-tools/mksquashfs.c
|
||||
@@ -304,7 +304,7 @@ void restorefs();
|
||||
struct dir_info *scan1_opendir(char *pathname, char *subpath, int depth);
|
||||
void write_filesystem_tables(struct squashfs_super_block *sBlk, int nopad);
|
||||
unsigned short get_checksum_mem(char *buff, int bytes);
|
||||
-int get_physical_memory();
|
||||
+void check_usable_phys_mem(int total_mem);
|
||||
|
||||
|
||||
void prep_exit()
|
||||
@@ -4053,11 +4053,7 @@ void initialise_threads(int readq, int fragq, int bwriteq, int fwriteq,
|
||||
BAD_ERROR("Queue sizes rediculously too large\n");
|
||||
total_mem += fwriteq;
|
||||
|
||||
- if(total_mem > get_physical_memory()) {
|
||||
- ERROR("Total queue sizes larger than physical memory.\n");
|
||||
- ERROR("Mksquashfs will exhaust physical memory and thrash.\n");
|
||||
- BAD_ERROR("Queues too large\n");
|
||||
- }
|
||||
+ check_usable_phys_mem(total_mem);
|
||||
|
||||
/*
|
||||
* convert from queue size in Mbytes to queue size in
|
||||
@@ -4879,6 +4875,72 @@ int get_physical_memory()
|
||||
}
|
||||
|
||||
|
||||
+void check_usable_phys_mem(int total_mem)
|
||||
+{
|
||||
+ /*
|
||||
+ * We want to allow users to use as much of their physical
|
||||
+ * memory as they wish. However, for practical reasons there are
|
||||
+ * limits which need to be imposed, to protect users from themselves
|
||||
+ * and to prevent people from using Mksquashfs as a DOS attack by using
|
||||
+ * all physical memory. Mksquashfs uses memory to cache data from disk
|
||||
+ * to optimise performance. It is pointless to ask it to use more
|
||||
+ * than 75% of physical memory, as this causes thrashing and it is thus
|
||||
+ * self-defeating.
|
||||
+ */
|
||||
+ int mem = get_physical_memory();
|
||||
+
|
||||
+ mem = (mem >> 1) + (mem >> 2); /* 75% */
|
||||
+
|
||||
+ if(total_mem > mem) {
|
||||
+ ERROR("Total memory requested is more than 75%% of physical "
|
||||
+ "memory.\n");
|
||||
+ ERROR("Mksquashfs uses memory to cache data from disk to "
|
||||
+ "optimise performance.\n");
|
||||
+ ERROR("It is pointless to ask it to use more than this amount "
|
||||
+ "of memory, as this\n");
|
||||
+ ERROR("causes thrashing and it is thus self-defeating.\n");
|
||||
+ BAD_ERROR("Requested memory size too large\n");
|
||||
+ }
|
||||
+
|
||||
+ if(sizeof(void *) == 4 && total_mem > 2048) {
|
||||
+ /*
|
||||
+ * If we're running on a kernel with PAE or on a 64-bit kernel,
|
||||
+ * then the 75% physical memory limit can still easily exceed
|
||||
+ * the addressable memory by this process.
|
||||
+ *
|
||||
+ * Due to the typical kernel/user-space split (1GB/3GB, or
|
||||
+ * 2GB/2GB), we have to conservatively assume the 32-bit
|
||||
+ * processes can only address 2-3GB. So refuse if the user
|
||||
+ * tries to allocate more than 2GB.
|
||||
+ */
|
||||
+ ERROR("Total memory requested may exceed maximum "
|
||||
+ "addressable memory by this process\n");
|
||||
+ BAD_ERROR("Requested memory size too large\n");
|
||||
+ }
|
||||
+}
|
||||
+
|
||||
+
|
||||
+int get_default_phys_mem()
|
||||
+{
|
||||
+ int mem = get_physical_memory() / SQUASHFS_TAKE;
|
||||
+
|
||||
+ if(sizeof(void *) == 4 && mem > 640) {
|
||||
+ /*
|
||||
+ * If we're running on a kernel with PAE or on a 64-bit kernel,
|
||||
+ * the default memory usage can exceed the addressable
|
||||
+ * memory by this process.
|
||||
+ * Due to the typical kernel/user-space split (1GB/3GB, or
|
||||
+ * 2GB/2GB), we have to conservatively assume the 32-bit
|
||||
+ * processes can only address 2-3GB. So limit the default
|
||||
+ * usage to 640M, which gives room for other data.
|
||||
+ */
|
||||
+ mem = 640;
|
||||
+ }
|
||||
+
|
||||
+ return mem;
|
||||
+}
|
||||
+
|
||||
+
|
||||
void calculate_queue_sizes(int mem, int *readq, int *fragq, int *bwriteq,
|
||||
int *fwriteq)
|
||||
{
|
||||
@@ -4890,7 +4952,7 @@ void calculate_queue_sizes(int mem, int *readq, int *fragq, int *bwriteq,
|
||||
|
||||
|
||||
#define VERSION() \
|
||||
- printf("mksquashfs version 4.3 (2014/05/12)\n");\
|
||||
+ printf("mksquashfs version 4.3-git (2014/06/09)\n");\
|
||||
printf("copyright (C) 2014 Phillip Lougher "\
|
||||
"<phillip@squashfs.org.uk>\n\n"); \
|
||||
printf("This program is free software; you can redistribute it and/or"\
|
||||
@@ -4918,7 +4980,7 @@ int main(int argc, char *argv[])
|
||||
int fragq;
|
||||
int bwriteq;
|
||||
int fwriteq;
|
||||
- int total_mem = get_physical_memory() / SQUASHFS_TAKE;
|
||||
+ int total_mem = get_default_phys_mem();
|
||||
int progress = TRUE;
|
||||
int force_progress = FALSE;
|
||||
struct file_buffer **fragment = NULL;
|
||||
--
|
||||
cgit v0.10.1
|
||||
|
@ -1,29 +0,0 @@
|
||||
diff --git a/squashfs-tools/unsquash-4.c b/squashfs-tools/unsquash-4.c
|
||||
index ecdaac796f09..2c0cf63daf67 100644
|
||||
--- a/squashfs-tools/unsquash-4.c
|
||||
+++ b/squashfs-tools/unsquash-4.c
|
||||
@@ -31,9 +31,9 @@ static unsigned int *id_table;
|
||||
int read_fragment_table_4(long long *directory_table_end)
|
||||
{
|
||||
int res, i;
|
||||
- int bytes = SQUASHFS_FRAGMENT_BYTES(sBlk.s.fragments);
|
||||
- int indexes = SQUASHFS_FRAGMENT_INDEXES(sBlk.s.fragments);
|
||||
- long long fragment_table_index[indexes];
|
||||
+ size_t bytes = SQUASHFS_FRAGMENT_BYTES(sBlk.s.fragments);
|
||||
+ size_t indexes = SQUASHFS_FRAGMENT_INDEXES(sBlk.s.fragments);
|
||||
+ long long *fragment_table_index;
|
||||
|
||||
TRACE("read_fragment_table: %d fragments, reading %d fragment indexes "
|
||||
"from 0x%llx\n", sBlk.s.fragments, indexes,
|
||||
@@ -44,6 +44,11 @@ int read_fragment_table_4(long long *directory_table_end)
|
||||
return TRUE;
|
||||
}
|
||||
|
||||
+ fragment_table_index = malloc(indexes*sizeof(long long));
|
||||
+ if(fragment_table_index == NULL)
|
||||
+ EXIT_UNSQUASH("read_fragment_table: failed to allocate "
|
||||
+ "fragment table index\n");
|
||||
+
|
||||
fragment_table = malloc(bytes);
|
||||
if(fragment_table == NULL)
|
||||
EXIT_UNSQUASH("read_fragment_table: failed to allocate "
|
11
gcc10.patch
11
gcc10.patch
@ -1,11 +0,0 @@
|
||||
--- squashfs-tools/mksquashfs.h.orig 2014-05-09 23:54:13.000000000 -0500
|
||||
+++ squashfs-tools/mksquashfs.h 2020-02-08 22:08:33.275388914 -0600
|
||||
@@ -132,7 +132,7 @@
|
||||
#define BLOCK_OFFSET 2
|
||||
|
||||
extern struct cache *reader_buffer, *fragment_buffer, *reserve_cache;
|
||||
-struct cache *bwriter_buffer, *fwriter_buffer;
|
||||
+extern struct cache *bwriter_buffer, *fwriter_buffer;
|
||||
extern struct queue *to_reader, *to_deflate, *to_writer, *from_writer,
|
||||
*to_frag, *locked_fragment, *to_process_frag;
|
||||
extern struct append_file **file_mapping;
|
20
glibc.patch
20
glibc.patch
@ -1,20 +0,0 @@
|
||||
--- squashfs-tools/mksquashfs.c.orig 2019-05-21 01:50:02.197183719 -0500
|
||||
+++ squashfs-tools/mksquashfs.c 2019-05-21 01:51:05.818179903 -0500
|
||||
@@ -50,6 +50,7 @@
|
||||
#include <sys/wait.h>
|
||||
#include <limits.h>
|
||||
#include <ctype.h>
|
||||
+#include <sys/sysmacros.h>
|
||||
|
||||
#ifndef linux
|
||||
#define __BYTE_ORDER BYTE_ORDER
|
||||
--- squashfs-tools/unsquashfs.c.orig 2019-05-21 01:57:45.984155904 -0500
|
||||
+++ squashfs-tools/unsquashfs.c 2019-05-21 01:58:39.837152674 -0500
|
||||
@@ -37,6 +37,7 @@
|
||||
#include <sys/resource.h>
|
||||
#include <limits.h>
|
||||
#include <ctype.h>
|
||||
+#include <sys/sysmacros.h>
|
||||
|
||||
struct cache *fragment_cache, *data_cache;
|
||||
struct queue *to_reader, *to_inflate, *to_writer, *from_writer;
|
12
inline.patch
12
inline.patch
@ -1,12 +0,0 @@
|
||||
diff -Nrup a/squashfs-tools/mksquashfs.c b/squashfs-tools/mksquashfs.c
|
||||
--- a/squashfs-tools/mksquashfs.c 2019-12-17 11:18:23.336924170 -0700
|
||||
+++ b/squashfs-tools/mksquashfs.c 2019-12-17 11:16:56.184159640 -0700
|
||||
@@ -3030,7 +3030,7 @@ inline void alloc_inode_no(struct inode_
|
||||
}
|
||||
|
||||
|
||||
-inline struct dir_ent *create_dir_entry(char *name, char *source_name,
|
||||
+static inline struct dir_ent *create_dir_entry(char *name, char *source_name,
|
||||
char *nonstandard_pathname, struct dir_info *dir)
|
||||
{
|
||||
struct dir_ent *dir_ent = malloc(sizeof(struct dir_ent));
|
@ -1,11 +0,0 @@
|
||||
--- squashfs-tools/unsquash-4.c.orig 2015-06-24 14:23:22.270710744 -0500
|
||||
+++ squashfs-tools/unsquash-4.c 2015-06-24 14:24:13.671243487 -0500
|
||||
@@ -35,7 +35,7 @@
|
||||
size_t indexes = SQUASHFS_FRAGMENT_INDEXES(sBlk.s.fragments);
|
||||
long long *fragment_table_index;
|
||||
|
||||
- TRACE("read_fragment_table: %d fragments, reading %d fragment indexes "
|
||||
+ TRACE("read_fragment_table: %u fragments, reading %zu fragment indexes "
|
||||
"from 0x%llx\n", sBlk.s.fragments, indexes,
|
||||
sBlk.s.fragment_table_start);
|
||||
|
@ -1,33 +0,0 @@
|
||||
From 604b607d8ac91eb8afc0b6e3d917d5c073096103 Mon Sep 17 00:00:00 2001
|
||||
From: Phillip Lougher <phillip@squashfs.org.uk>
|
||||
Date: Wed, 11 Jun 2014 04:51:37 +0100
|
||||
Subject: mksquashfs: ensure value does not overflow a signed int in -mem
|
||||
option
|
||||
|
||||
Signed-off-by: Phillip Lougher <phillip@squashfs.org.uk>
|
||||
|
||||
diff --git a/squashfs-tools/mksquashfs.c b/squashfs-tools/mksquashfs.c
|
||||
index 5370ecf..9676dc8 100644
|
||||
--- a/squashfs-tools/mksquashfs.c
|
||||
+++ b/squashfs-tools/mksquashfs.c
|
||||
@@ -5193,7 +5193,16 @@ print_compressor_options:
|
||||
argv[0]);
|
||||
exit(1);
|
||||
}
|
||||
- /* convert from bytes to Mbytes */
|
||||
+
|
||||
+ /*
|
||||
+ * convert from bytes to Mbytes, ensuring the value
|
||||
+ * does not overflow a signed int
|
||||
+ */
|
||||
+ if(number >= (1LL << 51)) {
|
||||
+ ERROR("%s: -mem invalid mem size\n", argv[0]);
|
||||
+ exit(1);
|
||||
+ }
|
||||
+
|
||||
total_mem = number / 1048576;
|
||||
if(total_mem < (SQUASHFS_LOWMEM / SQUASHFS_TAKE)) {
|
||||
ERROR("%s: -mem should be %d Mbytes or "
|
||||
--
|
||||
cgit v0.10.1
|
||||
|
Loading…
Reference in New Issue
Block a user