From 919f37117fe6255d502eb303b92d1e6582053c67 Mon Sep 17 00:00:00 2001 From: Milan Broz Date: Tue, 9 Apr 2024 16:11:58 +0200 Subject: [PATCH] WIP: Add support for high-priority dm-crypt flag. --- lib/libcryptsetup.h | 2 ++ lib/libdevmapper.c | 20 ++++++++++++++++++-- lib/luks2/luks2_json_metadata.c | 1 + lib/utils_dm.h | 1 + man/common_options.adoc | 11 +++++++++++ src/cryptsetup.c | 8 +++++--- src/cryptsetup_arg_list.h | 2 ++ src/utils_arg_names.h | 1 + src/utils_luks.c | 3 +++ tests/device-test | 28 +++++++++++++++++++++++++--- 10 files changed, 69 insertions(+), 8 deletions(-) diff --git a/lib/libcryptsetup.h b/lib/libcryptsetup.h index 88d03bab..72bc12fc 100644 --- a/lib/libcryptsetup.h +++ b/lib/libcryptsetup.h @@ -1517,6 +1517,8 @@ int crypt_keyslot_destroy(struct crypt_device *cd, int keyslot); #define CRYPT_ACTIVATE_RECALCULATE_RESET (UINT32_C(1) << 26) /** dm-verity: try to use tasklets */ #define CRYPT_ACTIVATE_TASKLETS (UINT32_C(1) << 27) +/** dm-crypt: use high-priority workqueues */ +#define CRYPT_ACTIVATE_HIGH_PRIORITY (UINT32_C(1) << 28) /** * Active device runtime attributes diff --git a/lib/libdevmapper.c b/lib/libdevmapper.c index ebee542a..2e77849c 100644 --- a/lib/libdevmapper.c +++ b/lib/libdevmapper.c @@ -170,6 +170,9 @@ static void _dm_set_crypt_compat(struct crypt_device *cd, if (_dm_satisfies_version(1, 22, 0, crypt_maj, crypt_min, crypt_patch)) _dm_flags |= DM_CRYPT_NO_WORKQUEUE_SUPPORTED; + if (_dm_satisfies_version(1, 26, 0, crypt_maj, crypt_min, crypt_patch)) + _dm_flags |= DM_CRYPT_HIGH_PRIORITY_SUPPORTED; + _dm_crypt_checked = true; } @@ -571,19 +574,22 @@ static char *get_dm_crypt_params(const struct dm_target *tgt, uint32_t flags) num_options++; if (flags & CRYPT_ACTIVATE_IV_LARGE_SECTORS) num_options++; + if (flags & CRYPT_ACTIVATE_HIGH_PRIORITY) + num_options++; if (tgt->u.crypt.integrity) num_options++; if (tgt->u.crypt.sector_size != SECTOR_SIZE) num_options++; - if (num_options) { /* MAX length int32 + 15 + 15 + 23 + 18 + 19 + 17 + 13 + int32 + integrity_str */ - r = snprintf(features, sizeof(features), " %d%s%s%s%s%s%s%s%s", num_options, + if (num_options) { /* MAX length int32 + 15 + 15 + 23 + 18 + 19 + 17 + 14 + 13 + int32 + integrity_str */ + r = snprintf(features, sizeof(features), " %d%s%s%s%s%s%s%s%s%s", num_options, (flags & CRYPT_ACTIVATE_ALLOW_DISCARDS) ? " allow_discards" : "", (flags & CRYPT_ACTIVATE_SAME_CPU_CRYPT) ? " same_cpu_crypt" : "", (flags & CRYPT_ACTIVATE_SUBMIT_FROM_CRYPT_CPUS) ? " submit_from_crypt_cpus" : "", (flags & CRYPT_ACTIVATE_NO_READ_WORKQUEUE) ? " no_read_workqueue" : "", (flags & CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE) ? " no_write_workqueue" : "", (flags & CRYPT_ACTIVATE_IV_LARGE_SECTORS) ? " iv_large_sectors" : "", + (flags & CRYPT_ACTIVATE_HIGH_PRIORITY) ? " high_priority" : "", (tgt->u.crypt.sector_size != SECTOR_SIZE) ? _uf(sector_feature, sizeof(sector_feature), "sector_size", tgt->u.crypt.sector_size) : "", integrity_dm); @@ -1593,6 +1599,14 @@ static int check_retry(struct crypt_device *cd, uint32_t *dmd_flags, uint32_t dm ret = 1; } + /* Drop high-priority workqueue options if not supported */ + if ((*dmd_flags & CRYPT_ACTIVATE_HIGH_PRIORITY) && + !(dmt_flags & DM_CRYPT_HIGH_PRIORITY_SUPPORTED)) { + log_dbg(cd, "dm-crypt does not support high-priority option"); + *dmd_flags = *dmd_flags & ~CRYPT_ACTIVATE_HIGH_PRIORITY; + ret = 1; + } + return ret; } @@ -1986,6 +2000,8 @@ static int _dm_target_query_crypt(struct crypt_device *cd, uint32_t get_flags, *act_flags |= CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE; else if (!strcasecmp(arg, "iv_large_sectors")) *act_flags |= CRYPT_ACTIVATE_IV_LARGE_SECTORS; + else if (!strcasecmp(arg, "high_priority")) + *act_flags |= CRYPT_ACTIVATE_HIGH_PRIORITY; else if (sscanf(arg, "integrity:%u:", &val) == 1) { tgt->u.crypt.tag_size = val; rintegrity = strchr(arg + strlen("integrity:"), ':'); diff --git a/lib/luks2/luks2_json_metadata.c b/lib/luks2/luks2_json_metadata.c index 22f3e3df..00f111f4 100644 --- a/lib/luks2/luks2_json_metadata.c +++ b/lib/luks2/luks2_json_metadata.c @@ -1566,6 +1566,7 @@ static const struct { { CRYPT_ACTIVATE_NO_JOURNAL, "no-journal" }, { CRYPT_ACTIVATE_NO_READ_WORKQUEUE, "no-read-workqueue" }, { CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE, "no-write-workqueue" }, + { CRYPT_ACTIVATE_HIGH_PRIORITY, "high_priority" }, { 0, NULL } }; diff --git a/lib/utils_dm.h b/lib/utils_dm.h index dbbd4701..a9ca9add 100644 --- a/lib/utils_dm.h +++ b/lib/utils_dm.h @@ -76,6 +76,7 @@ static inline uint32_t act2dmflags(uint32_t act_flags) #define DM_INTEGRITY_FIX_HMAC_SUPPORTED (1 << 26) /* hmac covers also superblock */ #define DM_INTEGRITY_RESET_RECALC_SUPPORTED (1 << 27) /* dm-integrity automatic recalculation supported */ #define DM_VERITY_TASKLETS_SUPPORTED (1 << 28) /* dm-verity tasklets supported */ +#define DM_CRYPT_HIGH_PRIORITY_SUPPORTED (1 << 29) /* dm-crypt high priority workqueue flag supported */ typedef enum { DM_CRYPT = 0, DM_VERITY, DM_INTEGRITY, DM_LINEAR, DM_ERROR, DM_ZERO, DM_UNKNOWN } dm_target_type; enum tdirection { TARGET_EMPTY = 0, TARGET_SET, TARGET_QUERY }; diff --git a/man/common_options.adoc b/man/common_options.adoc index 841929bd..4cba277d 100644 --- a/man/common_options.adoc +++ b/man/common_options.adoc @@ -773,6 +773,17 @@ that it is maximal value, it is decreased automatically if CPU online count is lower. This option is not available for PBKDF2. endif::[] +ifdef::ACTION_REFRESH,ACTION_OPEN[] +*--perf-high_priority*:: +Set dm-crypt workqueues and the writer thread to high priority. +This i mproves throughput and latency of dm-crypt while degrading general +responsiveness of the system. ++ +*NOTE:* This option is available only for low-level dm-crypt +performance tuning, use only if you need a change to default dm-crypt +behaviour. Needs kernel 6.10 or later. +endif::[] + ifdef::ACTION_REFRESH,ACTION_OPEN[] *--perf-no_read_workqueue, --perf-no_write_workqueue*:: Bypass dm-crypt internal workqueue and process read or write requests diff --git a/src/cryptsetup.c b/src/cryptsetup.c index fd580d73..57344e51 100644 --- a/src/cryptsetup.c +++ b/src/cryptsetup.c @@ -1036,13 +1036,15 @@ static int action_status(void) CRYPT_ACTIVATE_SAME_CPU_CRYPT| CRYPT_ACTIVATE_SUBMIT_FROM_CRYPT_CPUS| CRYPT_ACTIVATE_NO_READ_WORKQUEUE| - CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE)) - log_std(" flags: %s%s%s%s%s\n", + CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE| + CRYPT_ACTIVATE_HIGH_PRIORITY)) + log_std(" flags: %s%s%s%s%s%s\n", (cad.flags & CRYPT_ACTIVATE_ALLOW_DISCARDS) ? "discards " : "", (cad.flags & CRYPT_ACTIVATE_SAME_CPU_CRYPT) ? "same_cpu_crypt " : "", (cad.flags & CRYPT_ACTIVATE_SUBMIT_FROM_CRYPT_CPUS) ? "submit_from_crypt_cpus " : "", (cad.flags & CRYPT_ACTIVATE_NO_READ_WORKQUEUE) ? "no_read_workqueue " : "", - (cad.flags & CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE) ? "no_write_workqueue" : ""); + (cad.flags & CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE) ? "no_write_workqueue" : "", + (cad.flags & CRYPT_ACTIVATE_HIGH_PRIORITY) ? "high_priority" : ""); } out: crypt_free(cd); diff --git a/src/cryptsetup_arg_list.h b/src/cryptsetup_arg_list.h index 74967487..510b52d4 100644 --- a/src/cryptsetup_arg_list.h +++ b/src/cryptsetup_arg_list.h @@ -141,6 +141,8 @@ ARG(OPT_PBKDF_MEMORY, '\0', POPT_ARG_STRING, N_("PBKDF memory cost limit"), N_(" ARG(OPT_PBKDF_PARALLEL, '\0', POPT_ARG_STRING, N_("PBKDF parallel cost"), N_("threads"), CRYPT_ARG_UINT32, { .u32_value = DEFAULT_LUKS2_PARALLEL_THREADS }, {}) +ARG(OPT_PERF_HIGH_PRIORITY, '\0', POPT_ARG_NONE, N_("Set dm-crypt workqueues and the writer thread to high priority"), NULL, CRYPT_ARG_BOOL, {}, {}) + ARG(OPT_PERF_NO_READ_WORKQUEUE, '\0', POPT_ARG_NONE, N_("Bypass dm-crypt workqueue and process read requests synchronously"), NULL, CRYPT_ARG_BOOL, {}, {}) ARG(OPT_PERF_NO_WRITE_WORKQUEUE, '\0', POPT_ARG_NONE, N_("Bypass dm-crypt workqueue and process write requests synchronously"), NULL, CRYPT_ARG_BOOL, {}, {}) diff --git a/src/utils_arg_names.h b/src/utils_arg_names.h index 4ec5510b..604e1710 100644 --- a/src/utils_arg_names.h +++ b/src/utils_arg_names.h @@ -125,6 +125,7 @@ #define OPT_PBKDF_FORCE_ITERATIONS "pbkdf-force-iterations" #define OPT_PBKDF_MEMORY "pbkdf-memory" #define OPT_PBKDF_PARALLEL "pbkdf-parallel" +#define OPT_PERF_HIGH_PRIORITY "perf-high_priority" #define OPT_PERF_NO_READ_WORKQUEUE "perf-no_read_workqueue" #define OPT_PERF_NO_WRITE_WORKQUEUE "perf-no_write_workqueue" #define OPT_PERF_SAME_CPU_CRYPT "perf-same_cpu_crypt" diff --git a/src/utils_luks.c b/src/utils_luks.c index 5c51a5a2..95f768b8 100644 --- a/src/utils_luks.c +++ b/src/utils_luks.c @@ -92,6 +92,9 @@ void set_activation_flags(uint32_t *flags) if (ARG_SET(OPT_PERF_NO_WRITE_WORKQUEUE_ID)) *flags |= CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE; + if (ARG_SET(OPT_PERF_HIGH_PRIORITY_ID)) + *flags |= CRYPT_ACTIVATE_HIGH_PRIORITY; + if (ARG_SET(OPT_INTEGRITY_NO_JOURNAL_ID)) *flags |= CRYPT_ACTIVATE_NO_JOURNAL; diff --git a/tests/device-test b/tests/device-test index 9aaf03c8..212cc865 100755 --- a/tests/device-test +++ b/tests/device-test @@ -106,6 +106,9 @@ function dm_crypt_features() [ $VER_MIN -lt 22 ] && return DM_PERF_NO_WORKQUEUE=1 + + [ $VER_MIN -lt 26 ] && return + DM_PERF_HIGH_PRIORITY=1 } format() # format @@ -183,12 +186,18 @@ else $CRYPTSETUP status $DEV_NAME | grep -q same_cpu_crypt && fail echo -e "$PWD1" | $CRYPTSETUP refresh $PLAIN_OPT $DEV $DEV_NAME2 2>/dev/null && fail if [ -n "$DM_PERF_NO_WORKQUEUE" ]; then - echo -n "no_read_workqueue no_write_workqueue" + echo -n "no_read_workqueue no_write_workqueue " echo -e "$PWD1" | $CRYPTSETUP refresh $PLAIN_OPT -q $DEV_NAME --perf-no_read_workqueue --perf-no_write_workqueue || fail $CRYPTSETUP status $DEV_NAME | grep -q no_read_workqueue || fail $CRYPTSETUP status $DEV_NAME | grep -q no_write_workqueue || fail check_io fi + if [ -n "$DM_PERF_HIGH_PRIORITY" ]; then + echo -n "high_priority " + echo -e "$PWD1" | $CRYPTSETUP refresh $PLAIN_OPT -q $DEV_NAME --perf-high_priority || fail + $CRYPTSETUP status $DEV_NAME | grep -q high_priority || fail + check_io + fi $CRYPTSETUP close $DEV_NAME || fail echo @@ -215,11 +224,16 @@ else $CRYPTSETUP status $DEV_NAME | grep -q same_cpu_crypt && fail echo -e "$PWD1" | $CRYPTSETUP refresh $DEV $DEV_NAME2 2>/dev/null && fail if [ -n "$DM_PERF_NO_WORKQUEUE" ]; then - echo -n "no_read_workqueue no_write_workqueue" + echo -n "no_read_workqueue no_write_workqueue " echo -e "$PWD1" | $CRYPTSETUP refresh $DEV_NAME --perf-no_read_workqueue --perf-no_write_workqueue || fail $CRYPTSETUP status $DEV_NAME | grep -q no_read_workqueue || fail $CRYPTSETUP status $DEV_NAME | grep -q no_write_workqueue || fail fi + if [ -n "$DM_PERF_HIGH_PRIORITY" ]; then + echo -n "high_priority " + echo -e "$PWD1" | $CRYPTSETUP refresh $DEV_NAME --perf-high_priority || fail + $CRYPTSETUP status $DEV_NAME | grep -q high_priority || fail + fi $CRYPTSETUP close $DEV_NAME || fail echo @@ -280,7 +294,7 @@ else fi fi if [ -n "$DM_PERF_NO_WORKQUEUE" ]; then - echo -n "no_read_workqueue no_write_workqueue" + echo -n "no_read_workqueue no_write_workqueue " echo -e "$PWD1" | $CRYPTSETUP refresh $DEV $DEV_NAME --perf-no_read_workqueue --perf-no_write_workqueue --persistent || fail $CRYPTSETUP status $DEV_NAME | grep -q no_read_workqueue || fail $CRYPTSETUP status $DEV_NAME | grep -q no_write_workqueue || fail @@ -289,6 +303,14 @@ else $CRYPTSETUP status $DEV_NAME | grep -q no_read_workqueue || fail $CRYPTSETUP status $DEV_NAME | grep -q no_write_workqueue || fail fi + if [ -n "$DM_PERF_HIGH_PRIORITY" ]; then + echo -n "high_priority " + echo -e "$PWD1" | $CRYPTSETUP refresh $DEV $DEV_NAME --perf-high_priority --persistent || fail + $CRYPTSETUP status $DEV_NAME | grep -q high_priority || fail + $CRYPTSETUP close $DEV_NAME || fail + echo -e "$PWD1" | $CRYPTSETUP open $DEV $DEV_NAME || fail + $CRYPTSETUP status $DEV_NAME | grep -q high_priority || fail + fi echo -e "$PWD1" | $CRYPTSETUP refresh $DEV $DEV_NAME2 2>/dev/null && fail $CRYPTSETUP close $DEV_NAME || fail echo -- 2.44.0