diff --git a/lib/libcryptsetup.h b/lib/libcryptsetup.h index 7174907d..3d002c0c 100644 --- a/lib/libcryptsetup.h +++ b/lib/libcryptsetup.h @@ -1105,6 +1105,10 @@ int crypt_keyslot_destroy(struct crypt_device *cd, int keyslot); #define CRYPT_ACTIVATE_IV_LARGE_SECTORS (1 << 22) /** dm-verity: panic_on_corruption flag - panic kernel on corruption */ #define CRYPT_ACTIVATE_PANIC_ON_CORRUPTION (1 << 23) +/** dm-crypt: bypass internal workqueue and process read requests synchronously. */ +#define CRYPT_ACTIVATE_NO_READ_WORKQUEUE (1 << 24) +/** dm-crypt: bypass internal workqueue and process write requests synchronously. */ +#define CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE (1 << 25) /** * Active device runtime attributes diff --git a/lib/libdevmapper.c b/lib/libdevmapper.c index 9c943ffc..8f8c94b5 100644 --- a/lib/libdevmapper.c +++ b/lib/libdevmapper.c @@ -174,6 +174,9 @@ static void _dm_set_crypt_compat(struct crypt_device *cd, if (_dm_satisfies_version(1, 20, 0, crypt_maj, crypt_min, crypt_patch)) _dm_flags |= DM_BITLK_ELEPHANT_SUPPORTED; + if (_dm_satisfies_version(1, 22, 0, crypt_maj, crypt_min, crypt_patch)) + _dm_flags |= DM_CRYPT_NO_WORKQUEUE_SUPPORTED; + _dm_crypt_checked = true; } @@ -618,6 +621,10 @@ static char *get_dm_crypt_params(const struct dm_target *tgt, uint32_t flags) num_options++; if (flags & CRYPT_ACTIVATE_SUBMIT_FROM_CRYPT_CPUS) num_options++; + if (flags & CRYPT_ACTIVATE_NO_READ_WORKQUEUE) + num_options++; + if (flags & CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE) + num_options++; if (flags & CRYPT_ACTIVATE_IV_LARGE_SECTORS) num_options++; if (tgt->u.crypt.integrity) @@ -630,10 +637,12 @@ static char *get_dm_crypt_params(const struct dm_target *tgt, uint32_t flags) *sector_feature = '\0'; if (num_options) { - snprintf(features, sizeof(features)-1, " %d%s%s%s%s%s%s", num_options, + snprintf(features, sizeof(features)-1, " %d%s%s%s%s%s%s%s%s", num_options, (flags & CRYPT_ACTIVATE_ALLOW_DISCARDS) ? " allow_discards" : "", (flags & CRYPT_ACTIVATE_SAME_CPU_CRYPT) ? " same_cpu_crypt" : "", (flags & CRYPT_ACTIVATE_SUBMIT_FROM_CRYPT_CPUS) ? " submit_from_crypt_cpus" : "", + (flags & CRYPT_ACTIVATE_NO_READ_WORKQUEUE) ? " no_read_workqueue" : "", + (flags & CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE) ? " no_write_workqueue" : "", (flags & CRYPT_ACTIVATE_IV_LARGE_SECTORS) ? " iv_large_sectors" : "", sector_feature, integrity_dm); } else @@ -1610,6 +1619,14 @@ static int check_retry(struct crypt_device *cd, uint32_t *dmd_flags, uint32_t dm ret = 1; } + /* Drop no workqueue options if not supported */ + if ((*dmd_flags & (CRYPT_ACTIVATE_NO_READ_WORKQUEUE | CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE)) && + !(dmt_flags & DM_CRYPT_NO_WORKQUEUE_SUPPORTED)) { + log_dbg(cd, "dm-crypt does not support performance options"); + *dmd_flags = *dmd_flags & ~(CRYPT_ACTIVATE_NO_READ_WORKQUEUE | CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE); + ret = 1; + } + return ret; } @@ -1632,14 +1649,21 @@ int dm_create_device(struct crypt_device *cd, const char *name, goto out; if (r && (dmd->segment.type == DM_CRYPT || dmd->segment.type == DM_LINEAR || dmd->segment.type == DM_ZERO) && - check_retry(cd, &dmd->flags, dmt_flags)) + check_retry(cd, &dmd->flags, dmt_flags)) { + log_dbg(cd, "Retrying open without incompatible options."); r = _dm_create_device(cd, name, type, dmd->uuid, dmd); + } if (r == -EINVAL && dmd->flags & (CRYPT_ACTIVATE_SAME_CPU_CRYPT|CRYPT_ACTIVATE_SUBMIT_FROM_CRYPT_CPUS) && !(dmt_flags & (DM_SAME_CPU_CRYPT_SUPPORTED|DM_SUBMIT_FROM_CRYPT_CPUS_SUPPORTED))) log_err(cd, _("Requested dm-crypt performance options are not supported.")); + if (r == -EINVAL && + dmd->flags & (CRYPT_ACTIVATE_NO_READ_WORKQUEUE | CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE) && + !(dmt_flags & DM_CRYPT_NO_WORKQUEUE_SUPPORTED)) + log_err(cd, _("Requested dm-crypt performance options are not supported.")); + if (r == -EINVAL && dmd->flags & (CRYPT_ACTIVATE_IGNORE_CORRUPTION| CRYPT_ACTIVATE_RESTART_ON_CORRUPTION| CRYPT_ACTIVATE_IGNORE_ZERO_BLOCKS| @@ -1697,7 +1721,10 @@ int dm_reload_device(struct crypt_device *cd, const char *name, if (r == -EINVAL && (dmd->segment.type == DM_CRYPT || dmd->segment.type == DM_LINEAR)) { if ((dmd->flags & (CRYPT_ACTIVATE_SAME_CPU_CRYPT|CRYPT_ACTIVATE_SUBMIT_FROM_CRYPT_CPUS)) && - !dm_flags(cd, DM_CRYPT, &dmt_flags) && !(dmt_flags & (DM_SAME_CPU_CRYPT_SUPPORTED|DM_SUBMIT_FROM_CRYPT_CPUS_SUPPORTED))) + !dm_flags(cd, DM_CRYPT, &dmt_flags) && !(dmt_flags & (DM_SAME_CPU_CRYPT_SUPPORTED | DM_SUBMIT_FROM_CRYPT_CPUS_SUPPORTED))) + log_err(cd, _("Requested dm-crypt performance options are not supported.")); + if ((dmd->flags & (CRYPT_ACTIVATE_NO_READ_WORKQUEUE | CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE)) && + !dm_flags(cd, DM_CRYPT, &dmt_flags) && !(dmt_flags & DM_CRYPT_NO_WORKQUEUE_SUPPORTED)) log_err(cd, _("Requested dm-crypt performance options are not supported.")); if ((dmd->flags & CRYPT_ACTIVATE_ALLOW_DISCARDS) && !dm_flags(cd, DM_CRYPT, &dmt_flags) && !(dmt_flags & DM_DISCARDS_SUPPORTED)) @@ -1941,6 +1968,10 @@ static int _dm_target_query_crypt(struct crypt_device *cd, uint32_t get_flags, *act_flags |= CRYPT_ACTIVATE_SAME_CPU_CRYPT; else if (!strcasecmp(arg, "submit_from_crypt_cpus")) *act_flags |= CRYPT_ACTIVATE_SUBMIT_FROM_CRYPT_CPUS; + else if (!strcasecmp(arg, "no_read_workqueue")) + *act_flags |= CRYPT_ACTIVATE_NO_READ_WORKQUEUE; + else if (!strcasecmp(arg, "no_write_workqueue")) + *act_flags |= CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE; else if (!strcasecmp(arg, "iv_large_sectors")) *act_flags |= CRYPT_ACTIVATE_IV_LARGE_SECTORS; else if (sscanf(arg, "integrity:%u:", &val) == 1) { diff --git a/lib/luks2/luks2_json_metadata.c b/lib/luks2/luks2_json_metadata.c index e346067b..e0cc8eb2 100644 --- a/lib/luks2/luks2_json_metadata.c +++ b/lib/luks2/luks2_json_metadata.c @@ -1289,6 +1289,8 @@ static const struct { { CRYPT_ACTIVATE_SAME_CPU_CRYPT, "same-cpu-crypt" }, { CRYPT_ACTIVATE_SUBMIT_FROM_CRYPT_CPUS, "submit-from-crypt-cpus" }, { CRYPT_ACTIVATE_NO_JOURNAL, "no-journal" }, + { CRYPT_ACTIVATE_NO_READ_WORKQUEUE, "no-read-workqueue" }, + { CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE, "no-write-workqueue" }, { 0, NULL } }; diff --git a/lib/utils_dm.h b/lib/utils_dm.h index d384f350..22add180 100644 --- a/lib/utils_dm.h +++ b/lib/utils_dm.h @@ -70,6 +70,7 @@ static inline uint32_t act2dmflags(uint32_t act_flags) #define DM_VERITY_SIGNATURE_SUPPORTED (1 << 22) /* Verity option root_hash_sig_key_desc supported */ #define DM_INTEGRITY_DISCARDS_SUPPORTED (1 << 23) /* dm-integrity discards/TRIM option is supported */ #define DM_VERITY_PANIC_CORRUPTION_SUPPORTED (1 << 24) /* dm-verity panic on corruption */ +#define DM_CRYPT_NO_WORKQUEUE_SUPPORTED (1 << 25) /* dm-crypt suppot for bypassing workqueues */ typedef enum { DM_CRYPT = 0, DM_VERITY, DM_INTEGRITY, DM_LINEAR, DM_ERROR, DM_ZERO, DM_UNKNOWN } dm_target_type; enum tdirection { TARGET_SET = 1, TARGET_QUERY }; diff --git a/man/cryptsetup.8 b/man/cryptsetup.8 index 3154d479..643012d4 100644 --- a/man/cryptsetup.8 +++ b/man/cryptsetup.8 @@ -146,7 +146,8 @@ Mandatory parametrs are identical to those of an open action for respective device type. You may change following parameters on all devices \-\-perf\-same_cpu_crypt, -\-\-perf\-submit_from_crypt_cpus and \-\-allow\-discards. +\-\-perf\-submit_from_crypt_cpus, \-\-perf-no_read_workqueue, \-\-no_write_workqueue +and \-\-allow\-discards. Refreshing device without any optional parameter will refresh the device with default setting (respective to device type). @@ -1209,6 +1210,15 @@ This option is only relevant for \fIopen\fR action. performance tuning, use only if you need a change to default dm-crypt behaviour. Needs kernel 4.0 or later. .TP +.B "\-\-perf\-no_read_workqueue, \-\-perf\-no_write_workqueue\fR" +Bypass dm-crypt internal workqueue and process read or write requests +synchronously. +This option is only relevant for \fIopen\fR action. + +\fBNOTE:\fR These options are available only for low-level dm-crypt +performance tuning, use only if you need a change to default dm-crypt +behaviour. Needs kernel 5.9 or later. +.TP .B "\-\-test\-passphrase\fR" Do not activate the device, just verify passphrase. This option is only relevant for \fIopen\fR action (the device @@ -1325,7 +1335,8 @@ the flag you want to remove (e.g. to disable persistently stored discard flag, use \fI\-\-persistent\fR without \fI\-\-allow-discards\fR). Only \fI\-\-allow-discards\fR, \fI\-\-perf\-same_cpu_crypt\fR, -\fI\-\-perf\-submit_from_crypt_cpus\fR and \fI\-\-integrity\-no\-journal\fR +\fI\-\-perf\-submit_from_crypt_cpus\fR, \fI\-\-perf\-no_read_workqueue\fR, +\fI\-\-perf\-no_write_workqueue\fR and \fI\-\-integrity\-no\-journal\fR can be stored persistently. .TP .B "\-\-refresh" diff --git a/src/cryptsetup.c b/src/cryptsetup.c index 5c651edc..b32f167d 100644 --- a/src/cryptsetup.c +++ b/src/cryptsetup.c @@ -64,6 +64,8 @@ static int opt_shared = 0; static int opt_allow_discards = 0; static int opt_perf_same_cpu_crypt = 0; static int opt_perf_submit_from_crypt_cpus = 0; +static int opt_perf_no_read_workqueue = 0; +static int opt_perf_no_write_workqueue = 0; static int opt_test_passphrase = 0; static int opt_tcrypt_hidden = 0; static int opt_tcrypt_system = 0; @@ -182,6 +184,12 @@ static void _set_activation_flags(uint32_t *flags) if (opt_perf_submit_from_crypt_cpus) *flags |= CRYPT_ACTIVATE_SUBMIT_FROM_CRYPT_CPUS; + if (opt_perf_no_read_workqueue) + *flags |= CRYPT_ACTIVATE_NO_READ_WORKQUEUE; + + if (opt_perf_no_write_workqueue) + *flags |= CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE; + if (opt_integrity_nojournal) *flags |= CRYPT_ACTIVATE_NO_JOURNAL; @@ -815,11 +823,15 @@ static int action_status(void) (cad.flags & CRYPT_ACTIVATE_SUSPENDED) ? " (suspended)" : ""); if (cad.flags & (CRYPT_ACTIVATE_ALLOW_DISCARDS| CRYPT_ACTIVATE_SAME_CPU_CRYPT| - CRYPT_ACTIVATE_SUBMIT_FROM_CRYPT_CPUS)) - log_std(" flags: %s%s%s\n", + CRYPT_ACTIVATE_SUBMIT_FROM_CRYPT_CPUS| + CRYPT_ACTIVATE_NO_READ_WORKQUEUE| + CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE)) + log_std(" flags: %s%s%s%s%s\n", (cad.flags & CRYPT_ACTIVATE_ALLOW_DISCARDS) ? "discards " : "", (cad.flags & CRYPT_ACTIVATE_SAME_CPU_CRYPT) ? "same_cpu_crypt " : "", - (cad.flags & CRYPT_ACTIVATE_SUBMIT_FROM_CRYPT_CPUS) ? "submit_from_crypt_cpus" : ""); + (cad.flags & CRYPT_ACTIVATE_SUBMIT_FROM_CRYPT_CPUS) ? "submit_from_crypt_cpus " : "", + (cad.flags & CRYPT_ACTIVATE_NO_READ_WORKQUEUE) ? "no_read_workqueue " : "", + (cad.flags & CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE) ? "no_write_workqueue" : ""); } out: crypt_free(cd); @@ -3535,6 +3547,8 @@ int main(int argc, const char **argv) { "force-password", '\0', POPT_ARG_NONE, &opt_force_password, 0, N_("Disable password quality check (if enabled)"), NULL }, { "perf-same_cpu_crypt",'\0', POPT_ARG_NONE, &opt_perf_same_cpu_crypt, 0, N_("Use dm-crypt same_cpu_crypt performance compatibility option"), NULL }, { "perf-submit_from_crypt_cpus",'\0', POPT_ARG_NONE, &opt_perf_submit_from_crypt_cpus,0,N_("Use dm-crypt submit_from_crypt_cpus performance compatibility option"), NULL }, + { "perf-no_read_workqueue",'\0', POPT_ARG_NONE, &opt_perf_no_read_workqueue,0,N_("Bypass dm-crypt workqueue and process read requests synchronously"), NULL }, + { "perf-no_write_workqueue",'\0', POPT_ARG_NONE, &opt_perf_no_write_workqueue,0,N_("Bypass dm-crypt workqueue and process write requests synchronously"), NULL }, { "deferred", '\0', POPT_ARG_NONE, &opt_deferred_remove, 0, N_("Device removal is deferred until the last user closes it"), NULL }, { "serialize-memory-hard-pbkdf", '\0', POPT_ARG_NONE, &opt_serialize_memory_hard_pbkdf, 0, N_("Use global lock to serialize memory hard PBKDF (OOM workaround)"), NULL }, { "iter-time", 'i', POPT_ARG_INT, &opt_iteration_time, 0, N_("PBKDF iteration time for LUKS (in ms)"), N_("msecs") }, diff --git a/tests/device-test b/tests/device-test index 0898b455..b1ec5abe 100755 --- a/tests/device-test +++ b/tests/device-test @@ -38,6 +38,7 @@ skip() function dm_crypt_features() { + modprobe dm-crypt || fail "dm-crypt failed to load" VER_STR=$(dmsetup targets | grep crypt | cut -f2 -dv) [ -z "$VER_STR" ] && fail "Failed to parse dm-crypt version." @@ -61,6 +62,9 @@ function dm_crypt_features() if [ $VER_MIN -gt 18 -o \( $VER_MIN -eq 18 -a $VER_PTC -ge 1 \) ]; then test -d /proc/sys/kernel/keys && DM_KEYRING=1 fi + + [ $VER_MIN -lt 22 ] && return + DM_PERF_NO_WORKQUEUE=1 } function dm_crypt_keyring_support() @@ -141,7 +145,13 @@ else $CRYPTSETUP status $DEV_NAME | grep -q discards && fail $CRYPTSETUP status $DEV_NAME | grep -q same_cpu_crypt && fail echo -e "$PWD1" | $CRYPTSETUP refresh --hash sha256 $DEV $DEV_NAME2 2>/dev/null && fail + if [ -n "$DM_PERF_NO_WORKQUEUE" ]; then + echo -e "$PWD1" | $CRYPTSETUP refresh --hash sha256 -q $DEV_NAME --perf-no_read_workqueue --perf-no_write_workqueue || fail + $CRYPTSETUP status $DEV_NAME | grep -q no_read_workqueue || fail + $CRYPTSETUP status $DEV_NAME | grep -q no_write_workqueue || fail + fi $CRYPTSETUP close $DEV_NAME || fail + # LUKS echo -e "$PWD1" | $CRYPTSETUP open --type luks1 $DEV $DEV_NAME --perf-same_cpu_crypt --perf-submit_from_crypt_cpus || fail $CRYPTSETUP status $DEV_NAME | grep -q same_cpu_crypt || fail @@ -162,6 +172,11 @@ else $CRYPTSETUP status $DEV_NAME | grep -q discards && fail $CRYPTSETUP status $DEV_NAME | grep -q same_cpu_crypt && fail echo -e "$PWD1" | $CRYPTSETUP refresh $DEV $DEV_NAME2 2>/dev/null && fail + if [ -n "$DM_PERF_NO_WORKQUEUE" ]; then + echo -e "$PWD1" | $CRYPTSETUP refresh $DEV_NAME --perf-no_read_workqueue --perf-no_write_workqueue || fail + $CRYPTSETUP status $DEV_NAME | grep -q no_read_workqueue || fail + $CRYPTSETUP status $DEV_NAME | grep -q no_write_workqueue || fail + fi $CRYPTSETUP close $DEV_NAME || fail format luks2 @@ -214,6 +229,15 @@ else echo -e "$PWD1" | $CRYPTSETUP refresh $DEV $DEV_NAME || fail $CRYPTSETUP status $DEV_NAME | grep -q keyring || fail fi + if [ -n "$DM_PERF_NO_WORKQUEUE" ]; then + echo -e "$PWD1" | $CRYPTSETUP refresh $DEV $DEV_NAME --perf-no_read_workqueue --perf-no_write_workqueue --persistent || fail + $CRYPTSETUP status $DEV_NAME | grep -q no_read_workqueue || fail + $CRYPTSETUP status $DEV_NAME | grep -q no_write_workqueue || fail + $CRYPTSETUP close $DEV_NAME || fail + echo -e "$PWD1" | $CRYPTSETUP open $DEV $DEV_NAME || fail + $CRYPTSETUP status $DEV_NAME | grep -q no_read_workqueue || fail + $CRYPTSETUP status $DEV_NAME | grep -q no_write_workqueue || fail + fi echo -e "$PWD1" | $CRYPTSETUP refresh $DEV $DEV_NAME2 2>/dev/null && fail $CRYPTSETUP close $DEV_NAME || fail fi