Add no_read/write_wrokqueue to dm-crypt options.

These performance options, introduced in kernel 5.9, configures dm-crypt to bypass read or write workqueues and run encryption synchronously. Also support persistent storage of these flags for LUKS2.
2025-12-05 16:00:05 +01:00 · 2020-08-20 20:58:44 +02:00
parent 3c54d8a239
commit d5729bdf01
9 changed files with 97 additions and 8 deletions
--- a/lib/libcryptsetup.h
+++ b/lib/libcryptsetup.h
@@ -1105,6 +1105,10 @@ int crypt_keyslot_destroy(struct crypt_device *cd, int keyslot);
 #define CRYPT_ACTIVATE_IV_LARGE_SECTORS (1 << 22)
 /** dm-verity: panic_on_corruption flag - panic kernel on corruption */
 #define CRYPT_ACTIVATE_PANIC_ON_CORRUPTION (1 << 23)
+/** dm-crypt: bypass internal workqueue and process read requests synchronously. */
+#define CRYPT_ACTIVATE_NO_READ_WORKQUEUE (1 << 24)
+/** dm-crypt: bypass internal workqueue and process write requests synchronously. */
+#define CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE (1 << 25)

 /**
 * Active device runtime attributes
--- a/lib/libdevmapper.c
+++ b/lib/libdevmapper.c
@@ -174,6 +174,9 @@ static void _dm_set_crypt_compat(struct crypt_device *cd,
 	if (_dm_satisfies_version(1, 20, 0, crypt_maj, crypt_min, crypt_patch))
 		_dm_flags |= DM_BITLK_ELEPHANT_SUPPORTED;

+	if (_dm_satisfies_version(1, 22, 0, crypt_maj, crypt_min, crypt_patch))
+		_dm_flags |= DM_CRYPT_NO_WORKQUEUE_SUPPORTED;
+
 	_dm_crypt_checked = true;
 }

@@ -618,6 +621,10 @@ static char *get_dm_crypt_params(const struct dm_target *tgt, uint32_t flags)
 		num_options++;
 	if (flags & CRYPT_ACTIVATE_SUBMIT_FROM_CRYPT_CPUS)
 		num_options++;
+	if (flags & CRYPT_ACTIVATE_NO_READ_WORKQUEUE)
+		num_options++;
+	if (flags & CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE)
+		num_options++;
 	if (flags & CRYPT_ACTIVATE_IV_LARGE_SECTORS)
 		num_options++;
 	if (tgt->u.crypt.integrity)
@@ -630,10 +637,12 @@ static char *get_dm_crypt_params(const struct dm_target *tgt, uint32_t flags)
 		*sector_feature = '\0';

 	if (num_options) {
-		snprintf(features, sizeof(features)-1, " %d%s%s%s%s%s%s", num_options,
+		snprintf(features, sizeof(features)-1, " %d%s%s%s%s%s%s%s%s", num_options,
 		(flags & CRYPT_ACTIVATE_ALLOW_DISCARDS) ? " allow_discards" : "",
 		(flags & CRYPT_ACTIVATE_SAME_CPU_CRYPT) ? " same_cpu_crypt" : "",
 		(flags & CRYPT_ACTIVATE_SUBMIT_FROM_CRYPT_CPUS) ? " submit_from_crypt_cpus" : "",
+		(flags & CRYPT_ACTIVATE_NO_READ_WORKQUEUE) ? " no_read_workqueue" : "",
+		(flags & CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE) ? " no_write_workqueue" : "",
 		(flags & CRYPT_ACTIVATE_IV_LARGE_SECTORS) ? " iv_large_sectors" : "",
 		sector_feature, integrity_dm);
 	} else
@@ -1610,6 +1619,14 @@ static int check_retry(struct crypt_device *cd, uint32_t *dmd_flags, uint32_t dm
 		ret = 1;
 	}

+	/* Drop no workqueue options if not supported */
+	if ((*dmd_flags & (CRYPT_ACTIVATE_NO_READ_WORKQUEUE | CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE)) &&
+	    !(dmt_flags & DM_CRYPT_NO_WORKQUEUE_SUPPORTED)) {
+		log_dbg(cd, "dm-crypt does not support performance options");
+		*dmd_flags = *dmd_flags & ~(CRYPT_ACTIVATE_NO_READ_WORKQUEUE | CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE);
+		ret = 1;
+	}
+
 	return ret;
 }

@@ -1632,14 +1649,21 @@ int dm_create_device(struct crypt_device *cd, const char *name,
 		goto out;

 	if (r && (dmd->segment.type == DM_CRYPT || dmd->segment.type == DM_LINEAR || dmd->segment.type == DM_ZERO) &&
-		check_retry(cd, &dmd->flags, dmt_flags))
+		check_retry(cd, &dmd->flags, dmt_flags)) {
+		log_dbg(cd, "Retrying open without incompatible options.");
 		r = _dm_create_device(cd, name, type, dmd->uuid, dmd);
+	}

 	if (r == -EINVAL &&
 	    dmd->flags & (CRYPT_ACTIVATE_SAME_CPU_CRYPT|CRYPT_ACTIVATE_SUBMIT_FROM_CRYPT_CPUS) &&
 	    !(dmt_flags & (DM_SAME_CPU_CRYPT_SUPPORTED|DM_SUBMIT_FROM_CRYPT_CPUS_SUPPORTED)))
 		log_err(cd, _("Requested dm-crypt performance options are not supported."));

+	if (r == -EINVAL &&
+	    dmd->flags & (CRYPT_ACTIVATE_NO_READ_WORKQUEUE | CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE) &&
+	    !(dmt_flags & DM_CRYPT_NO_WORKQUEUE_SUPPORTED))
+		log_err(cd, _("Requested dm-crypt performance options are not supported."));
+
 	if (r == -EINVAL && dmd->flags & (CRYPT_ACTIVATE_IGNORE_CORRUPTION|
 					  CRYPT_ACTIVATE_RESTART_ON_CORRUPTION|
 					  CRYPT_ACTIVATE_IGNORE_ZERO_BLOCKS|
@@ -1697,7 +1721,10 @@ int dm_reload_device(struct crypt_device *cd, const char *name,

 	if (r == -EINVAL && (dmd->segment.type == DM_CRYPT || dmd->segment.type == DM_LINEAR)) {
 		if ((dmd->flags & (CRYPT_ACTIVATE_SAME_CPU_CRYPT|CRYPT_ACTIVATE_SUBMIT_FROM_CRYPT_CPUS)) &&
-	    !dm_flags(cd, DM_CRYPT, &dmt_flags) && !(dmt_flags & (DM_SAME_CPU_CRYPT_SUPPORTED|DM_SUBMIT_FROM_CRYPT_CPUS_SUPPORTED)))
+		    !dm_flags(cd, DM_CRYPT, &dmt_flags) && !(dmt_flags & (DM_SAME_CPU_CRYPT_SUPPORTED | DM_SUBMIT_FROM_CRYPT_CPUS_SUPPORTED)))
+			log_err(cd, _("Requested dm-crypt performance options are not supported."));
+		if ((dmd->flags & (CRYPT_ACTIVATE_NO_READ_WORKQUEUE | CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE)) &&
+		    !dm_flags(cd, DM_CRYPT, &dmt_flags) && !(dmt_flags & DM_CRYPT_NO_WORKQUEUE_SUPPORTED))
 			log_err(cd, _("Requested dm-crypt performance options are not supported."));
 		if ((dmd->flags & CRYPT_ACTIVATE_ALLOW_DISCARDS) &&
 		    !dm_flags(cd, DM_CRYPT, &dmt_flags) && !(dmt_flags & DM_DISCARDS_SUPPORTED))
@@ -1941,6 +1968,10 @@ static int _dm_target_query_crypt(struct crypt_device *cd, uint32_t get_flags,
 				*act_flags |= CRYPT_ACTIVATE_SAME_CPU_CRYPT;
 			else if (!strcasecmp(arg, "submit_from_crypt_cpus"))
 				*act_flags |= CRYPT_ACTIVATE_SUBMIT_FROM_CRYPT_CPUS;
+			else if (!strcasecmp(arg, "no_read_workqueue"))
+				*act_flags |= CRYPT_ACTIVATE_NO_READ_WORKQUEUE;
+			else if (!strcasecmp(arg, "no_write_workqueue"))
+				*act_flags |= CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE;
 			else if (!strcasecmp(arg, "iv_large_sectors"))
 				*act_flags |= CRYPT_ACTIVATE_IV_LARGE_SECTORS;
 			else if (sscanf(arg, "integrity:%u:", &val) == 1) {
--- a/lib/luks2/luks2_json_metadata.c
+++ b/lib/luks2/luks2_json_metadata.c
@@ -1304,6 +1304,8 @@ static const struct  {
 	{ CRYPT_ACTIVATE_SAME_CPU_CRYPT,         "same-cpu-crypt" },
 	{ CRYPT_ACTIVATE_SUBMIT_FROM_CRYPT_CPUS, "submit-from-crypt-cpus" },
 	{ CRYPT_ACTIVATE_NO_JOURNAL,             "no-journal" },
+	{ CRYPT_ACTIVATE_NO_READ_WORKQUEUE,      "no-read-workqueue" },
+	{ CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE,     "no-write-workqueue" },
 	{ 0, NULL }
 };

--- a/lib/utils_dm.h
+++ b/lib/utils_dm.h
@@ -70,6 +70,7 @@ static inline uint32_t act2dmflags(uint32_t act_flags)
 #define DM_VERITY_SIGNATURE_SUPPORTED (1 << 22) /* Verity option root_hash_sig_key_desc supported */
 #define DM_INTEGRITY_DISCARDS_SUPPORTED (1 << 23) /* dm-integrity discards/TRIM option is supported */
 #define DM_VERITY_PANIC_CORRUPTION_SUPPORTED (1 << 24) /* dm-verity panic on corruption  */
+#define DM_CRYPT_NO_WORKQUEUE_SUPPORTED (1 << 25) /* dm-crypt suppot for bypassing workqueues  */

 typedef enum { DM_CRYPT = 0, DM_VERITY, DM_INTEGRITY, DM_LINEAR, DM_ERROR, DM_ZERO, DM_UNKNOWN } dm_target_type;
 enum tdirection { TARGET_SET = 1, TARGET_QUERY };
--- a/man/cryptsetup.8
+++ b/man/cryptsetup.8
@@ -146,7 +146,8 @@ Mandatory parametrs are identical to those of an open action for respective
 device type.

 You may change following parameters on all devices \-\-perf\-same_cpu_crypt,
-\-\-perf\-submit_from_crypt_cpus and \-\-allow\-discards.
+\-\-perf\-submit_from_crypt_cpus, \-\-perf-no_read_workqueue, \-\-no_write_workqueue
+and \-\-allow\-discards.

 Refreshing device without any optional parameter will refresh the device
 with default setting (respective to device type).
@@ -1209,6 +1210,15 @@ This option is only relevant for \fIopen\fR action.
 performance tuning, use only if you need a change to default dm-crypt
 behaviour. Needs kernel 4.0 or later.
 .TP
+.B "\-\-perf\-no_read_workqueue, \-\-perf\-no_write_workqueue\fR"
+Bypass dm-crypt internal workqueue and process read or write requests
+synchronously.
+This option is only relevant for \fIopen\fR action.
+
+\fBNOTE:\fR These options are available only for low-level dm-crypt
+performance tuning, use only if you need a change to default dm-crypt
+behaviour. Needs kernel 5.9 or later.
+.TP
 .B "\-\-test\-passphrase\fR"
 Do not activate the device, just verify passphrase.
 This option is only relevant for \fIopen\fR action (the device
@@ -1325,7 +1335,8 @@ the flag you want to remove (e.g. to disable persistently stored discard flag,
 use \fI\-\-persistent\fR without \fI\-\-allow-discards\fR).

 Only \fI\-\-allow-discards\fR, \fI\-\-perf\-same_cpu_crypt\fR,
-\fI\-\-perf\-submit_from_crypt_cpus\fR and \fI\-\-integrity\-no\-journal\fR
+\fI\-\-perf\-submit_from_crypt_cpus\fR, \fI\-\-perf\-no_read_workqueue\fR,
+\fI\-\-perf\-no_write_workqueue\fR and \fI\-\-integrity\-no\-journal\fR
 can be stored persistently.
 .TP
 .B "\-\-refresh"
--- a/src/cryptsetup.c
+++ b/src/cryptsetup.c
@@ -111,6 +111,12 @@ static void _set_activation_flags(uint32_t *flags)
 	if (ARG_SET(OPT_PERF_SUBMIT_FROM_CRYPT_CPUS_ID))
 		*flags |= CRYPT_ACTIVATE_SUBMIT_FROM_CRYPT_CPUS;

+	if (ARG_SET(OPT_PERF_NO_READ_WORKQUEUE_ID))
+		*flags |= CRYPT_ACTIVATE_NO_READ_WORKQUEUE;
+
+	if (ARG_SET(OPT_PERF_NO_WRITE_WORKQUEUE_ID))
+		*flags |= CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE;
+
 	if (ARG_SET(OPT_INTEGRITY_NO_JOURNAL_ID))
 		*flags |= CRYPT_ACTIVATE_NO_JOURNAL;

@@ -766,11 +772,15 @@ static int action_status(void)
 					   (cad.flags & CRYPT_ACTIVATE_SUSPENDED) ? " (suspended)" : "");
 		if (cad.flags & (CRYPT_ACTIVATE_ALLOW_DISCARDS|
 				 CRYPT_ACTIVATE_SAME_CPU_CRYPT|
-				 CRYPT_ACTIVATE_SUBMIT_FROM_CRYPT_CPUS))
-			log_std("  flags:   %s%s%s\n",
+				 CRYPT_ACTIVATE_SUBMIT_FROM_CRYPT_CPUS|
+				 CRYPT_ACTIVATE_NO_READ_WORKQUEUE|
+				 CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE))
+			log_std("  flags:   %s%s%s%s%s\n",
 				(cad.flags & CRYPT_ACTIVATE_ALLOW_DISCARDS) ? "discards " : "",
 				(cad.flags & CRYPT_ACTIVATE_SAME_CPU_CRYPT) ? "same_cpu_crypt " : "",
-				(cad.flags & CRYPT_ACTIVATE_SUBMIT_FROM_CRYPT_CPUS) ? "submit_from_crypt_cpus" : "");
+				(cad.flags & CRYPT_ACTIVATE_SUBMIT_FROM_CRYPT_CPUS) ? "submit_from_crypt_cpus " : "",
+				(cad.flags & CRYPT_ACTIVATE_NO_READ_WORKQUEUE) ? "no_read_workqueue " : "",
+				(cad.flags & CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE) ? "no_write_workqueue" : "");
 	}
 out:
 	crypt_free(cd);
--- a/src/cryptsetup_arg_list.h
+++ b/src/cryptsetup_arg_list.h
@@ -113,6 +113,10 @@ ARG(OPT_PBKDF_MEMORY, '\0', POPT_ARG_STRING, N_("PBKDF memory cost limit"), N_("

 ARG(OPT_PBKDF_PARALLEL, '\0', POPT_ARG_STRING, N_("PBKDF parallel cost"), N_("threads"), CRYPT_ARG_UINT32, { .u32_value = DEFAULT_LUKS2_PARALLEL_THREADS }, {})

+ARG(OPT_PERF_NO_READ_WORKQUEUE, '\0', POPT_ARG_NONE, N_("Bypass dm-crypt workqueue and process read requests synchronously"), NULL, CRYPT_ARG_BOOL, {}, {})
+
+ARG(OPT_PERF_NO_WRITE_WORKQUEUE, '\0', POPT_ARG_NONE, N_("Bypass dm-crypt workqueue and process write requests synchronously"), NULL, CRYPT_ARG_BOOL, {}, {})
+
 ARG(OPT_PERF_SAME_CPU_CRYPT, '\0', POPT_ARG_NONE, N_("Use dm-crypt same_cpu_crypt performance compatibility option"), NULL, CRYPT_ARG_BOOL, {}, {})

 ARG(OPT_PERF_SUBMIT_FROM_CRYPT_CPUS, '\0', POPT_ARG_NONE, N_("Use dm-crypt submit_from_crypt_cpus performance compatibility option"), NULL, CRYPT_ARG_BOOL, {}, {})
--- a/src/utils_arg_names.h
+++ b/src/utils_arg_names.h
@@ -104,6 +104,8 @@
 #define OPT_PBKDF_FORCE_ITERATIONS	"pbkdf-force-iterations"
 #define OPT_PBKDF_MEMORY		"pbkdf-memory"
 #define OPT_PBKDF_PARALLEL		"pbkdf-parallel"
+#define OPT_PERF_NO_READ_WORKQUEUE	"perf-no_read_workqueue"
+#define OPT_PERF_NO_WRITE_WORKQUEUE	"perf-no_write_workqueue"
 #define OPT_PERF_SAME_CPU_CRYPT		"perf-same_cpu_crypt"
 #define OPT_PERF_SUBMIT_FROM_CRYPT_CPUS	"perf-submit_from_crypt_cpus"
 #define OPT_PERSISTENT			"persistent"
--- a/tests/device-test
+++ b/tests/device-test
@@ -38,6 +38,7 @@ skip()

 function dm_crypt_features()
 {
+	modprobe dm-crypt || fail "dm-crypt failed to load"
 	VER_STR=$(dmsetup targets | grep crypt | cut -f2 -dv)
 	[ -z "$VER_STR" ] && fail "Failed to parse dm-crypt version."

@@ -61,6 +62,9 @@ function dm_crypt_features()
 	if [ $VER_MIN -gt 18 -o \( $VER_MIN -eq 18 -a $VER_PTC -ge 1 \) ]; then
 		 test -d /proc/sys/kernel/keys && DM_KEYRING=1
 	fi
+
+	[ $VER_MIN -lt 22 ] && return
+	DM_PERF_NO_WORKQUEUE=1
 }

 function dm_crypt_keyring_support()
@@ -141,7 +145,13 @@ else
 	$CRYPTSETUP status $DEV_NAME | grep -q discards && fail
 	$CRYPTSETUP status $DEV_NAME | grep -q same_cpu_crypt && fail
 	echo -e "$PWD1" | $CRYPTSETUP refresh --hash sha256 $DEV $DEV_NAME2 2>/dev/null && fail
+	if [ -n "$DM_PERF_NO_WORKQUEUE" ]; then
+		echo -e "$PWD1" | $CRYPTSETUP refresh --hash sha256 -q $DEV_NAME --perf-no_read_workqueue --perf-no_write_workqueue || fail
+		$CRYPTSETUP status $DEV_NAME | grep -q no_read_workqueue || fail
+		$CRYPTSETUP status $DEV_NAME | grep -q no_write_workqueue || fail
+	fi
 	$CRYPTSETUP close $DEV_NAME || fail
+
 	# LUKS
 	echo -e "$PWD1" | $CRYPTSETUP open --type luks1 $DEV $DEV_NAME --perf-same_cpu_crypt --perf-submit_from_crypt_cpus || fail
 	$CRYPTSETUP status $DEV_NAME | grep -q same_cpu_crypt || fail
@@ -162,6 +172,11 @@ else
 	$CRYPTSETUP status $DEV_NAME | grep -q discards && fail
 	$CRYPTSETUP status $DEV_NAME | grep -q same_cpu_crypt && fail
 	echo -e "$PWD1" | $CRYPTSETUP refresh $DEV $DEV_NAME2 2>/dev/null && fail
+	if [ -n "$DM_PERF_NO_WORKQUEUE" ]; then
+		echo -e "$PWD1" | $CRYPTSETUP refresh $DEV_NAME  --perf-no_read_workqueue --perf-no_write_workqueue || fail
+		$CRYPTSETUP status $DEV_NAME | grep -q no_read_workqueue || fail
+		$CRYPTSETUP status $DEV_NAME | grep -q no_write_workqueue || fail
+	fi
 	$CRYPTSETUP close $DEV_NAME || fail

 	format luks2
@@ -214,6 +229,15 @@ else
 		echo -e "$PWD1" | $CRYPTSETUP refresh $DEV $DEV_NAME || fail
 		$CRYPTSETUP status $DEV_NAME | grep -q keyring || fail
 	fi
+	if [ -n "$DM_PERF_NO_WORKQUEUE" ]; then
+		echo -e "$PWD1" | $CRYPTSETUP refresh $DEV $DEV_NAME --perf-no_read_workqueue --perf-no_write_workqueue --persistent || fail
+		$CRYPTSETUP status $DEV_NAME | grep -q no_read_workqueue || fail
+		$CRYPTSETUP status $DEV_NAME | grep -q no_write_workqueue || fail
+		$CRYPTSETUP close $DEV_NAME || fail
+		echo -e "$PWD1" | $CRYPTSETUP open $DEV $DEV_NAME || fail
+		$CRYPTSETUP status $DEV_NAME | grep -q no_read_workqueue || fail
+		$CRYPTSETUP status $DEV_NAME | grep -q no_write_workqueue || fail
+	fi
 	echo -e "$PWD1" | $CRYPTSETUP refresh $DEV $DEV_NAME2 2>/dev/null && fail
 	$CRYPTSETUP close $DEV_NAME || fail
 fi