Add no_read/write_wrokqueue to dm-crypt options.

These performance options, introduced in kernel 5.9, configures
dm-crypt to bypass read or write workqueues and run encryption
synchronously.

Also support persistent storage of these flags for LUKS2.

lib/libcryptsetup.h

@@ -1105,6 +1105,10 @@ int crypt_keyslot_destroy(struct crypt_device *cd, int keyslot);
 #define CRYPT_ACTIVATE_IV_LARGE_SECTORS (1 << 22)
 /** dm-verity: panic_on_corruption flag - panic kernel on corruption */
 #define CRYPT_ACTIVATE_PANIC_ON_CORRUPTION (1 << 23)
+/** dm-crypt: bypass internal workqueue and process read requests synchronously. */
+#define CRYPT_ACTIVATE_NO_READ_WORKQUEUE (1 << 24)
+/** dm-crypt: bypass internal workqueue and process write requests synchronously. */
+#define CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE (1 << 25)
 
 /**
  * Active device runtime attributes

lib/libdevmapper.c

@@ -174,6 +174,9 @@ static void _dm_set_crypt_compat(struct crypt_device *cd,
 	if (_dm_satisfies_version(1, 20, 0, crypt_maj, crypt_min, crypt_patch))
 		_dm_flags |= DM_BITLK_ELEPHANT_SUPPORTED;
 
+	if (_dm_satisfies_version(1, 22, 0, crypt_maj, crypt_min, crypt_patch))
+		_dm_flags |= DM_CRYPT_NO_WORKQUEUE_SUPPORTED;
+
 	_dm_crypt_checked = true;
 }
 
@@ -618,6 +621,10 @@ static char *get_dm_crypt_params(const struct dm_target *tgt, uint32_t flags)
 		num_options++;
 	if (flags & CRYPT_ACTIVATE_SUBMIT_FROM_CRYPT_CPUS)
 		num_options++;
+	if (flags & CRYPT_ACTIVATE_NO_READ_WORKQUEUE)
+		num_options++;
+	if (flags & CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE)
+		num_options++;
 	if (flags & CRYPT_ACTIVATE_IV_LARGE_SECTORS)
 		num_options++;
 	if (tgt->u.crypt.integrity)
@@ -630,10 +637,12 @@ static char *get_dm_crypt_params(const struct dm_target *tgt, uint32_t flags)
 		*sector_feature = '\0';
 
 	if (num_options) {
-		snprintf(features, sizeof(features)-1, " %d%s%s%s%s%s%s", num_options,
+		snprintf(features, sizeof(features)-1, " %d%s%s%s%s%s%s%s%s", num_options,
 		(flags & CRYPT_ACTIVATE_ALLOW_DISCARDS) ? " allow_discards" : "",
 		(flags & CRYPT_ACTIVATE_SAME_CPU_CRYPT) ? " same_cpu_crypt" : "",
 		(flags & CRYPT_ACTIVATE_SUBMIT_FROM_CRYPT_CPUS) ? " submit_from_crypt_cpus" : "",
+		(flags & CRYPT_ACTIVATE_NO_READ_WORKQUEUE) ? " no_read_workqueue" : "",
+		(flags & CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE) ? " no_write_workqueue" : "",
 		(flags & CRYPT_ACTIVATE_IV_LARGE_SECTORS) ? " iv_large_sectors" : "",
 		sector_feature, integrity_dm);
 	} else
@@ -1610,6 +1619,14 @@ static int check_retry(struct crypt_device *cd, uint32_t *dmd_flags, uint32_t dm
 		ret = 1;
 	}
 
+	/* Drop no workqueue options if not supported */
+	if ((*dmd_flags & (CRYPT_ACTIVATE_NO_READ_WORKQUEUE | CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE)) &&
+	    !(dmt_flags & DM_CRYPT_NO_WORKQUEUE_SUPPORTED)) {
+		log_dbg(cd, "dm-crypt does not support performance options");
+		*dmd_flags = *dmd_flags & ~(CRYPT_ACTIVATE_NO_READ_WORKQUEUE | CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE);
+		ret = 1;
+	}
+
 	return ret;
 }
 
@@ -1632,14 +1649,21 @@ int dm_create_device(struct crypt_device *cd, const char *name,
 		goto out;
 
 	if (r && (dmd->segment.type == DM_CRYPT || dmd->segment.type == DM_LINEAR || dmd->segment.type == DM_ZERO) &&
-		check_retry(cd, &dmd->flags, dmt_flags))
+		check_retry(cd, &dmd->flags, dmt_flags)) {
+		log_dbg(cd, "Retrying open without incompatible options.");
 		r = _dm_create_device(cd, name, type, dmd->uuid, dmd);
+	}
 
 	if (r == -EINVAL &&
 	    dmd->flags & (CRYPT_ACTIVATE_SAME_CPU_CRYPT|CRYPT_ACTIVATE_SUBMIT_FROM_CRYPT_CPUS) &&
 	    !(dmt_flags & (DM_SAME_CPU_CRYPT_SUPPORTED|DM_SUBMIT_FROM_CRYPT_CPUS_SUPPORTED)))
 		log_err(cd, _("Requested dm-crypt performance options are not supported."));
 
+	if (r == -EINVAL &&
+	    dmd->flags & (CRYPT_ACTIVATE_NO_READ_WORKQUEUE | CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE) &&
+	    !(dmt_flags & DM_CRYPT_NO_WORKQUEUE_SUPPORTED))
+		log_err(cd, _("Requested dm-crypt performance options are not supported."));
+
 	if (r == -EINVAL && dmd->flags & (CRYPT_ACTIVATE_IGNORE_CORRUPTION|
 					  CRYPT_ACTIVATE_RESTART_ON_CORRUPTION|
 					  CRYPT_ACTIVATE_IGNORE_ZERO_BLOCKS|
@@ -1697,7 +1721,10 @@ int dm_reload_device(struct crypt_device *cd, const char *name,
 
 	if (r == -EINVAL && (dmd->segment.type == DM_CRYPT || dmd->segment.type == DM_LINEAR)) {
 		if ((dmd->flags & (CRYPT_ACTIVATE_SAME_CPU_CRYPT|CRYPT_ACTIVATE_SUBMIT_FROM_CRYPT_CPUS)) &&
-	    !dm_flags(cd, DM_CRYPT, &dmt_flags) && !(dmt_flags & (DM_SAME_CPU_CRYPT_SUPPORTED|DM_SUBMIT_FROM_CRYPT_CPUS_SUPPORTED)))
+		    !dm_flags(cd, DM_CRYPT, &dmt_flags) && !(dmt_flags & (DM_SAME_CPU_CRYPT_SUPPORTED | DM_SUBMIT_FROM_CRYPT_CPUS_SUPPORTED)))
+			log_err(cd, _("Requested dm-crypt performance options are not supported."));
+		if ((dmd->flags & (CRYPT_ACTIVATE_NO_READ_WORKQUEUE | CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE)) &&
+		    !dm_flags(cd, DM_CRYPT, &dmt_flags) && !(dmt_flags & DM_CRYPT_NO_WORKQUEUE_SUPPORTED))
 			log_err(cd, _("Requested dm-crypt performance options are not supported."));
 		if ((dmd->flags & CRYPT_ACTIVATE_ALLOW_DISCARDS) &&
 		    !dm_flags(cd, DM_CRYPT, &dmt_flags) && !(dmt_flags & DM_DISCARDS_SUPPORTED))
@@ -1941,6 +1968,10 @@ static int _dm_target_query_crypt(struct crypt_device *cd, uint32_t get_flags,
 				*act_flags |= CRYPT_ACTIVATE_SAME_CPU_CRYPT;
 			else if (!strcasecmp(arg, "submit_from_crypt_cpus"))
 				*act_flags |= CRYPT_ACTIVATE_SUBMIT_FROM_CRYPT_CPUS;
+			else if (!strcasecmp(arg, "no_read_workqueue"))
+				*act_flags |= CRYPT_ACTIVATE_NO_READ_WORKQUEUE;
+			else if (!strcasecmp(arg, "no_write_workqueue"))
+				*act_flags |= CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE;
 			else if (!strcasecmp(arg, "iv_large_sectors"))
 				*act_flags |= CRYPT_ACTIVATE_IV_LARGE_SECTORS;
 			else if (sscanf(arg, "integrity:%u:", &val) == 1) {

lib/luks2/luks2_json_metadata.c

@@ -1289,6 +1289,8 @@ static const struct  {
 	{ CRYPT_ACTIVATE_SAME_CPU_CRYPT,         "same-cpu-crypt" },
 	{ CRYPT_ACTIVATE_SUBMIT_FROM_CRYPT_CPUS, "submit-from-crypt-cpus" },
 	{ CRYPT_ACTIVATE_NO_JOURNAL,             "no-journal" },
+	{ CRYPT_ACTIVATE_NO_READ_WORKQUEUE,      "no-read-workqueue" },
+	{ CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE,     "no-write-workqueue" },
 	{ 0, NULL }
 };
 

lib/utils_dm.h

@@ -70,6 +70,7 @@ static inline uint32_t act2dmflags(uint32_t act_flags)
 #define DM_VERITY_SIGNATURE_SUPPORTED (1 << 22) /* Verity option root_hash_sig_key_desc supported */
 #define DM_INTEGRITY_DISCARDS_SUPPORTED (1 << 23) /* dm-integrity discards/TRIM option is supported */
 #define DM_VERITY_PANIC_CORRUPTION_SUPPORTED (1 << 24) /* dm-verity panic on corruption  */
+#define DM_CRYPT_NO_WORKQUEUE_SUPPORTED (1 << 25) /* dm-crypt suppot for bypassing workqueues  */
 
 typedef enum { DM_CRYPT = 0, DM_VERITY, DM_INTEGRITY, DM_LINEAR, DM_ERROR, DM_ZERO, DM_UNKNOWN } dm_target_type;
 enum tdirection { TARGET_SET = 1, TARGET_QUERY };

man/cryptsetup.8

@@ -146,7 +146,8 @@ Mandatory parametrs are identical to those of an open action for respective
 device type.
 
 You may change following parameters on all devices \-\-perf\-same_cpu_crypt,
-\-\-perf\-submit_from_crypt_cpus and \-\-allow\-discards.
+\-\-perf\-submit_from_crypt_cpus, \-\-perf-no_read_workqueue, \-\-no_write_workqueue
+and \-\-allow\-discards.
 
 Refreshing device without any optional parameter will refresh the device
 with default setting (respective to device type).
@@ -1209,6 +1210,15 @@ This option is only relevant for \fIopen\fR action.
 performance tuning, use only if you need a change to default dm-crypt
 behaviour. Needs kernel 4.0 or later.
 .TP
+.B "\-\-perf\-no_read_workqueue, \-\-perf\-no_write_workqueue\fR"
+Bypass dm-crypt internal workqueue and process read or write requests
+synchronously.
+This option is only relevant for \fIopen\fR action.
+
+\fBNOTE:\fR These options are available only for low-level dm-crypt
+performance tuning, use only if you need a change to default dm-crypt
+behaviour. Needs kernel 5.9 or later.
+.TP
 .B "\-\-test\-passphrase\fR"
 Do not activate the device, just verify passphrase.
 This option is only relevant for \fIopen\fR action (the device
@@ -1325,7 +1335,8 @@ the flag you want to remove (e.g. to disable persistently stored discard flag,
 use \fI\-\-persistent\fR without \fI\-\-allow-discards\fR).
 
 Only \fI\-\-allow-discards\fR, \fI\-\-perf\-same_cpu_crypt\fR,
-\fI\-\-perf\-submit_from_crypt_cpus\fR and \fI\-\-integrity\-no\-journal\fR
+\fI\-\-perf\-submit_from_crypt_cpus\fR, \fI\-\-perf\-no_read_workqueue\fR,
+\fI\-\-perf\-no_write_workqueue\fR and \fI\-\-integrity\-no\-journal\fR
 can be stored persistently.
 .TP
 .B "\-\-refresh"

src/cryptsetup.c

@@ -64,6 +64,8 @@ static int opt_shared = 0;
 static int opt_allow_discards = 0;
 static int opt_perf_same_cpu_crypt = 0;
 static int opt_perf_submit_from_crypt_cpus = 0;
+static int opt_perf_no_read_workqueue = 0;
+static int opt_perf_no_write_workqueue = 0;
 static int opt_test_passphrase = 0;
 static int opt_tcrypt_hidden = 0;
 static int opt_tcrypt_system = 0;
@@ -182,6 +184,12 @@ static void _set_activation_flags(uint32_t *flags)
 	if (opt_perf_submit_from_crypt_cpus)
 		*flags |= CRYPT_ACTIVATE_SUBMIT_FROM_CRYPT_CPUS;
 
+	if (opt_perf_no_read_workqueue)
+		*flags |= CRYPT_ACTIVATE_NO_READ_WORKQUEUE;
+
+	if (opt_perf_no_write_workqueue)
+		*flags |= CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE;
+
 	if (opt_integrity_nojournal)
 		*flags |= CRYPT_ACTIVATE_NO_JOURNAL;
 
@@ -815,11 +823,15 @@ static int action_status(void)
 					   (cad.flags & CRYPT_ACTIVATE_SUSPENDED) ? " (suspended)" : "");
 		if (cad.flags & (CRYPT_ACTIVATE_ALLOW_DISCARDS|
 				 CRYPT_ACTIVATE_SAME_CPU_CRYPT|
-				 CRYPT_ACTIVATE_SUBMIT_FROM_CRYPT_CPUS))
+				 CRYPT_ACTIVATE_SUBMIT_FROM_CRYPT_CPUS|
-			log_std("  flags:   %s%s%s\n",
+				 CRYPT_ACTIVATE_NO_READ_WORKQUEUE|
+				 CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE))
+			log_std("  flags:   %s%s%s%s%s\n",
 				(cad.flags & CRYPT_ACTIVATE_ALLOW_DISCARDS) ? "discards " : "",
 				(cad.flags & CRYPT_ACTIVATE_SAME_CPU_CRYPT) ? "same_cpu_crypt " : "",
-				(cad.flags & CRYPT_ACTIVATE_SUBMIT_FROM_CRYPT_CPUS) ? "submit_from_crypt_cpus" : "");
+				(cad.flags & CRYPT_ACTIVATE_SUBMIT_FROM_CRYPT_CPUS) ? "submit_from_crypt_cpus " : "",
+				(cad.flags & CRYPT_ACTIVATE_NO_READ_WORKQUEUE) ? "no_read_workqueue " : "",
+				(cad.flags & CRYPT_ACTIVATE_NO_WRITE_WORKQUEUE) ? "no_write_workqueue" : "");
 	}
 out:
 	crypt_free(cd);
@@ -3535,6 +3547,8 @@ int main(int argc, const char **argv)
 		{ "force-password",    '\0', POPT_ARG_NONE, &opt_force_password,        0, N_("Disable password quality check (if enabled)"), NULL },
 		{ "perf-same_cpu_crypt",'\0', POPT_ARG_NONE, &opt_perf_same_cpu_crypt,  0, N_("Use dm-crypt same_cpu_crypt performance compatibility option"), NULL },
 		{ "perf-submit_from_crypt_cpus",'\0', POPT_ARG_NONE, &opt_perf_submit_from_crypt_cpus,0,N_("Use dm-crypt submit_from_crypt_cpus performance compatibility option"), NULL },
+		{ "perf-no_read_workqueue",'\0', POPT_ARG_NONE, &opt_perf_no_read_workqueue,0,N_("Bypass dm-crypt workqueue and process read requests synchronously"), NULL },
+		{ "perf-no_write_workqueue",'\0', POPT_ARG_NONE, &opt_perf_no_write_workqueue,0,N_("Bypass dm-crypt workqueue and process write requests synchronously"), NULL },
 		{ "deferred",          '\0', POPT_ARG_NONE, &opt_deferred_remove,       0, N_("Device removal is deferred until the last user closes it"), NULL },
 		{ "serialize-memory-hard-pbkdf", '\0', POPT_ARG_NONE, &opt_serialize_memory_hard_pbkdf, 0, N_("Use global lock to serialize memory hard PBKDF (OOM workaround)"), NULL },
 		{ "iter-time",         'i',  POPT_ARG_INT, &opt_iteration_time,         0, N_("PBKDF iteration time for LUKS (in ms)"), N_("msecs") },

tests/device-test

@@ -38,6 +38,7 @@ skip()
 
 function dm_crypt_features()
 {
+	modprobe dm-crypt || fail "dm-crypt failed to load"
 	VER_STR=$(dmsetup targets | grep crypt | cut -f2 -dv)
 	[ -z "$VER_STR" ] && fail "Failed to parse dm-crypt version."
 
@@ -61,6 +62,9 @@ function dm_crypt_features()
 	if [ $VER_MIN -gt 18 -o \( $VER_MIN -eq 18 -a $VER_PTC -ge 1 \) ]; then
 		 test -d /proc/sys/kernel/keys && DM_KEYRING=1
 	fi
+
+	[ $VER_MIN -lt 22 ] && return
+	DM_PERF_NO_WORKQUEUE=1
 }
 
 function dm_crypt_keyring_support()
@@ -141,7 +145,13 @@ else
 	$CRYPTSETUP status $DEV_NAME | grep -q discards && fail
 	$CRYPTSETUP status $DEV_NAME | grep -q same_cpu_crypt && fail
 	echo -e "$PWD1" | $CRYPTSETUP refresh --hash sha256 $DEV $DEV_NAME2 2>/dev/null && fail
+	if [ -n "$DM_PERF_NO_WORKQUEUE" ]; then
+		echo -e "$PWD1" | $CRYPTSETUP refresh --hash sha256 -q $DEV_NAME --perf-no_read_workqueue --perf-no_write_workqueue || fail
+		$CRYPTSETUP status $DEV_NAME | grep -q no_read_workqueue || fail
+		$CRYPTSETUP status $DEV_NAME | grep -q no_write_workqueue || fail
+	fi
 	$CRYPTSETUP close $DEV_NAME || fail
+
 	# LUKS
 	echo -e "$PWD1" | $CRYPTSETUP open --type luks1 $DEV $DEV_NAME --perf-same_cpu_crypt --perf-submit_from_crypt_cpus || fail
 	$CRYPTSETUP status $DEV_NAME | grep -q same_cpu_crypt || fail
@@ -162,6 +172,11 @@ else
 	$CRYPTSETUP status $DEV_NAME | grep -q discards && fail
 	$CRYPTSETUP status $DEV_NAME | grep -q same_cpu_crypt && fail
 	echo -e "$PWD1" | $CRYPTSETUP refresh $DEV $DEV_NAME2 2>/dev/null && fail
+	if [ -n "$DM_PERF_NO_WORKQUEUE" ]; then
+		echo -e "$PWD1" | $CRYPTSETUP refresh $DEV_NAME  --perf-no_read_workqueue --perf-no_write_workqueue || fail
+		$CRYPTSETUP status $DEV_NAME | grep -q no_read_workqueue || fail
+		$CRYPTSETUP status $DEV_NAME | grep -q no_write_workqueue || fail
+	fi
 	$CRYPTSETUP close $DEV_NAME || fail
 
 	format luks2
@@ -214,6 +229,15 @@ else
 		echo -e "$PWD1" | $CRYPTSETUP refresh $DEV $DEV_NAME || fail
 		$CRYPTSETUP status $DEV_NAME | grep -q keyring || fail
 	fi
+	if [ -n "$DM_PERF_NO_WORKQUEUE" ]; then
+		echo -e "$PWD1" | $CRYPTSETUP refresh $DEV $DEV_NAME --perf-no_read_workqueue --perf-no_write_workqueue --persistent || fail
+		$CRYPTSETUP status $DEV_NAME | grep -q no_read_workqueue || fail
+		$CRYPTSETUP status $DEV_NAME | grep -q no_write_workqueue || fail
+		$CRYPTSETUP close $DEV_NAME || fail
+		echo -e "$PWD1" | $CRYPTSETUP open $DEV $DEV_NAME || fail
+		$CRYPTSETUP status $DEV_NAME | grep -q no_read_workqueue || fail
+		$CRYPTSETUP status $DEV_NAME | grep -q no_write_workqueue || fail
+	fi
 	echo -e "$PWD1" | $CRYPTSETUP refresh $DEV $DEV_NAME2 2>/dev/null && fail
 	$CRYPTSETUP close $DEV_NAME || fail
 fi