Add low-level performance options for dmcrypt tuning.

The patch adds the two options
  --perf-same_cpu_crypt
  --perf-submit_from_crypt_cpus
that set the same named options inside dmcrypt
(available in Linux kernel 3.20 and later).

lib/libcryptsetup.h

@@ -3,8 +3,8 @@
  *
  * Copyright (C) 2004, Jana Saout <jana@saout.de>
  * Copyright (C) 2004-2007, Clemens Fruhwirth <clemens@endorphin.org>
- * Copyright (C) 2009-2012, Red Hat, Inc. All rights reserved.
- * Copyright (C) 2009-2014, Milan Broz
+ * Copyright (C) 2009-2015, Red Hat, Inc. All rights reserved.
+ * Copyright (C) 2009-2015, Milan Broz
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -739,6 +739,11 @@ int crypt_keyslot_destroy(struct crypt_device *cd, int keyslot);
 #define CRYPT_ACTIVATE_PRIVATE (1 << 4)
 /** corruption detected (verity), output only */
 #define CRYPT_ACTIVATE_CORRUPTED (1 << 5)
+/** use same_cpu_crypt option for dm-crypt */
+#define CRYPT_ACTIVATE_SAME_CPU_CRYPT (1 << 6)
+/** use submit_from_crypt_cpus for dm-crypt */
+#define CRYPT_ACTIVATE_SUBMIT_FROM_CRYPT_CPUS (1 << 7)
+
 
 /**
  * Active device runtime attributes

lib/libdevmapper.c

@@ -3,8 +3,8 @@
  *
  * Copyright (C) 2004, Jana Saout <jana@saout.de>
  * Copyright (C) 2004-2007, Clemens Fruhwirth <clemens@endorphin.org>
- * Copyright (C) 2009-2012, Red Hat, Inc. All rights reserved.
- * Copyright (C) 2009-2012, Milan Broz
+ * Copyright (C) 2009-2015, Red Hat, Inc. All rights reserved.
+ * Copyright (C) 2009-2015, Milan Broz
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -143,6 +143,11 @@ static void _dm_set_crypt_compat(const char *dm_version, unsigned crypt_maj,
 	if (_dm_satisfies_version(1, 13, crypt_maj, crypt_min))
 		_dm_crypt_flags |= DM_TCW_SUPPORTED;
 
+	if (_dm_satisfies_version(1, 14, crypt_maj, crypt_min)) {
+		_dm_crypt_flags |= DM_SAME_CPU_CRYPT_SUPPORTED;
+		_dm_crypt_flags |= DM_SUBMIT_FROM_CRYPT_CPUS_SUPPORTED;
+	}
+
 	/* Repeat test if dm-crypt is not present */
 	if (crypt_maj > 0)
 		_dm_crypt_checked = 1;
@@ -295,17 +300,27 @@ static void hex_key(char *hexkey, size_t key_size, const char *key)
 /* http://code.google.com/p/cryptsetup/wiki/DMCrypt */
 static char *get_dm_crypt_params(struct crypt_dm_active_device *dmd, uint32_t flags)
 {
-	int r, max_size, null_cipher = 0;
+	int r, max_size, null_cipher = 0, num_options = 0;
 	char *params, *hexkey;
-	const char *features;
+	char features[256];
 
 	if (!dmd)
 		return NULL;
 
 	if (flags & CRYPT_ACTIVATE_ALLOW_DISCARDS)
-		features = " 1 allow_discards";
+		num_options++;
+	if (flags & CRYPT_ACTIVATE_SAME_CPU_CRYPT)
+		num_options++;
+	if (flags & CRYPT_ACTIVATE_SUBMIT_FROM_CRYPT_CPUS)
+		num_options++;
+
+	if (num_options)
+		snprintf(features, sizeof(features)-1, " %d%s%s%s", num_options,
+		(flags & CRYPT_ACTIVATE_ALLOW_DISCARDS) ? " allow_discards" : "",
+		(flags & CRYPT_ACTIVATE_SAME_CPU_CRYPT) ? " same_cpu_crypt" : "",
+		(flags & CRYPT_ACTIVATE_SUBMIT_FROM_CRYPT_CPUS) ? " submit_from_crypt_cpus" : "");
 	else
-		features = "";
+		*features = '\0';
 
 	if (!strncmp(dmd->u.crypt.cipher, "cipher_null-", 12))
 		null_cipher = 1;
@@ -677,6 +692,11 @@ int dm_create_device(struct crypt_device *cd, const char *name,
 				      dmd->uuid, dmd->size, table_params, reload);
 	}
 
+	if (r == -EINVAL &&
+	    dmd_flags & (CRYPT_ACTIVATE_SAME_CPU_CRYPT|CRYPT_ACTIVATE_SUBMIT_FROM_CRYPT_CPUS) &&
+	    !(dm_flags() & (DM_SAME_CPU_CRYPT_SUPPORTED|DM_SUBMIT_FROM_CRYPT_CPUS_SUPPORTED)))
+		log_err(cd, _("Requested dmcrypt performance options are not supported.\n"));
+
 	crypt_safe_free(table_params);
 	dm_exit_context();
 	return r;
@@ -864,6 +884,10 @@ static int _dm_query_crypt(uint32_t get_flags,
 			arg = strsep(&params, " ");
 			if (!strcasecmp(arg, "allow_discards"))
 				dmd->flags |= CRYPT_ACTIVATE_ALLOW_DISCARDS;
+			else if (!strcasecmp(arg, "same_cpu_crypt"))
+				dmd->flags |= CRYPT_ACTIVATE_SAME_CPU_CRYPT;
+			else if (!strcasecmp(arg, "submit_from_crypt_cpus"))
+				dmd->flags |= CRYPT_ACTIVATE_SUBMIT_FROM_CRYPT_CPUS;
 			else /* unknown option */
 				return -EINVAL;
 		}

lib/utils_dm.h

@@ -3,8 +3,8 @@
  *
  * Copyright (C) 2004, Jana Saout <jana@saout.de>
  * Copyright (C) 2004-2007, Clemens Fruhwirth <clemens@endorphin.org>
- * Copyright (C) 2009-2012, Red Hat, Inc. All rights reserved.
- * Copyright (C) 2009-2012, Milan Broz
+ * Copyright (C) 2009-2015, Red Hat, Inc. All rights reserved.
+ * Copyright (C) 2009-2015, Milan Broz
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
@@ -40,6 +40,9 @@ struct device;
 #define DM_DISCARDS_SUPPORTED (1 << 4)	/* discards/TRIM option is supported */
 #define DM_VERITY_SUPPORTED   (1 << 5)	/* dm-verity target supported */
 #define DM_TCW_SUPPORTED      (1 << 6)	/* tcw (TCRYPT CBC with whitening) */
+#define DM_SAME_CPU_CRYPT_SUPPORTED (1 << 7) /* same_cpu_crypt */
+#define DM_SUBMIT_FROM_CRYPT_CPUS_SUPPORTED (1 << 8) /* submit_from_crypt_cpus */
+
 uint32_t dm_flags(void);
 
 #define DM_ACTIVE_DEVICE	(1 << 0)

man/cryptsetup.8

@@ -813,6 +813,28 @@ later. If in doubt, do not use it.
 A kernel version of 3.1 or later is needed. For earlier kernels
 this option is ignored.
 .TP
+.B "\-\-perf\-same_cpu_crypt\fR"
+Perform encryption using the same cpu that IO was submitted on.
+The default is to use an unbound workqueue so that encryption work
+is automatically balanced between available CPUs.
+This option is only relevant for \fIopen\fR action.
+
+\fBNOTE:\fR This option is available only for low-level dm-crypt
+performance tuning, use only if you need a change to default dm-crypt
+behaviour. Needs kernel 3.20 or later.
+.TP
+.B "\-\-perf\-submit_from_crypt_cpus\fR"
+Disable offloading writes to a separate thread after encryption.
+There are some situations where offloading write bios from the
+encryption threads to a single thread degrades performance
+significantly.  The default is to offload write bios to the same
+thread.
+This option is only relevant for \fIopen\fR action.
+
+\fBNOTE:\fR This option is available only for low-level dm-crypt
+performance tuning, use only if you need a change to default dm-crypt
+behaviour. Needs kernel 3.20 or later.
+.TP
 .B "\-\-test\-passphrase\fR"
 Do not activate device, just verify passphrase.
 This option is only relevant for \fIopen\fR action (the device
@@ -1037,7 +1059,7 @@ Copyright \(co 2004 Jana Saout
 .br
 Copyright \(co 2004-2006 Clemens Fruhwirth
 .br
-Copyright \(co 2009-2012 Red Hat, Inc.
+Copyright \(co 2009-2015 Red Hat, Inc.
 .br
 Copyright \(co 2009-2015 Milan Broz
 .br

src/cryptsetup.c

@@ -3,7 +3,7 @@
  *
  * Copyright (C) 2004, Jana Saout <jana@saout.de>
  * Copyright (C) 2004-2007, Clemens Fruhwirth <clemens@endorphin.org>
- * Copyright (C) 2009-2012, Red Hat, Inc. All rights reserved.
+ * Copyright (C) 2009-2015, Red Hat, Inc. All rights reserved.
  * Copyright (C) 2009-2015, Milan Broz
  *
  * This program is free software; you can redistribute it and/or
@@ -57,6 +57,8 @@ static int opt_urandom = 0;
 static int opt_dump_master_key = 0;
 static int opt_shared = 0;
 static int opt_allow_discards = 0;
+static int opt_perf_same_cpu_crypt = 0;
+static int opt_perf_submit_from_crypt_cpus = 0;
 static int opt_test_passphrase = 0;
 static int opt_tcrypt_hidden = 0;
 static int opt_tcrypt_system = 0;
@@ -92,6 +94,21 @@ static int _verify_passphrase(int def)
 	return def;
 }
 
+static void _set_activation_flags(uint32_t *flags)
+{
+	if (opt_readonly)
+		*flags |= CRYPT_ACTIVATE_READONLY;
+
+	if (opt_allow_discards)
+		*flags |= CRYPT_ACTIVATE_ALLOW_DISCARDS;
+
+	if (opt_perf_same_cpu_crypt)
+		*flags |= CRYPT_ACTIVATE_SAME_CPU_CRYPT;
+
+	if (opt_perf_submit_from_crypt_cpus)
+		*flags |= CRYPT_ACTIVATE_SUBMIT_FROM_CRYPT_CPUS;
+}
+
 static int action_open_plain(void)
 {
 	struct crypt_device *cd = NULL;
@@ -149,14 +166,10 @@ static int action_open_plain(void)
 	if (r < 0)
 		goto out;
 
-	if (opt_readonly)
-		activate_flags |= CRYPT_ACTIVATE_READONLY;
-
 	if (opt_shared)
 		activate_flags |= CRYPT_ACTIVATE_SHARED;
 
-	if (opt_allow_discards)
-		activate_flags |= CRYPT_ACTIVATE_ALLOW_DISCARDS;
+	_set_activation_flags(&activate_flags);
 
 	if (opt_key_file) {
 		/* If no hash, key is read directly, read size is always key_size
@@ -205,11 +218,7 @@ static int action_open_loopaes(void)
 		return -EINVAL;
 	}
 
-	if (opt_readonly)
-		activate_flags |= CRYPT_ACTIVATE_READONLY;
-
-	if (opt_allow_discards)
-		activate_flags |= CRYPT_ACTIVATE_ALLOW_DISCARDS;
+	_set_activation_flags(&activate_flags);
 
 	if ((r = crypt_init(&cd, action_argv[0])))
 		goto out;
@@ -282,7 +291,7 @@ static int action_open_tcrypt(void)
 		.flags = CRYPT_TCRYPT_LEGACY_MODES,
 	};
 	const char *activated_name;
-	uint32_t flags = 0;
+	uint32_t activate_flags = 0;
 	int r;
 
 	activated_name = opt_test_passphrase ? NULL : action_argv[1];
@@ -294,14 +303,10 @@ static int action_open_tcrypt(void)
 	if (r < 0)
 		goto out;
 
-	if (opt_readonly)
-		flags |= CRYPT_ACTIVATE_READONLY;
-
-	if (opt_allow_discards)
-		flags |= CRYPT_ACTIVATE_ALLOW_DISCARDS;
+	_set_activation_flags(&activate_flags);
 
 	if (activated_name)
-		r = crypt_activate_by_volume_key(cd, activated_name, NULL, 0, flags);
+		r = crypt_activate_by_volume_key(cd, activated_name, NULL, 0, activate_flags);
 out:
 	crypt_free(cd);
 	crypt_safe_free(CONST_CAST(char*)params.passphrase);
@@ -462,8 +467,13 @@ static int action_status(void)
 			log_std("  skipped: %" PRIu64 " sectors\n", cad.iv_offset);
 		log_std("  mode:    %s\n", cad.flags & CRYPT_ACTIVATE_READONLY ?
 					   "readonly" : "read/write");
-		if (cad.flags & CRYPT_ACTIVATE_ALLOW_DISCARDS)
-			log_std("  flags:   discards\n");
+		if (cad.flags & (CRYPT_ACTIVATE_ALLOW_DISCARDS|
+				 CRYPT_ACTIVATE_ALLOW_DISCARDS|
+				 CRYPT_ACTIVATE_SUBMIT_FROM_CRYPT_CPUS))
+			log_std("  flags:   %s%s%s\n",
+				(cad.flags & CRYPT_ACTIVATE_ALLOW_DISCARDS) ? "discards " : "",
+				(cad.flags & CRYPT_ACTIVATE_SAME_CPU_CRYPT) ? "same_cpu_crypt " : "",
+				(cad.flags & CRYPT_ACTIVATE_SUBMIT_FROM_CRYPT_CPUS) ? "submit_from_crypt_cpus" : "");
 	}
 out:
 	crypt_free(cd);
@@ -750,7 +760,7 @@ static int action_open_luks(void)
 	struct crypt_device *cd = NULL;
 	const char *data_device, *header_device, *activated_name;
 	char *key = NULL;
-	uint32_t flags = 0;
+	uint32_t activate_flags = 0;
 	int r, keysize;
 
 	header_device = uuid_or_device_header(&data_device);
@@ -780,11 +790,7 @@ static int action_open_luks(void)
 	if (opt_iteration_time)
 		crypt_set_iteration_time(cd, opt_iteration_time);
 
-	if (opt_readonly)
-		flags |= CRYPT_ACTIVATE_READONLY;
-
-	if (opt_allow_discards)
-		flags |= CRYPT_ACTIVATE_ALLOW_DISCARDS;
+	_set_activation_flags(&activate_flags);
 
 	if (opt_master_key_file) {
 		keysize = crypt_get_volume_key_size(cd);
@@ -792,15 +798,15 @@ static int action_open_luks(void)
 		if (r < 0)
 			goto out;
 		r = crypt_activate_by_volume_key(cd, activated_name,
-						 key, keysize, flags);
+						 key, keysize, activate_flags);
 	} else if (opt_key_file) {
 		crypt_set_password_retry(cd, 1);
 		r = crypt_activate_by_keyfile_offset(cd, activated_name,
 			opt_key_slot, opt_key_file, opt_keyfile_size,
-			opt_keyfile_offset, flags);
+			opt_keyfile_offset, activate_flags);
 	} else
 		r = crypt_activate_by_passphrase(cd, activated_name,
-			opt_key_slot, NULL, 0, flags);
+			opt_key_slot, NULL, 0, activate_flags);
 out:
 	crypt_safe_free(key);
 	crypt_free(cd);
@@ -1501,6 +1507,8 @@ int main(int argc, const char **argv)
 		{ "tcrypt-backup",     '\0', POPT_ARG_NONE, &opt_tcrypt_backup,         0, N_("Use backup (secondary) TCRYPT header."), NULL },
 		{ "type",               'M', POPT_ARG_STRING, &opt_type,                0, N_("Type of device metadata: luks, plain, loopaes, tcrypt."), NULL },
 		{ "force-password",    '\0', POPT_ARG_NONE, &opt_force_password,        0, N_("Disable password quality check (if enabled)."), NULL },
+		{ "perf-same_cpu_crypt",'\0', POPT_ARG_NONE, &opt_perf_same_cpu_crypt,  0, N_("Use dm-crypt same_cpu_crypt performance compatibility option."), NULL },
+		{ "perf-submit_from_crypt_cpus",'\0', POPT_ARG_NONE, &opt_perf_submit_from_crypt_cpus,0,N_("Use dm-crypt submit_from_crypt_cpus performance compatibility option."), NULL },
 		POPT_TABLEEND
 	};
 	poptContext popt_context;

tests/device-test

@@ -2,10 +2,12 @@
 
 CRYPTSETUP="../src/cryptsetup"
 MNT_DIR="./mnt_luks"
+DEV_NAME="dummy"
 PWD1="93R4P4pIqAH8"
 PWD2="mymJeD8ivEhE"
 
 cleanup() {
+	[ -b /dev/mapper/$DEV_NAME ] && dmsetup remove $DEV_NAME
 	udevadm settle >/dev/null 2>&1
 	if [ -d "$MNT_DIR" ] ; then
 	    umount -f $MNT_DIR 2>/dev/null
@@ -55,4 +57,30 @@ DEV="$MNT_DIR/test.img"
 mount -t tmpfs none $MNT_DIR || skip "Mounting tmpfs not available."
 format
 
+echo "[2] Kernel dmcrypt performace options"
+echo -e "$PWD1" | $CRYPTSETUP open --type plain $DEV $DEV_NAME --perf-same_cpu_crypt >/dev/null 2>&1
+if [ $? -ne 0 ] ; then
+	echo "TEST SKIPPED: dmcrypt options not available"
+else
+	$CRYPTSETUP close $DEV_NAME || fail
+	# plain
+	echo -e "$PWD1" | $CRYPTSETUP open --type plain $DEV $DEV_NAME --perf-same_cpu_crypt --perf-submit_from_crypt_cpus || fail
+	$CRYPTSETUP status $DEV_NAME | grep -q same_cpu_crypt || fail
+	$CRYPTSETUP status $DEV_NAME | grep -q submit_from_crypt_cpus || fail
+	$CRYPTSETUP close $DEV_NAME || fail
+	echo -e "$PWD1" | $CRYPTSETUP open --type plain $DEV $DEV_NAME --perf-same_cpu_crypt --allow-discards || fail
+	$CRYPTSETUP status $DEV_NAME | grep -q same_cpu_crypt || fail
+	$CRYPTSETUP status $DEV_NAME | grep -q discards || fail
+	$CRYPTSETUP close $DEV_NAME || fail
+	# LUKS
+	echo -e "$PWD1" | $CRYPTSETUP open --type luks1 $DEV $DEV_NAME --perf-same_cpu_crypt --perf-submit_from_crypt_cpus || fail
+	$CRYPTSETUP status $DEV_NAME | grep -q same_cpu_crypt || fail
+	$CRYPTSETUP status $DEV_NAME | grep -q submit_from_crypt_cpus || fail
+	$CRYPTSETUP close $DEV_NAME || fail
+	echo -e "$PWD1" | $CRYPTSETUP open --type luks1 $DEV $DEV_NAME --perf-same_cpu_crypt --allow-discards || fail
+	$CRYPTSETUP status $DEV_NAME | grep -q same_cpu_crypt || fail
+	$CRYPTSETUP status $DEV_NAME | grep -q discards || fail
+	$CRYPTSETUP close $DEV_NAME || fail
+fi
+
 cleanup