Add global serialization lock for memory hard PBKDF.

This is very ugly workaround for situation when multiple
devices are being activated in parallel (systemd crypttab)
and system  instead of returning ENOMEM use OOM killer
to randomly kill processes.

This flag is intended to be used only in very specific situations.

lib/internal.h

@@ -236,4 +236,7 @@ static inline uint64_t version(uint16_t major, uint16_t minor, uint16_t patch, u
 
 int kernel_version(uint64_t *kversion);
 
+int crypt_serialize_lock(struct crypt_device *cd);
+void crypt_serialize_unlock(struct crypt_device *cd);
+
 #endif /* INTERNAL_H */

lib/libcryptsetup.h

@@ -1057,6 +1057,8 @@ int crypt_keyslot_destroy(struct crypt_device *cd, int keyslot);
 #define CRYPT_ACTIVATE_RECALCULATE (1 << 17)
 /** reactivate existing and update flags, input only */
 #define CRYPT_ACTIVATE_REFRESH	(1 << 18)
+/** Use global lock to serialize memory hard KDF on activation (OOM workaround) */
+#define CRYPT_ACTIVATE_SERIALIZE_MEMORY_HARD_PBKDF (1 << 19)
 
 /**
  * Active device runtime attributes

lib/luks2/luks2_keyslot_luks2.c

@@ -339,6 +339,12 @@ static int luks2_keyslot_get_key(struct crypt_device *cd,
 		return -EINVAL;
 	keyslot_key_len = json_object_get_int(jobj2);
 
+	/*
+	 * If requested, serialize unlocking for memory-hard KDF. Usually NOOP.
+	 */
+	if (pbkdf.max_memory_kb && crypt_serialize_lock(cd))
+		return -EINVAL;
+
 	/*
 	 * Allocate derived key storage space.
 	 */
@@ -361,6 +367,9 @@ static int luks2_keyslot_get_key(struct crypt_device *cd,
 			pbkdf.iterations, pbkdf.max_memory_kb,
 			pbkdf.parallel_threads);
 
+	if (pbkdf.max_memory_kb)
+		crypt_serialize_unlock(cd);
+
 	if (r == 0) {
 		log_dbg(cd, "Reading keyslot area [0x%04x].", (unsigned)area_offset);
 		/* FIXME: sector_offset should be size_t, fix LUKS_decrypt... accordingly */

lib/setup.c

@@ -36,6 +36,7 @@
 #include "verity.h"
 #include "tcrypt.h"
 #include "integrity.h"
+#include "utils_device_locking.h"
 #include "internal.h"
 
 #define CRYPT_CD_UNRESTRICTED	(1 << 0)
@@ -58,6 +59,10 @@ struct crypt_device {
 	uint64_t metadata_size; /* Used in LUKS2 format */
 	uint64_t keyslots_size; /* Used in LUKS2 format */
 
+	/* Workaround for OOM during parallel activation (like in systemd) */
+	bool memory_hard_pbkdf_lock_enabled;
+	struct crypt_lock_handle *pbkdf_memory_hard_lock;
+
 	// FIXME: private binary headers and access it properly
 	// through sub-library (LUKS1, TCRYPT)
 
@@ -3641,10 +3646,14 @@ static int _activate_by_passphrase(struct crypt_device *cd,
 	if (r < 0)
 		return r;
 
+	if (flags & CRYPT_ACTIVATE_SERIALIZE_MEMORY_HARD_PBKDF)
+		cd->memory_hard_pbkdf_lock_enabled = true;
+
 	/* plain, use hashed passphrase */
 	if (isPLAIN(cd->type)) {
+		r = -EINVAL;
 		if (!name)
-			return -EINVAL;
+			goto out;
 
 		r = process_key(cd, cd->u.plain.hdr.hash,
 				cd->u.plain.key_size,
@@ -3691,6 +3700,8 @@ out:
 		crypt_drop_keyring_key(cd, vk);
 	crypt_free_volume_key(vk);
 
+	cd->memory_hard_pbkdf_lock_enabled = false;
+
 	return r < 0 ? r : keyslot;
 }
 
@@ -5481,6 +5492,35 @@ int crypt_activate_by_keyring(struct crypt_device *cd,
 	return r;
 }
 
+/*
+ * Workaround for serialization of parallel activation and memory-hard PBKDF
+ * In specific situation (systemd activation) this causes OOM killer activation.
+ * For now, let's provide this ugly way to serialize unlocking of devices.
+ */
+int crypt_serialize_lock(struct crypt_device *cd)
+{
+	if (!cd->memory_hard_pbkdf_lock_enabled )
+		return 0;
+
+	log_dbg(cd, "Taking global memory-hard access serialization lock.");
+	if (crypt_write_lock(cd, "memory-hard-access", true, &cd->pbkdf_memory_hard_lock)) {
+		log_err(cd, "Failed to acquire global memory-hard access serialization lock.");
+		cd->pbkdf_memory_hard_lock = NULL;
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+void crypt_serialize_unlock(struct crypt_device *cd)
+{
+	if (!cd->memory_hard_pbkdf_lock_enabled )
+		return;
+
+	crypt_unlock_internal(cd, cd->pbkdf_memory_hard_lock);
+	cd->pbkdf_memory_hard_lock = NULL;
+}
+
 static void __attribute__((destructor)) libcryptsetup_exit(void)
 {
 	crypt_backend_destroy();

man/cryptsetup.8

@@ -276,7 +276,8 @@ the command prompts for it interactively.
 \fB<options>\fR can be [\-\-key\-file, \-\-keyfile\-offset,
 \-\-keyfile\-size, \-\-readonly, \-\-test\-passphrase,
 \-\-allow\-discards, \-\-header, \-\-key-slot, \-\-master\-key\-file, \-\-token\-id,
-\-\-token\-only, \-\-disable\-keyring, \-\-disable\-locks, \-\-type, \-\-refresh].
+\-\-token\-only, \-\-disable\-keyring, \-\-disable\-locks, \-\-type, \-\-refresh,
+\-\-serialize\-memory\-hard\-pbkdf].
 .PP
 \fIluksSuspend\fR <name>
 .IP
@@ -1284,6 +1285,16 @@ See \fITCRYPT\fR section for more info.
 Use a custom Personal Iteration Multiplier (PIM) for VeraCrypt device.
 See \fITCRYPT\fR section for more info.
 .TP
+.B "\-\-serialize\-memory\-hard\-pbkdf"
+Use a global lock to serialize unlocking of keyslots using memory-hard PBKDF.
+
+\fBNOTE:\fR This is (ugly) workaround for a specific situation when multiple
+devices are activated in parallel and system instead of reporting out of memory
+starts unconditionally stop processes using out-of-memory killer.
+
+\fBDO NOT USE\fR this switch until you are implementing boot environment
+with parallel devices activation!
+.TP
 .B "\-\-version"
 Show the program version.
 .TP

src/cryptsetup.c

@@ -73,6 +73,7 @@ static int opt_veracrypt = 0;
 static int opt_veracrypt_pim = -1;
 static int opt_veracrypt_query_pim = 0;
 static int opt_deferred_remove = 0;
+static int opt_serialize_memory_hard_pbkdf = 0;
 //FIXME: check uint32 overflow for long type
 static const char *opt_pbkdf = NULL;
 static long opt_pbkdf_memory = DEFAULT_LUKS2_MEMORY_KB;
@@ -169,6 +170,9 @@ static void _set_activation_flags(uint32_t *flags)
 	/* Only for LUKS2 but ignored elsewhere */
 	if (opt_test_passphrase)
 		*flags |= CRYPT_ACTIVATE_ALLOW_UNBOUND_KEY;
+
+	if (opt_serialize_memory_hard_pbkdf)
+		*flags |= CRYPT_ACTIVATE_SERIALIZE_MEMORY_HARD_PBKDF;
 }
 
 static int _set_keyslot_encryption_params(struct crypt_device *cd)
@@ -2545,6 +2549,7 @@ int main(int argc, const char **argv)
 		{ "perf-same_cpu_crypt",'\0', POPT_ARG_NONE, &opt_perf_same_cpu_crypt,  0, N_("Use dm-crypt same_cpu_crypt performance compatibility option"), NULL },
 		{ "perf-submit_from_crypt_cpus",'\0', POPT_ARG_NONE, &opt_perf_submit_from_crypt_cpus,0,N_("Use dm-crypt submit_from_crypt_cpus performance compatibility option"), NULL },
 		{ "deferred",          '\0', POPT_ARG_NONE, &opt_deferred_remove,       0, N_("Device removal is deferred until the last user closes it"), NULL },
+		{ "serialize-memory-hard-pbkdf", '\0', POPT_ARG_NONE, &opt_serialize_memory_hard_pbkdf, 0, N_("Use global lock to serialize memory hard PBKDF (OOM workaround)"), NULL },
 		{ "iter-time",         'i',  POPT_ARG_INT, &opt_iteration_time,         0, N_("PBKDF iteration time for LUKS (in ms)"), N_("msecs") },
 		{ "pbkdf",             '\0', POPT_ARG_STRING, &opt_pbkdf,               0, N_("PBKDF algorithm (for LUKS2): argon2i, argon2id, pbkdf2"), NULL },
 		{ "pbkdf-memory",      '\0', POPT_ARG_LONG, &opt_pbkdf_memory,          0, N_("PBKDF memory cost limit"), N_("kilobytes") },
@@ -2743,6 +2748,11 @@ int main(int argc, const char **argv)
 		      _("Option --persistent is allowed only for open operation.\n"),
 		      poptGetInvocationName(popt_context));
 
+	if (opt_serialize_memory_hard_pbkdf && strcmp(aname, "open"))
+		usage(popt_context, EXIT_FAILURE,
+		      _("Option --serialize-memory-hard-pbkdf is allowed only for open operation.\n"),
+		      poptGetInvocationName(popt_context));
+
 	if (opt_persistent && opt_test_passphrase)
 		usage(popt_context, EXIT_FAILURE,
 		      _("Option --persistent is not allowed with --test-passphrase.\n"),