Remove VLAs from FEC verity code.

lib/verity/rs_decode_char.c

@@ -25,15 +25,19 @@
 
 #include "rs.h"
 
+#define MAX_NR_BUF 256
+
 int decode_rs_char(struct rs* rs, data_t* data)
 {
 	int deg_lambda, el, deg_omega, syn_error, count;
 	int i, j, r, k;
 	data_t q, tmp, num1, num2, den, discr_r;
-	/* FIXME: remove VLAs here */
+	data_t lambda[MAX_NR_BUF], s[MAX_NR_BUF]; /* Err+Eras Locator poly and syndrome poly */
-	data_t lambda[rs->nroots + 1], s[rs->nroots]; /* Err+Eras Locator poly and syndrome poly */
+	data_t b[MAX_NR_BUF], t[MAX_NR_BUF], omega[MAX_NR_BUF];
-	data_t b[rs->nroots + 1], t[rs->nroots + 1], omega[rs->nroots + 1];
+	data_t root[MAX_NR_BUF], reg[MAX_NR_BUF], loc[MAX_NR_BUF];
-	data_t root[rs->nroots], reg[rs->nroots + 1], loc[rs->nroots];
+
+	if (rs->nroots >= MAX_NR_BUF)
+		return -1;
 
 	memset(s, 0, rs->nroots * sizeof(data_t));
 	memset(b, 0, (rs->nroots + 1) * sizeof(data_t));

lib/verity/verity_fec.c

@@ -178,6 +178,7 @@ static int FEC_process_inputs(struct crypt_device *cd,
 				r = decode_rs_char(rs, rs_block);
 				if (r < 0) {
 					log_err(cd, _("Failed to repair parity for block %" PRIu64 "."), n);
+					r = -EPERM;
 					goto out;
 				}
 				/* return number of detected errors */

lib/verity/verity_hash.c

@@ -28,6 +28,7 @@
 #include "internal.h"
 
 #define VERITY_MAX_LEVELS	63
+#define VERITY_MAX_DIGEST_SIZE	1024
 
 static unsigned get_bits_up(size_t u)
 {
@@ -47,20 +48,30 @@ static unsigned get_bits_down(size_t u)
 
 static int verify_zero(struct crypt_device *cd, FILE *wr, size_t bytes)
 {
-	char block[bytes];
+	char *block = NULL;
 	size_t i;
+	int r;
+
+	block = malloc(bytes);
+	if (!block)
+		return -ENOMEM;
 
 	if (fread(block, bytes, 1, wr) != 1) {
 		log_dbg(cd, "EIO while reading spare area.");
-		return -EIO;
+		r = -EIO;
+		goto out;
 	}
 	for (i = 0; i < bytes; i++)
 		if (block[i]) {
 			log_err(cd, _("Spare area is not zeroed at position %" PRIu64 "."),
 				ftello(wr) - bytes);
-			return -EPERM;
+			r = -EPERM;
+			goto out;
 		}
-	return 0;
+	r = 0;
+out:
+	free(block);
+	return r;
 }
 
 static int verify_hash_block(const char *hash_name, int version,
@@ -138,9 +149,8 @@ static int create_or_verify(struct crypt_device *cd, FILE *rd, FILE *wr,
 				   char *calculated_digest, size_t digest_size,
 				   const char *salt, size_t salt_size)
 {
-	char left_block[hash_block_size];
+	char *left_block, *data_buffer;
-	char data_buffer[data_block_size];
+	char read_digest[VERITY_MAX_DIGEST_SIZE];
-	char read_digest[digest_size];
 	size_t hash_per_block = 1 << get_bits_down(hash_block_size / digest_size);
 	size_t digest_size_full = 1 << get_bits_up(digest_size);
 	uint64_t blocks_to_write = (blocks + hash_per_block - 1) / hash_per_block;
@@ -149,6 +159,9 @@ static int create_or_verify(struct crypt_device *cd, FILE *rd, FILE *wr,
 	unsigned i;
 	int r;
 
+	if (digest_size > sizeof(read_digest))
+		return -EINVAL;
+
 	if (uint64_mult_overflow(&seek_rd, data_block, data_block_size) ||
 	    uint64_mult_overflow(&seek_wr, hash_block, hash_block_size)) {
 		log_err(cd, _("Device offset overflow."));
@@ -165,6 +178,13 @@ static int create_or_verify(struct crypt_device *cd, FILE *rd, FILE *wr,
 		return -EIO;
 	}
 
+	left_block = malloc(hash_block_size);
+	data_buffer = malloc(data_block_size);
+	if (!left_block || !data_buffer) {
+		r = -ENOMEM;
+		goto out;
+	}
+
 	memset(left_block, 0, hash_block_size);
 	while (blocks_to_write--) {
 		left_bytes = hash_block_size;
@@ -174,31 +194,37 @@ static int create_or_verify(struct crypt_device *cd, FILE *rd, FILE *wr,
 			blocks--;
 			if (fread(data_buffer, data_block_size, 1, rd) != 1) {
 				log_dbg(cd, "Cannot read data device block.");
-				return -EIO;
+				r = -EIO;
+				goto out;
 			}
 
 			if (verify_hash_block(hash_name, version,
 					calculated_digest, digest_size,
 					data_buffer, data_block_size,
-					salt, salt_size))
+					salt, salt_size)) {
-				return -EINVAL;
+				r = -EINVAL;
+				goto out;
+			}
 
 			if (!wr)
 				break;
 			if (verify) {
 				if (fread(read_digest, digest_size, 1, wr) != 1) {
 					log_dbg(cd, "Cannot read digest form hash device.");
-					return -EIO;
+					r = -EIO;
+					goto out;
 				}
 				if (memcmp(read_digest, calculated_digest, digest_size)) {
 					log_err(cd, _("Verification failed at position %" PRIu64 "."),
 						ftello(rd) - data_block_size);
-					return -EPERM;
+					r = -EPERM;
+					goto out;
 				}
 			} else {
 				if (fwrite(calculated_digest, digest_size, 1, wr) != 1) {
 					log_dbg(cd, "Cannot write digest to hash device.");
-					return -EIO;
+					r = -EIO;
+					goto out;
 				}
 			}
 			if (version == 0) {
@@ -208,10 +234,11 @@ static int create_or_verify(struct crypt_device *cd, FILE *rd, FILE *wr,
 					if (verify) {
 						r = verify_zero(cd, wr, digest_size_full - digest_size);
 						if (r)
-							return r;
+							goto out;
 					} else if (fwrite(left_block, digest_size_full - digest_size, 1, wr) != 1) {
 						log_dbg(cd, "Cannot write spare area to hash device.");
-						return -EIO;
+						r = -EIO;
+						goto out;
 					}
 				}
 				left_bytes -= digest_size_full;
@@ -221,22 +248,26 @@ static int create_or_verify(struct crypt_device *cd, FILE *rd, FILE *wr,
 			if (verify) {
 				r = verify_zero(cd , wr, left_bytes);
 				if (r)
-					return r;
+					goto out;
 			} else if (fwrite(left_block, left_bytes, 1, wr) != 1) {
 				log_dbg(cd, "Cannot write remaining spare area to hash device.");
-				return -EIO;
+				r = -EIO;
+				goto out;
 			}
 		}
 	}
-
+	r = 0;
-	return 0;
+out:
+	free(left_block);
+	free(data_buffer);
+	return r;
 }
 
 static int VERITY_create_or_verify_hash(struct crypt_device *cd, bool verify,
 	struct crypt_params_verity *params,
 	char *root_hash, size_t digest_size)
 {
-	char calculated_digest[digest_size];
+	char calculated_digest[VERITY_MAX_DIGEST_SIZE];
 	FILE *data_file = NULL;
 	FILE *hash_file = NULL, *hash_file_2;
 	uint64_t hash_level_block[VERITY_MAX_LEVELS];
@@ -253,6 +284,9 @@ static int VERITY_create_or_verify_hash(struct crypt_device *cd, bool verify,
 		device_path(crypt_data_device(cd)), params->data_size,
 		device_path(crypt_metadata_device(cd)), hash_position);
 
+	if (digest_size > sizeof(calculated_digest))
+		return -EINVAL;
+
 	if (!params->data_size) {
 		r = device_size(crypt_data_device(cd), &dev_size);
 		if (r < 0)