From 4c412580fdb689638541790cda0a920d0c13bb7b Mon Sep 17 00:00:00 2001 From: Anton Khirnov Date: Wed, 10 Apr 2013 09:59:36 +0200 Subject: [PATCH 1/4] indeo3: fix data size check The data offsets are relative to the bistream header, which is 16 bytes after the start of the data. Fixes invalid reads with corrupted files. Reported-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC:libav-stable@libav.org (cherry picked from commit 34e6af9e204ca6bb18d8cf8ec68fe19b0e083e95) Signed-off-by: Reinhard Tartler --- libavcodec/indeo3.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libavcodec/indeo3.c b/libavcodec/indeo3.c index 261c651b52..2d199971c4 100644 --- a/libavcodec/indeo3.c +++ b/libavcodec/indeo3.c @@ -888,8 +888,7 @@ static int decode_frame_headers(Indeo3DecodeContext *ctx, AVCodecContext *avctx, if (ctx->data_size == 16) return 4; - if (ctx->data_size > buf_size) - ctx->data_size = buf_size; + ctx->data_size = FFMIN(ctx->data_size, buf_size - 16); buf_ptr += 3; // skip reserved byte and checksum From fa4192e31f4b9d3b6bf68daa5702926ae521e4ea Mon Sep 17 00:00:00 2001 From: Anton Khirnov Date: Tue, 9 Apr 2013 20:33:25 +0200 Subject: [PATCH 2/4] rv10: check that extradata is large enough Reported-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC:libav-stable@libav.org (cherry picked from commit 01d376f598fe95478036f5d1e3e5e14ffe32d4bf) Conflicts: libavcodec/rv10.c --- libavcodec/rv10.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/libavcodec/rv10.c b/libavcodec/rv10.c index 9239cf7d94..26af29ff26 100644 --- a/libavcodec/rv10.c +++ b/libavcodec/rv10.c @@ -347,6 +347,11 @@ static int rv20_decode_picture_header(RVDecContext *rv) f = get_bits(&s->gb, rpr_bits); if(f){ + if (s->avctx->extradata_size < 8 + 2 * f) { + av_log(s->avctx, AV_LOG_ERROR, "Extradata too small.\n"); + return AVERROR_INVALIDDATA; + } + new_w= 4*((uint8_t*)s->avctx->extradata)[6+2*f]; new_h= 4*((uint8_t*)s->avctx->extradata)[7+2*f]; }else{ From b0b33ce14806ac11860c982d701505538430437b Mon Sep 17 00:00:00 2001 From: Anton Khirnov Date: Tue, 16 Apr 2013 09:41:28 +0200 Subject: [PATCH 3/4] indeo3: check motion vectors. Reported-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC:libav-stable@libav.org (cherry picked from commit a0a872d0733f60876b0c93f236bc4606f36fbf89) Signed-off-by: Reinhard Tartler --- libavcodec/indeo3.c | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/libavcodec/indeo3.c b/libavcodec/indeo3.c index 2d199971c4..a59d404860 100644 --- a/libavcodec/indeo3.c +++ b/libavcodec/indeo3.c @@ -223,7 +223,7 @@ static av_cold void free_frame_buffers(Indeo3DecodeContext *ctx) * @param plane pointer to the plane descriptor * @param cell pointer to the cell descriptor */ -static void copy_cell(Indeo3DecodeContext *ctx, Plane *plane, Cell *cell) +static int copy_cell(Indeo3DecodeContext *ctx, Plane *plane, Cell *cell) { int h, w, mv_x, mv_y, offset, offset_dst; uint8_t *src, *dst; @@ -233,6 +233,16 @@ static void copy_cell(Indeo3DecodeContext *ctx, Plane *plane, Cell *cell) dst = plane->pixels[ctx->buf_sel] + offset_dst; mv_y = cell->mv_ptr[0]; mv_x = cell->mv_ptr[1]; + + /* -1 because there is an extra line on top for prediction */ + if ((cell->ypos << 2) + mv_y < -1 || (cell->xpos << 2) + mv_x < 0 || + ((cell->ypos + cell->height) << 2) + mv_y >= plane->height || + ((cell->xpos + cell->width) << 2) + mv_x >= plane->width) { + av_log(ctx->avctx, AV_LOG_ERROR, + "Motion vectors point out of the frame.\n"); + return AVERROR_INVALIDDATA; + } + offset = offset_dst + mv_y * plane->pitch + mv_x; src = plane->pixels[ctx->buf_sel ^ 1] + offset; @@ -260,6 +270,8 @@ static void copy_cell(Indeo3DecodeContext *ctx, Plane *plane, Cell *cell) dst += 4; } } + + return 0; } @@ -585,11 +597,23 @@ static int decode_cell(Indeo3DecodeContext *ctx, AVCodecContext *avctx, } else if (mode >= 10) { /* for mode 10 and 11 INTER first copy the predicted cell into the current one */ /* so we don't need to do data copying for each RLE code later */ - copy_cell(ctx, plane, cell); + int ret = copy_cell(ctx, plane, cell); + if (ret < 0) + return ret; } else { /* set the pointer to the reference pixels for modes 0-4 INTER */ mv_y = cell->mv_ptr[0]; mv_x = cell->mv_ptr[1]; + + /* -1 because there is an extra line on top for prediction */ + if ((cell->ypos << 2) + mv_y < -1 || (cell->xpos << 2) + mv_x < 0 || + ((cell->ypos + cell->height) << 2) + mv_y >= plane->height || + ((cell->xpos + cell->width) << 2) + mv_x >= plane->width) { + av_log(ctx->avctx, AV_LOG_ERROR, + "Motion vectors point out of the frame.\n"); + return AVERROR_INVALIDDATA; + } + offset += mv_y * plane->pitch + mv_x; ref_block = plane->pixels[ctx->buf_sel ^ 1] + offset; } @@ -721,7 +745,7 @@ static int parse_bintree(Indeo3DecodeContext *ctx, AVCodecContext *avctx, const int depth, const int strip_width) { Cell curr_cell; - int bytes_used; + int bytes_used, ret; if (depth <= 0) { av_log(avctx, AV_LOG_ERROR, "Stack overflow (corrupted binary tree)!\n"); @@ -772,8 +796,8 @@ static int parse_bintree(Indeo3DecodeContext *ctx, AVCodecContext *avctx, CHECK_CELL if (!curr_cell.mv_ptr) return AVERROR_INVALIDDATA; - copy_cell(ctx, plane, &curr_cell); - return 0; + ret = copy_cell(ctx, plane, &curr_cell); + return ret; } break; case INTER_DATA: From 95db1624ef98ccc4ba7ff70d50c4b4d0f8ffed54 Mon Sep 17 00:00:00 2001 From: Anton Khirnov Date: Wed, 10 Apr 2013 09:40:20 +0200 Subject: [PATCH 4/4] indeo3: switch parsing the header to bytestream2 Also add an additional sanity check to the alt_quant table. Fixes invalid reads with corrupted files. Reported-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC:libav-stable@libav.org (cherry picked from commit 66531d634e75b834e89e4a6a0f7470ca018712a1) Signed-off-by: Reinhard Tartler --- libavcodec/indeo3.c | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/libavcodec/indeo3.c b/libavcodec/indeo3.c index a59d404860..eacd15043a 100644 --- a/libavcodec/indeo3.c +++ b/libavcodec/indeo3.c @@ -880,17 +880,20 @@ static int decode_plane(Indeo3DecodeContext *ctx, AVCodecContext *avctx, static int decode_frame_headers(Indeo3DecodeContext *ctx, AVCodecContext *avctx, const uint8_t *buf, int buf_size) { - const uint8_t *buf_ptr = buf, *bs_hdr; + GetByteContext gb; + const uint8_t *bs_hdr; uint32_t frame_num, word2, check_sum, data_size; uint32_t y_offset, u_offset, v_offset, starts[3], ends[3]; uint16_t height, width; int i, j; + bytestream2_init(&gb, buf, buf_size); + /* parse and check the OS header */ - frame_num = bytestream_get_le32(&buf_ptr); - word2 = bytestream_get_le32(&buf_ptr); - check_sum = bytestream_get_le32(&buf_ptr); - data_size = bytestream_get_le32(&buf_ptr); + frame_num = bytestream2_get_le32(&gb); + word2 = bytestream2_get_le32(&gb); + check_sum = bytestream2_get_le32(&gb); + data_size = bytestream2_get_le32(&gb); if ((frame_num ^ word2 ^ data_size ^ OS_HDR_ID) != check_sum) { av_log(avctx, AV_LOG_ERROR, "OS header checksum mismatch!\n"); @@ -898,27 +901,27 @@ static int decode_frame_headers(Indeo3DecodeContext *ctx, AVCodecContext *avctx, } /* parse the bitstream header */ - bs_hdr = buf_ptr; + bs_hdr = gb.buffer; - if (bytestream_get_le16(&buf_ptr) != 32) { + if (bytestream2_get_le16(&gb) != 32) { av_log(avctx, AV_LOG_ERROR, "Unsupported codec version!\n"); return AVERROR_INVALIDDATA; } ctx->frame_num = frame_num; - ctx->frame_flags = bytestream_get_le16(&buf_ptr); - ctx->data_size = (bytestream_get_le32(&buf_ptr) + 7) >> 3; - ctx->cb_offset = *buf_ptr++; + ctx->frame_flags = bytestream2_get_le16(&gb); + ctx->data_size = (bytestream2_get_le32(&gb) + 7) >> 3; + ctx->cb_offset = bytestream2_get_byte(&gb); if (ctx->data_size == 16) return 4; ctx->data_size = FFMIN(ctx->data_size, buf_size - 16); - buf_ptr += 3; // skip reserved byte and checksum + bytestream2_skip(&gb, 3); // skip reserved byte and checksum /* check frame dimensions */ - height = bytestream_get_le16(&buf_ptr); - width = bytestream_get_le16(&buf_ptr); + height = bytestream2_get_le16(&gb); + width = bytestream2_get_le16(&gb); if (av_image_check_size(width, height, 0, avctx)) return AVERROR_INVALIDDATA; @@ -944,9 +947,10 @@ static int decode_frame_headers(Indeo3DecodeContext *ctx, AVCodecContext *avctx, avcodec_set_dimensions(avctx, width, height); } - y_offset = bytestream_get_le32(&buf_ptr); - v_offset = bytestream_get_le32(&buf_ptr); - u_offset = bytestream_get_le32(&buf_ptr); + y_offset = bytestream2_get_le32(&gb); + v_offset = bytestream2_get_le32(&gb); + u_offset = bytestream2_get_le32(&gb); + bytestream2_skip(&gb, 4); /* unfortunately there is no common order of planes in the buffer */ /* so we use that sorting algo for determining planes data sizes */ @@ -965,6 +969,7 @@ static int decode_frame_headers(Indeo3DecodeContext *ctx, AVCodecContext *avctx, ctx->v_data_size = ends[1] - starts[1]; ctx->u_data_size = ends[2] - starts[2]; if (FFMAX3(y_offset, v_offset, u_offset) >= ctx->data_size - 16 || + FFMIN3(y_offset, v_offset, u_offset) < gb.buffer - bs_hdr + 16 || FFMIN3(ctx->y_data_size, ctx->v_data_size, ctx->u_data_size) <= 0) { av_log(avctx, AV_LOG_ERROR, "One of the y/u/v offsets is invalid\n"); return AVERROR_INVALIDDATA; @@ -973,7 +978,7 @@ static int decode_frame_headers(Indeo3DecodeContext *ctx, AVCodecContext *avctx, ctx->y_data_ptr = bs_hdr + y_offset; ctx->v_data_ptr = bs_hdr + v_offset; ctx->u_data_ptr = bs_hdr + u_offset; - ctx->alt_quant = buf_ptr + sizeof(uint32_t); + ctx->alt_quant = gb.buffer; if (ctx->data_size == 16) { av_log(avctx, AV_LOG_DEBUG, "Sync frame encountered!\n");