From 350d06d63fc758d047c050e0835f540277799f60 Mon Sep 17 00:00:00 2001 From: Anton Khirnov Date: Thu, 8 Dec 2011 06:57:44 +0100 Subject: [PATCH 001/154] lavc: add avcodec_is_open(). It allows to check whether an AVCodecContext is open in a documented way. Right now the undocumented way this check is done in lavf/lavc is by checking whether AVCodecContext.codec is NULL. However it's desirable to be able to set AVCodecContext.codec before avcodec_open2(). (cherry picked from commit af08d9aeea870de017139f7b1c44b7d816cf8e56) Conflicts: doc/APIchanges --- doc/APIchanges | 3 +++ libavcodec/avcodec.h | 6 ++++++ libavcodec/options.c | 2 +- libavcodec/utils.c | 8 ++++++++ libavcodec/version.h | 2 +- libavformat/utils.c | 5 ++--- 6 files changed, 21 insertions(+), 5 deletions(-) diff --git a/doc/APIchanges b/doc/APIchanges index 904e3462f7..1e326cac3f 100644 --- a/doc/APIchanges +++ b/doc/APIchanges @@ -13,6 +13,9 @@ libavutil: 2011-04-18 API changes, most recent first: +2012-02-17 - xxxxxxx - lavc 53.35.0 + Add avcodec_is_open() function. + 2012-01-15 - lavc 53.34.0 New audio encoding API: b2c75b6 Add CODEC_CAP_VARIABLE_FRAME_SIZE capability for use by audio diff --git a/libavcodec/avcodec.h b/libavcodec/avcodec.h index be1b2021bd..6db34fa78e 100644 --- a/libavcodec/avcodec.h +++ b/libavcodec/avcodec.h @@ -4737,4 +4737,10 @@ enum AVMediaType avcodec_get_type(enum CodecID codec_id); */ const AVClass *avcodec_get_class(void); +/** + * @return a positive value if s is open (i.e. avcodec_open2() was called on it + * with no corresponding avcodec_close()), 0 otherwise. + */ +int avcodec_is_open(AVCodecContext *s); + #endif /* AVCODEC_AVCODEC_H */ diff --git a/libavcodec/options.c b/libavcodec/options.c index 2689d32a92..7481f1a685 100644 --- a/libavcodec/options.c +++ b/libavcodec/options.c @@ -634,7 +634,7 @@ AVCodecContext *avcodec_alloc_context(void){ int avcodec_copy_context(AVCodecContext *dest, const AVCodecContext *src) { - if (dest->codec) { // check that the dest context is uninitialized + if (avcodec_is_open(dest)) { // check that the dest context is uninitialized av_log(dest, AV_LOG_ERROR, "Tried to copy AVCodecContext %p into already-initialized %p\n", src, dest); diff --git a/libavcodec/utils.c b/libavcodec/utils.c index ff3f065064..b097c9b421 100644 --- a/libavcodec/utils.c +++ b/libavcodec/utils.c @@ -637,6 +637,9 @@ int attribute_align_arg avcodec_open2(AVCodecContext *avctx, AVCodec *codec, AVD int ret = 0; AVDictionary *tmp = NULL; + if (avcodec_is_open(avctx)) + return 0; + if (avctx->extradata_size < 0 || avctx->extradata_size >= FF_MAX_EXTRADATA_SIZE) return AVERROR(EINVAL); @@ -1836,3 +1839,8 @@ enum AVMediaType avcodec_get_type(enum CodecID codec_id) return AVMEDIA_TYPE_UNKNOWN; } + +int avcodec_is_open(AVCodecContext *s) +{ + return !!s->internal; +} diff --git a/libavcodec/version.h b/libavcodec/version.h index c7b4c15b7a..77e16823f9 100644 --- a/libavcodec/version.h +++ b/libavcodec/version.h @@ -21,7 +21,7 @@ #define AVCODEC_VERSION_H #define LIBAVCODEC_VERSION_MAJOR 53 -#define LIBAVCODEC_VERSION_MINOR 34 +#define LIBAVCODEC_VERSION_MINOR 35 #define LIBAVCODEC_VERSION_MICRO 0 #define LIBAVCODEC_VERSION_INT AV_VERSION_INT(LIBAVCODEC_VERSION_MAJOR, \ diff --git a/libavformat/utils.c b/libavformat/utils.c index 22ee13b51f..3733a50409 100644 --- a/libavformat/utils.c +++ b/libavformat/utils.c @@ -2137,7 +2137,7 @@ static int try_decode_frame(AVStream *st, AVPacket *avpkt, AVDictionary **option AVFrame picture; AVPacket pkt = *avpkt; - if(!st->codec->codec){ + if (!avcodec_is_open(st->codec)) { AVDictionary *thread_opt = NULL; codec = avcodec_find_decoder(st->codec->codec_id); @@ -2487,8 +2487,7 @@ int avformat_find_stream_info(AVFormatContext *ic, AVDictionary **options) // close codecs which were opened in try_decode_frame() for(i=0;inb_streams;i++) { st = ic->streams[i]; - if(st->codec->codec) - avcodec_close(st->codec); + avcodec_close(st->codec); } for(i=0;inb_streams;i++) { st = ic->streams[i]; From bafd38a352126385ec0dcea51017229373b1c2f3 Mon Sep 17 00:00:00 2001 From: Anton Khirnov Date: Sun, 29 Jan 2012 12:17:30 +0100 Subject: [PATCH 002/154] lavc: make avcodec_close() work properly on unopened codecs. I.e. free the priv_data and other stuff allocated in avcodec_alloc_context3() and not segfault. (cherry picked from commit 0e72ad95f9fef6a6b8ae55e47339a5c40526502f) --- libavcodec/avcodec.h | 12 +++++++++++- libavcodec/utils.c | 19 +++++++++++-------- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/libavcodec/avcodec.h b/libavcodec/avcodec.h index 6db34fa78e..95e14d7c2d 100644 --- a/libavcodec/avcodec.h +++ b/libavcodec/avcodec.h @@ -3912,7 +3912,8 @@ AVCodecContext *avcodec_alloc_context2(enum AVMediaType); /** * Allocate an AVCodecContext and set its fields to default values. The - * resulting struct can be deallocated by simply calling av_free(). + * resulting struct can be deallocated by calling avcodec_close() on it followed + * by av_free(). * * @param codec if non-NULL, allocate private data and initialize defaults * for the given codec. It is illegal to then call avcodec_open2() @@ -4343,6 +4344,15 @@ int avcodec_encode_video(AVCodecContext *avctx, uint8_t *buf, int buf_size, int avcodec_encode_subtitle(AVCodecContext *avctx, uint8_t *buf, int buf_size, const AVSubtitle *sub); +/** + * Close a given AVCodecContext and free all the data associated with it + * (but not the AVCodecContext itself). + * + * Calling this function on an AVCodecContext that hasn't been opened will free + * the codec-specific data allocated in avcodec_alloc_context3() / + * avcodec_get_context_defaults3() with a non-NULL codec. Subsequent calls will + * do nothing. + */ int avcodec_close(AVCodecContext *avctx); /** diff --git a/libavcodec/utils.c b/libavcodec/utils.c index b097c9b421..b2bd70246a 100644 --- a/libavcodec/utils.c +++ b/libavcodec/utils.c @@ -1281,14 +1281,17 @@ av_cold int avcodec_close(AVCodecContext *avctx) return -1; } - if (HAVE_THREADS && avctx->thread_opaque) - ff_thread_free(avctx); - if (avctx->codec && avctx->codec->close) - avctx->codec->close(avctx); - avcodec_default_free_buffers(avctx); - avctx->coded_frame = NULL; - av_freep(&avctx->internal); - if (avctx->codec && avctx->codec->priv_class) + if (avcodec_is_open(avctx)) { + if (HAVE_THREADS && avctx->thread_opaque) + ff_thread_free(avctx); + if (avctx->codec && avctx->codec->close) + avctx->codec->close(avctx); + avcodec_default_free_buffers(avctx); + avctx->coded_frame = NULL; + av_freep(&avctx->internal); + } + + if (avctx->priv_data && avctx->codec && avctx->codec->priv_class) av_opt_free(avctx->priv_data); av_opt_free(avctx); av_freep(&avctx->priv_data); From 571a4cf273a84b6f7f38697b462e667d4f0fddc4 Mon Sep 17 00:00:00 2001 From: Anton Khirnov Date: Sat, 28 Jan 2012 19:15:15 +0100 Subject: [PATCH 003/154] lavc: set AVCodecContext.codec in avcodec_get_context_defaults3(). This way, if the AVCodecContext is allocated for a specific codec, the caller doesn't need to store this codec separately and then pass it again to avcodec_open2(). It also allows to set codec private options using av_opt_set_* before opening the codec. (cherry picked from commit bc901998487bf9b77a423961d9f961bcc28a9291) Signed-off-by: Reinhard Tartler --- libavcodec/avcodec.h | 5 +++++ libavcodec/options.c | 1 + libavcodec/utils.c | 17 ++++++++++++----- libavformat/utils.c | 8 +++++--- 4 files changed, 23 insertions(+), 8 deletions(-) diff --git a/libavcodec/avcodec.h b/libavcodec/avcodec.h index 95e14d7c2d..2451294c1b 100644 --- a/libavcodec/avcodec.h +++ b/libavcodec/avcodec.h @@ -4059,6 +4059,11 @@ int avcodec_open(AVCodecContext *avctx, AVCodec *codec); * @endcode * * @param avctx The context to initialize. + * @param codec The codec to open this context for. If a non-NULL codec has been + * previously passed to avcodec_alloc_context3() or + * avcodec_get_context_defaults3() for this context, then this + * parameter MUST be either NULL or equal to the previously passed + * codec. * @param options A dictionary filled with AVCodecContext and codec-private options. * On return this object will be filled with options that were not found. * diff --git a/libavcodec/options.c b/libavcodec/options.c index 7481f1a685..26f3ab3b11 100644 --- a/libavcodec/options.c +++ b/libavcodec/options.c @@ -561,6 +561,7 @@ int avcodec_get_context_defaults3(AVCodecContext *s, AVCodec *codec){ s->av_class = &av_codec_context_class; s->codec_type = codec ? codec->type : AVMEDIA_TYPE_UNKNOWN; + s->codec = codec; av_opt_set_defaults(s); s->time_base = (AVRational){0,1}; diff --git a/libavcodec/utils.c b/libavcodec/utils.c index b2bd70246a..5109bf8b31 100644 --- a/libavcodec/utils.c +++ b/libavcodec/utils.c @@ -640,6 +640,18 @@ int attribute_align_arg avcodec_open2(AVCodecContext *avctx, AVCodec *codec, AVD if (avcodec_is_open(avctx)) return 0; + if ((!codec && !avctx->codec)) { + av_log(avctx, AV_LOG_ERROR, "No codec provided to avcodec_open2().\n"); + return AVERROR(EINVAL); + } + if ((codec && avctx->codec && codec != avctx->codec)) { + av_log(avctx, AV_LOG_ERROR, "This AVCodecContext was allocated for %s, " + "but %s passed to avcodec_open2().\n", avctx->codec->name, codec->name); + return AVERROR(EINVAL); + } + if (!codec) + codec = avctx->codec; + if (avctx->extradata_size < 0 || avctx->extradata_size >= FF_MAX_EXTRADATA_SIZE) return AVERROR(EINVAL); @@ -659,11 +671,6 @@ int attribute_align_arg avcodec_open2(AVCodecContext *avctx, AVCodec *codec, AVD goto end; } - if(avctx->codec || !codec) { - ret = AVERROR(EINVAL); - goto end; - } - avctx->internal = av_mallocz(sizeof(AVCodecInternal)); if (!avctx->internal) { ret = AVERROR(ENOMEM); diff --git a/libavformat/utils.c b/libavformat/utils.c index 3733a50409..f2d55028f9 100644 --- a/libavformat/utils.c +++ b/libavformat/utils.c @@ -2140,7 +2140,9 @@ static int try_decode_frame(AVStream *st, AVPacket *avpkt, AVDictionary **option if (!avcodec_is_open(st->codec)) { AVDictionary *thread_opt = NULL; - codec = avcodec_find_decoder(st->codec->codec_id); + codec = st->codec->codec ? st->codec->codec : + avcodec_find_decoder(st->codec->codec_id); + if (!codec) return -1; @@ -2306,8 +2308,8 @@ int avformat_find_stream_info(AVFormatContext *ic, AVDictionary **options) st->parser->flags |= PARSER_FLAG_COMPLETE_FRAMES; } } - assert(!st->codec->codec); - codec = avcodec_find_decoder(st->codec->codec_id); + codec = st->codec->codec ? st->codec->codec : + avcodec_find_decoder(st->codec->codec_id); /* force thread count to 1 since the h264 decoder will not extract SPS * and PPS to extradata during multi-threaded decoding */ From e364f507183634a9134eea0e004c8ae448e54469 Mon Sep 17 00:00:00 2001 From: Alex Converse Date: Wed, 25 Jan 2012 15:27:11 -0800 Subject: [PATCH 004/154] qdm2: Check data block size for bytes to bits overflow. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit dac56d9ce01eb9963f28f26b97a81db5cbd46c1c) Signed-off-by: Reinhard Tartler --- libavcodec/qdm2.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libavcodec/qdm2.c b/libavcodec/qdm2.c index 91c47a8ec2..6acb7d8362 100644 --- a/libavcodec/qdm2.c +++ b/libavcodec/qdm2.c @@ -1819,6 +1819,10 @@ static av_cold int qdm2_decode_init(AVCodecContext *avctx) extradata += 4; s->checksum_size = AV_RB32(extradata); + if (s->checksum_size >= 1U << 28) { + av_log(avctx, AV_LOG_ERROR, "data block size too large (%u)\n", s->checksum_size); + return AVERROR_INVALIDDATA; + } s->fft_order = av_log2(s->fft_size) + 1; s->fft_frame_size = 2 * s->fft_size; // complex has two floats From fc89f15497c2b5b78a992c98eaba9fca7cc82f8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Storsj=C3=B6?= Date: Thu, 26 Jan 2012 21:37:38 +0200 Subject: [PATCH 005/154] libavcodec: Don't crash in avcodec_encode_audio if time_base isn't set MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Earlier, calling avcodec_encode_audio worked fine even if time_base wasn't set. Now it crashes due to trying to scale the output pts to the codec context time base. This affects e.g. VLC. If no time_base is set for audio codecs, set it to the sample rate. CC: libav-stable@libav.org Signed-off-by: Martin Storsjö (cherry picked from commit 9a7dc618c50902e7a171f2deda6430d52c277a95) Signed-off-by: Reinhard Tartler --- libavcodec/utils.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libavcodec/utils.c b/libavcodec/utils.c index 5109bf8b31..f64bff8ff6 100644 --- a/libavcodec/utils.c +++ b/libavcodec/utils.c @@ -744,6 +744,12 @@ int attribute_align_arg avcodec_open2(AVCodecContext *avctx, AVCodec *codec, AVD avctx->error_recognition, avctx->err_recognition); #endif + if (avctx->codec_type == AVMEDIA_TYPE_AUDIO && + (!avctx->time_base.num || !avctx->time_base.den)) { + avctx->time_base.num = 1; + avctx->time_base.den = avctx->sample_rate; + } + if (HAVE_THREADS && !avctx->thread_opaque) { ret = ff_thread_init(avctx); if (ret < 0) { From a2c8db1b792670f8987c0580bb71ca0f29708d8b Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Tue, 7 Feb 2012 11:33:20 -0800 Subject: [PATCH 006/154] swscale: fix V plane memory location in bilinear/unscaled RGB/YUYV case. Fixes bug 221. CC: libav-stable@libav.org (cherry picked from commit b7542dd3d71d1ee873277020b6a8eab2674bb167) Signed-off-by: Reinhard Tartler --- libswscale/x86/swscale_template.c | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/libswscale/x86/swscale_template.c b/libswscale/x86/swscale_template.c index e38f58b5d0..5db166b00a 100644 --- a/libswscale/x86/swscale_template.c +++ b/libswscale/x86/swscale_template.c @@ -688,10 +688,10 @@ static void RENAME(yuv2yuyv422_X)(SwsContext *c, const int16_t *lumFilter, "1: \n\t"\ "movq (%2, "#index"), %%mm2 \n\t" /* uvbuf0[eax]*/\ "movq (%3, "#index"), %%mm3 \n\t" /* uvbuf1[eax]*/\ - "add "UV_OFF_PX"("#c"), "#index" \n\t" \ + "add "UV_OFF_BYTE"("#c"), "#index" \n\t" \ "movq (%2, "#index"), %%mm5 \n\t" /* uvbuf0[eax+2048]*/\ "movq (%3, "#index"), %%mm4 \n\t" /* uvbuf1[eax+2048]*/\ - "sub "UV_OFF_PX"("#c"), "#index" \n\t" \ + "sub "UV_OFF_BYTE"("#c"), "#index" \n\t" \ "psubw %%mm3, %%mm2 \n\t" /* uvbuf0[eax] - uvbuf1[eax]*/\ "psubw %%mm4, %%mm5 \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048]*/\ "movq "CHR_MMX_FILTER_OFFSET"+8("#c"), %%mm0 \n\t"\ @@ -919,10 +919,10 @@ static void RENAME(yuv2rgb565_2)(SwsContext *c, const int16_t *buf[2], "1: \n\t"\ "movq (%2, "#index"), %%mm2 \n\t" /* uvbuf0[eax]*/\ "movq (%3, "#index"), %%mm3 \n\t" /* uvbuf1[eax]*/\ - "add "UV_OFF_PX"("#c"), "#index" \n\t" \ + "add "UV_OFF_BYTE"("#c"), "#index" \n\t" \ "movq (%2, "#index"), %%mm5 \n\t" /* uvbuf0[eax+2048]*/\ "movq (%3, "#index"), %%mm4 \n\t" /* uvbuf1[eax+2048]*/\ - "sub "UV_OFF_PX"("#c"), "#index" \n\t" \ + "sub "UV_OFF_BYTE"("#c"), "#index" \n\t" \ "psubw %%mm3, %%mm2 \n\t" /* uvbuf0[eax] - uvbuf1[eax]*/\ "psubw %%mm4, %%mm5 \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048]*/\ "movq "CHR_MMX_FILTER_OFFSET"+8("#c"), %%mm0 \n\t"\ @@ -974,9 +974,9 @@ static void RENAME(yuv2yuyv422_2)(SwsContext *c, const int16_t *buf[2], ".p2align 4 \n\t"\ "1: \n\t"\ "movq (%2, "#index"), %%mm3 \n\t" /* uvbuf0[eax]*/\ - "add "UV_OFF_PX"("#c"), "#index" \n\t" \ + "add "UV_OFF_BYTE"("#c"), "#index" \n\t" \ "movq (%2, "#index"), %%mm4 \n\t" /* uvbuf0[eax+2048]*/\ - "sub "UV_OFF_PX"("#c"), "#index" \n\t" \ + "sub "UV_OFF_BYTE"("#c"), "#index" \n\t" \ "psraw $4, %%mm3 \n\t" /* uvbuf0[eax] - uvbuf1[eax] >>4*/\ "psraw $4, %%mm4 \n\t" /* uvbuf0[eax+2048] - uvbuf1[eax+2048] >>4*/\ "psubw "U_OFFSET"("#c"), %%mm3 \n\t" /* (U-128)8*/\ @@ -1027,10 +1027,10 @@ static void RENAME(yuv2yuyv422_2)(SwsContext *c, const int16_t *buf[2], "1: \n\t"\ "movq (%2, "#index"), %%mm2 \n\t" /* uvbuf0[eax]*/\ "movq (%3, "#index"), %%mm3 \n\t" /* uvbuf1[eax]*/\ - "add "UV_OFF_PX"("#c"), "#index" \n\t" \ + "add "UV_OFF_BYTE"("#c"), "#index" \n\t" \ "movq (%2, "#index"), %%mm5 \n\t" /* uvbuf0[eax+2048]*/\ "movq (%3, "#index"), %%mm4 \n\t" /* uvbuf1[eax+2048]*/\ - "sub "UV_OFF_PX"("#c"), "#index" \n\t" \ + "sub "UV_OFF_BYTE"("#c"), "#index" \n\t" \ "paddw %%mm2, %%mm3 \n\t" /* uvbuf0[eax] + uvbuf1[eax]*/\ "paddw %%mm5, %%mm4 \n\t" /* uvbuf0[eax+2048] + uvbuf1[eax+2048]*/\ "psrlw $5, %%mm3 \n\t" /*FIXME might overflow*/\ @@ -1294,9 +1294,9 @@ static void RENAME(yuv2rgb565_1)(SwsContext *c, const int16_t *buf0, ".p2align 4 \n\t"\ "1: \n\t"\ "movq (%2, "#index"), %%mm3 \n\t" /* uvbuf0[eax]*/\ - "add "UV_OFF_PX"("#c"), "#index" \n\t" \ + "add "UV_OFF_BYTE"("#c"), "#index" \n\t" \ "movq (%2, "#index"), %%mm4 \n\t" /* uvbuf0[eax+2048]*/\ - "sub "UV_OFF_PX"("#c"), "#index" \n\t" \ + "sub "UV_OFF_BYTE"("#c"), "#index" \n\t" \ "psraw $7, %%mm3 \n\t" \ "psraw $7, %%mm4 \n\t" \ "movq (%0, "#index", 2), %%mm1 \n\t" /*buf0[eax]*/\ @@ -1312,10 +1312,10 @@ static void RENAME(yuv2rgb565_1)(SwsContext *c, const int16_t *buf0, "1: \n\t"\ "movq (%2, "#index"), %%mm2 \n\t" /* uvbuf0[eax]*/\ "movq (%3, "#index"), %%mm3 \n\t" /* uvbuf1[eax]*/\ - "add "UV_OFF_PX"("#c"), "#index" \n\t" \ + "add "UV_OFF_BYTE"("#c"), "#index" \n\t" \ "movq (%2, "#index"), %%mm5 \n\t" /* uvbuf0[eax+2048]*/\ "movq (%3, "#index"), %%mm4 \n\t" /* uvbuf1[eax+2048]*/\ - "sub "UV_OFF_PX"("#c"), "#index" \n\t" \ + "sub "UV_OFF_BYTE"("#c"), "#index" \n\t" \ "paddw %%mm2, %%mm3 \n\t" /* uvbuf0[eax] + uvbuf1[eax]*/\ "paddw %%mm5, %%mm4 \n\t" /* uvbuf0[eax+2048] + uvbuf1[eax+2048]*/\ "psrlw $8, %%mm3 \n\t" \ From 4c7879775e81ccca8f0f1d2a7b70524ee47b16ca Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Thu, 9 Feb 2012 22:57:01 -0800 Subject: [PATCH 007/154] h264: disallow constrained intra prediction modes for luma. Conversion of the luma intra prediction mode to one of the constrained ("alzheimer") ones can happen by crafting special bitstreams, causing a crash because we'll call a NULL function pointer for 16x16 block intra prediction, since constrained intra prediction functions are only implemented for chroma (8x8 blocks). Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 45b7bd7c53b41bc5ff6fc2158831f2b1b1256113) Signed-off-by: Reinhard Tartler --- libavcodec/h264.c | 4 ++-- libavcodec/h264.h | 2 +- libavcodec/h264_cabac.c | 4 ++-- libavcodec/h264_cavlc.c | 4 ++-- libavcodec/svq3.c | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/libavcodec/h264.c b/libavcodec/h264.c index 581848be16..e92acbd7a8 100644 --- a/libavcodec/h264.c +++ b/libavcodec/h264.c @@ -105,7 +105,7 @@ int ff_h264_check_intra4x4_pred_mode(H264Context *h){ * Check if the top & left blocks are available if needed and * change the dc mode so it only uses the available blocks. */ -int ff_h264_check_intra_pred_mode(H264Context *h, int mode){ +int ff_h264_check_intra_pred_mode(H264Context *h, int mode, int is_chroma){ MpegEncContext * const s = &h->s; static const int8_t top [7]= {LEFT_DC_PRED8x8, 1,-1,-1}; static const int8_t left[7]= { TOP_DC_PRED8x8,-1, 2,-1,DC_128_PRED8x8}; @@ -125,7 +125,7 @@ int ff_h264_check_intra_pred_mode(H264Context *h, int mode){ if((h->left_samples_available&0x8080) != 0x8080){ mode= left[ mode ]; - if(h->left_samples_available&0x8080){ //mad cow disease mode, aka MBAFF + constrained_intra_pred + if(is_chroma && (h->left_samples_available&0x8080)){ //mad cow disease mode, aka MBAFF + constrained_intra_pred mode= ALZHEIMER_DC_L0T_PRED8x8 + (!(h->left_samples_available&0x8000)) + 2*(mode == DC_128_PRED8x8); } if(mode<0){ diff --git a/libavcodec/h264.h b/libavcodec/h264.h index 50255389fa..8680f5fdbd 100644 --- a/libavcodec/h264.h +++ b/libavcodec/h264.h @@ -657,7 +657,7 @@ int ff_h264_check_intra4x4_pred_mode(H264Context *h); /** * Check if the top & left blocks are available if needed & change the dc mode so it only uses the available blocks. */ -int ff_h264_check_intra_pred_mode(H264Context *h, int mode); +int ff_h264_check_intra_pred_mode(H264Context *h, int mode, int is_chroma); void ff_h264_hl_decode_mb(H264Context *h); int ff_h264_frame_start(H264Context *h); diff --git a/libavcodec/h264_cabac.c b/libavcodec/h264_cabac.c index a49ac6d498..75fb02cb63 100644 --- a/libavcodec/h264_cabac.c +++ b/libavcodec/h264_cabac.c @@ -2040,14 +2040,14 @@ decode_intra_mb: write_back_intra_pred_mode(h); if( ff_h264_check_intra4x4_pred_mode(h) < 0 ) return -1; } else { - h->intra16x16_pred_mode= ff_h264_check_intra_pred_mode( h, h->intra16x16_pred_mode ); + h->intra16x16_pred_mode= ff_h264_check_intra_pred_mode( h, h->intra16x16_pred_mode, 0 ); if( h->intra16x16_pred_mode < 0 ) return -1; } if(decode_chroma){ h->chroma_pred_mode_table[mb_xy] = pred_mode = decode_cabac_mb_chroma_pre_mode( h ); - pred_mode= ff_h264_check_intra_pred_mode( h, pred_mode ); + pred_mode= ff_h264_check_intra_pred_mode( h, pred_mode, 1 ); if( pred_mode < 0 ) return -1; h->chroma_pred_mode= pred_mode; } else { diff --git a/libavcodec/h264_cavlc.c b/libavcodec/h264_cavlc.c index db74602f66..da9e1cb70c 100644 --- a/libavcodec/h264_cavlc.c +++ b/libavcodec/h264_cavlc.c @@ -822,12 +822,12 @@ decode_intra_mb: if( ff_h264_check_intra4x4_pred_mode(h) < 0) return -1; }else{ - h->intra16x16_pred_mode= ff_h264_check_intra_pred_mode(h, h->intra16x16_pred_mode); + h->intra16x16_pred_mode= ff_h264_check_intra_pred_mode(h, h->intra16x16_pred_mode, 0); if(h->intra16x16_pred_mode < 0) return -1; } if(decode_chroma){ - pred_mode= ff_h264_check_intra_pred_mode(h, get_ue_golomb_31(&s->gb)); + pred_mode= ff_h264_check_intra_pred_mode(h, get_ue_golomb_31(&s->gb), 1); if(pred_mode < 0) return -1; h->chroma_pred_mode= pred_mode; diff --git a/libavcodec/svq3.c b/libavcodec/svq3.c index 3cd95ba594..5cc57a745d 100644 --- a/libavcodec/svq3.c +++ b/libavcodec/svq3.c @@ -612,7 +612,7 @@ static int svq3_decode_mb(SVQ3Context *svq3, unsigned int mb_type) dir = i_mb_type_info[mb_type - 8].pred_mode; dir = (dir >> 1) ^ 3*(dir & 1) ^ 1; - if ((h->intra16x16_pred_mode = ff_h264_check_intra_pred_mode(h, dir)) == -1){ + if ((h->intra16x16_pred_mode = ff_h264_check_intra_pred_mode(h, dir, 0)) == -1){ av_log(h->s.avctx, AV_LOG_ERROR, "check_intra_pred_mode = -1\n"); return -1; } @@ -711,7 +711,7 @@ static int svq3_decode_mb(SVQ3Context *svq3, unsigned int mb_type) s->current_picture.f.mb_type[mb_xy] = mb_type; if (IS_INTRA(mb_type)) { - h->chroma_pred_mode = ff_h264_check_intra_pred_mode(h, DC_PRED8x8); + h->chroma_pred_mode = ff_h264_check_intra_pred_mode(h, DC_PRED8x8, 1); } return 0; From 697a45d861b7cd6a96718383a44f41348487f844 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Sun, 25 Dec 2011 00:10:27 +0100 Subject: [PATCH 008/154] ws_snd1: Fix wrong samples count and crash. Signed-off-by: Michael Niedermayer (cherry picked from commit 9fb7a5af97d8c084c3af2566070d09eae0ab49fc) Addresses CVE-2012-0848 Reviewed-by: Justin Ruggles Signed-off-by: Reinhard Tartler --- libavcodec/ws-snd1.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libavcodec/ws-snd1.c b/libavcodec/ws-snd1.c index b2d086e073..15eb6f895a 100644 --- a/libavcodec/ws-snd1.c +++ b/libavcodec/ws-snd1.c @@ -112,8 +112,8 @@ static int ws_snd_decode_frame(AVCodecContext *avctx, void *data, /* make sure we don't write past the output buffer */ switch (code) { - case 0: smp = 4; break; - case 1: smp = 2; break; + case 0: smp = 4*(count+1); break; + case 1: smp = 2*(count+1); break; case 2: smp = (count & 0x20) ? 1 : count + 1; break; default: smp = count + 1; break; } From f43b6e2b1ed47a1254a5d44c700a7fad5e9784be Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Sat, 17 Dec 2011 03:18:58 +0100 Subject: [PATCH 009/154] atrac3: Fix crash in tonal component decoding. Add a check to avoid writing past the end of the channel_unit.components[] array. Bug Found by: cosminamironesei Fixes CVE-2012-0853 CC: libav-stable@libav.org Signed-off-by: Michael Niedermayer Signed-off-by: Justin Ruggles (cherry picked from commit c509f4f74713b035a06f79cb4d00e708f5226bc5) Signed-off-by: Reinhard Tartler --- libavcodec/atrac3.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libavcodec/atrac3.c b/libavcodec/atrac3.c index 6dec6a3abe..107c6ffeb0 100644 --- a/libavcodec/atrac3.c +++ b/libavcodec/atrac3.c @@ -402,6 +402,8 @@ static int decodeTonalComponents (GetBitContext *gb, tonal_component *pComponent for (k=0; k= 64) + return AVERROR_INVALIDDATA; pComponent[component_count].pos = j * 64 + (get_bits(gb,6)); max_coded_values = SAMPLES_PER_FRAME - pComponent[component_count].pos; coded_values = coded_values_per_component + 1; From 6fc3287b9ccece290c5881b92948772bbf72e68c Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Sun, 25 Dec 2011 12:28:50 +0100 Subject: [PATCH 010/154] shorten: Use separate pointers for the allocated memory for decoded samples. Fixes invalid free() if any of the buffers are not allocated due to either not decoding a header or an error prior to allocating all buffers. Fixes CVE-2012-0858 CC: libav-stable@libav.org Signed-off-by: Michael Niedermayer Signed-off-by: Justin Ruggles (cherry picked from commit 204cb29b3c84a74cbcd059d353c70c8bdc567d98) Signed-off-by: Reinhard Tartler --- libavcodec/shorten.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/libavcodec/shorten.c b/libavcodec/shorten.c index eb67df7bea..83777fb934 100644 --- a/libavcodec/shorten.c +++ b/libavcodec/shorten.c @@ -86,6 +86,7 @@ typedef struct ShortenContext { int channels; int32_t *decoded[MAX_CHANNELS]; + int32_t *decoded_base[MAX_CHANNELS]; int32_t *offset[MAX_CHANNELS]; int *coeffs; uint8_t *bitstream; @@ -140,13 +141,14 @@ static int allocate_buffers(ShortenContext *s) return AVERROR(ENOMEM); s->offset[chan] = tmp_ptr; - tmp_ptr = av_realloc(s->decoded[chan], sizeof(int32_t)*(s->blocksize + s->nwrap)); + tmp_ptr = av_realloc(s->decoded_base[chan], (s->blocksize + s->nwrap) * + sizeof(s->decoded_base[0][0])); if (!tmp_ptr) return AVERROR(ENOMEM); - s->decoded[chan] = tmp_ptr; + s->decoded_base[chan] = tmp_ptr; for (i=0; inwrap; i++) - s->decoded[chan][i] = 0; - s->decoded[chan] += s->nwrap; + s->decoded_base[chan][i] = 0; + s->decoded[chan] = s->decoded_base[chan] + s->nwrap; } coeffs = av_realloc(s->coeffs, s->nwrap * sizeof(*s->coeffs)); @@ -615,8 +617,8 @@ static av_cold int shorten_decode_close(AVCodecContext *avctx) int i; for (i = 0; i < s->channels; i++) { - s->decoded[i] -= s->nwrap; - av_freep(&s->decoded[i]); + s->decoded[i] = NULL; + av_freep(&s->decoded_base[i]); av_freep(&s->offset[i]); } av_freep(&s->bitstream); From e1f2a6a32b86fef0916338e21851c9b4f499f706 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Tue, 14 Feb 2012 11:50:57 -0800 Subject: [PATCH 011/154] golomb: avoid infinite loop on all-zero input (or end of buffer). Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit c6643fddba73560f26f90d327c84d8832222a720) Signed-off-by: Reinhard Tartler --- libavcodec/golomb.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libavcodec/golomb.h b/libavcodec/golomb.h index 503aa1416a..e19064c642 100644 --- a/libavcodec/golomb.h +++ b/libavcodec/golomb.h @@ -123,7 +123,7 @@ static inline int svq3_get_ue_golomb(GetBitContext *gb){ }else{ int ret = 1; - while (1) { + do { buf >>= 32 - 8; LAST_SKIP_BITS(re, gb, FFMIN(ff_interleaved_golomb_vlc_len[buf], 8)); @@ -135,7 +135,7 @@ static inline int svq3_get_ue_golomb(GetBitContext *gb){ ret = (ret << 4) | ff_interleaved_dirac_golomb_vlc_code[buf]; UPDATE_CACHE(re, gb); buf = GET_CACHE(re, gb); - } + } while (ret); CLOSE_READER(re, gb); return ret - 1; From 25b4ed053f0e4c48b4b4afdcf84306bbd7752314 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Wed, 22 Feb 2012 12:09:33 -0800 Subject: [PATCH 012/154] get_bits: add HAVE_BITS_REMAINING macro. (cherry picked from commit b44b41633f110e9d938165e0f79c9d32191fc135) Signed-off-by: Reinhard Tartler --- libavcodec/get_bits.h | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/libavcodec/get_bits.h b/libavcodec/get_bits.h index 1668600b2d..ee47441899 100644 --- a/libavcodec/get_bits.h +++ b/libavcodec/get_bits.h @@ -120,10 +120,23 @@ for examples see get_bits, show_bits, skip_bits, get_vlc # define MIN_CACHE_BITS 25 #endif +#if UNCHECKED_BITSTREAM_READER #define OPEN_READER(name, gb) \ unsigned int name##_index = (gb)->index; \ unsigned int av_unused name##_cache = 0 +#define HAVE_BITS_REMAINING(name, gb) 1 +#else +#define OPEN_READER(name, gb) \ + unsigned int name##_index = (gb)->index; \ + unsigned int av_unused name##_cache = 0; \ + unsigned int av_unused name##_size_plus8 = \ + (gb)->size_in_bits_plus8 + +#define HAVE_BITS_REMAINING(name, gb) \ + name##_index < name##_size_plus8 +#endif + #define CLOSE_READER(name, gb) (gb)->index = name##_index #ifdef BITSTREAM_READER_LE @@ -156,7 +169,7 @@ for examples see get_bits, show_bits, skip_bits, get_vlc # define SKIP_COUNTER(name, gb, num) name##_index += (num) #else # define SKIP_COUNTER(name, gb, num) \ - name##_index = FFMIN((gb)->size_in_bits_plus8, name##_index + (num)) + name##_index = FFMIN(name##_size_plus8, name##_index + (num)) #endif #define SKIP_BITS(name, gb, num) do { \ From e43bd4fa58b8e72eedad9a1c160b12bf8915d45e Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Fri, 17 Feb 2012 12:54:37 -0800 Subject: [PATCH 013/154] golomb: use HAVE_BITS_REMAINING() macro to prevent infloop on EOF. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 46b3fbc30b7aaf7fdd52391734cfd6d93af8720a) Signed-off-by: Reinhard Tartler --- libavcodec/golomb.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavcodec/golomb.h b/libavcodec/golomb.h index e19064c642..0deab87a36 100644 --- a/libavcodec/golomb.h +++ b/libavcodec/golomb.h @@ -135,7 +135,7 @@ static inline int svq3_get_ue_golomb(GetBitContext *gb){ ret = (ret << 4) | ff_interleaved_dirac_golomb_vlc_code[buf]; UPDATE_CACHE(re, gb); buf = GET_CACHE(re, gb); - } while (ret); + } while (HAVE_BITS_REMAINING(re, gb)); CLOSE_READER(re, gb); return ret - 1; From 6dcbbdc0116a50370d66f0f20d74a70d56568382 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Wed, 15 Feb 2012 09:52:11 -0800 Subject: [PATCH 014/154] flac: fix infinite loops on all-zero input or end-of-stream. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 52e4018be47697a60f4f18f83551766df31f5adf) Signed-off-by: Reinhard Tartler --- libavcodec/flacdec.c | 9 +++++++++ libavcodec/golomb.h | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/libavcodec/flacdec.c b/libavcodec/flacdec.c index 58eb66def9..440a55d3e9 100644 --- a/libavcodec/flacdec.c +++ b/libavcodec/flacdec.c @@ -422,7 +422,16 @@ static inline int decode_subframe(FLACContext *s, int channel) type = get_bits(&s->gb, 6); if (get_bits1(&s->gb)) { + int left = get_bits_left(&s->gb); wasted = 1; + if ( left < 0 || + (left < s->curr_bps && !show_bits_long(&s->gb, left)) || + !show_bits_long(&s->gb, s->curr_bps)) { + av_log(s->avctx, AV_LOG_ERROR, + "Invalid number of wasted bits > available bits (%d) - left=%d\n", + s->curr_bps, left); + return AVERROR_INVALIDDATA; + } while (!get_bits1(&s->gb)) wasted++; s->curr_bps -= wasted; diff --git a/libavcodec/golomb.h b/libavcodec/golomb.h index 0deab87a36..1712540fd3 100644 --- a/libavcodec/golomb.h +++ b/libavcodec/golomb.h @@ -301,7 +301,7 @@ static inline int get_ur_golomb_jpegls(GetBitContext *gb, int k, int limit, int return buf; }else{ int i; - for(i=0; SHOW_UBITS(re, gb, 1) == 0; i++){ + for (i = 0; i < limit && SHOW_UBITS(re, gb, 1) == 0; i++) { LAST_SKIP_BITS(re, gb, 1); UPDATE_CACHE(re, gb); } From ba418ad4005a2cc2f18cdfa089d0bcd55225b30e Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Tue, 24 Jan 2012 21:50:50 +0100 Subject: [PATCH 015/154] rv20: prevent calling ff_h263_decode_mba() with unset height/width Prevents a crash of VLC during playback of a invalid matroska file, found by John Villamil . CC: libav-stable@libav.org (cherry picked from commit c3e10ae4127c998b809066926a410f40ebd47593) Signed-off-by: Anton Khirnov --- libavcodec/rv10.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libavcodec/rv10.c b/libavcodec/rv10.c index 1d78c92c46..ccc09443ec 100644 --- a/libavcodec/rv10.c +++ b/libavcodec/rv10.c @@ -362,7 +362,8 @@ static int rv20_decode_picture_header(MpegEncContext *s) if(s->avctx->debug & FF_DEBUG_PICT_INFO){ av_log(s->avctx, AV_LOG_DEBUG, "F %d/%d\n", f, rpr_bits); } - } + } else if (av_image_check_size(s->width, s->height, 0, s->avctx) < 0) + return AVERROR_INVALIDDATA; mb_pos = ff_h263_decode_mba(s); From ad0ee682b3cf663eb319020086f64da11d17dd82 Mon Sep 17 00:00:00 2001 From: Alex Converse Date: Tue, 24 Jan 2012 18:43:43 -0800 Subject: [PATCH 016/154] wma: Clip WMA1 and WMA2 frame length to 11 bits. The MDCT buffers in the decoder are only sized for up to 11 bits. The reverse engineered documentation for WMA1/2 headers say that that for all samplerates above 32kHz 11 bits are used. 12 and 13 bit support were added for WMAPro. I was unable to make any Microsoft tools generate a test file at a samplerate above 48kHz. Discovered by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit d78bb1a4b2a3a415b68e4e6dd448779eccec64e3) Signed-off-by: Anton Khirnov --- libavcodec/wma.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavcodec/wma.c b/libavcodec/wma.c index 4cdffcd101..d82fde7b18 100644 --- a/libavcodec/wma.c +++ b/libavcodec/wma.c @@ -85,7 +85,7 @@ int av_cold ff_wma_get_frame_len_bits(int sample_rate, int version, } else if (sample_rate <= 22050 || (sample_rate <= 32000 && version == 1)) { frame_len_bits = 10; - } else if (sample_rate <= 48000) { + } else if (sample_rate <= 48000 || version < 3) { frame_len_bits = 11; } else if (sample_rate <= 96000) { frame_len_bits = 12; From 683213230e6978302109253a48610a6b069ea43d Mon Sep 17 00:00:00 2001 From: Alex Converse Date: Wed, 22 Feb 2012 11:05:42 -0800 Subject: [PATCH 017/154] aac: fix infinite loop on end-of-frame with sequence of 1-bits. Based-on-work-by: Ronald S. Bultje Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 1cd9a6154bc1ac1193c703cea980ed21c3e53792) Signed-off-by: Anton Khirnov --- libavcodec/aacdec.c | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/libavcodec/aacdec.c b/libavcodec/aacdec.c index ca1a876436..2b9b45c9e8 100644 --- a/libavcodec/aacdec.c +++ b/libavcodec/aacdec.c @@ -807,19 +807,20 @@ static int decode_band_types(AACContext *ac, enum BandType band_type[120], av_log(ac->avctx, AV_LOG_ERROR, "invalid band type\n"); return -1; } - while ((sect_len_incr = get_bits(gb, bits)) == (1 << bits) - 1) + do { + sect_len_incr = get_bits(gb, bits); sect_end += sect_len_incr; - sect_end += sect_len_incr; - if (get_bits_left(gb) < 0) { - av_log(ac->avctx, AV_LOG_ERROR, overread_err); - return -1; - } - if (sect_end > ics->max_sfb) { - av_log(ac->avctx, AV_LOG_ERROR, - "Number of bands (%d) exceeds limit (%d).\n", - sect_end, ics->max_sfb); - return -1; - } + if (get_bits_left(gb) < 0) { + av_log(ac->avctx, AV_LOG_ERROR, overread_err); + return -1; + } + if (sect_end > ics->max_sfb) { + av_log(ac->avctx, AV_LOG_ERROR, + "Number of bands (%d) exceeds limit (%d).\n", + sect_end, ics->max_sfb); + return -1; + } + } while (sect_len_incr == (1 << bits) - 1); for (; k < sect_end; k++) { band_type [idx] = sect_band_type; band_type_run_end[idx++] = sect_end; From be0b3137d02e2e19bd470f2de888bdeb281b0214 Mon Sep 17 00:00:00 2001 From: Alex Converse Date: Wed, 25 Jan 2012 14:34:21 -0800 Subject: [PATCH 018/154] matroskadec: Pad AAC extradata. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit d2ee8c17793201ce969afd1f433ba1580c143cd2) Signed-off-by: Anton Khirnov --- libavformat/matroskadec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavformat/matroskadec.c b/libavformat/matroskadec.c index e5fbd43266..5b919449f5 100644 --- a/libavformat/matroskadec.c +++ b/libavformat/matroskadec.c @@ -1440,7 +1440,7 @@ static int matroska_read_header(AVFormatContext *s, AVFormatParameters *ap) } else if (codec_id == CODEC_ID_AAC && !track->codec_priv.size) { int profile = matroska_aac_profile(track->codec_id); int sri = matroska_aac_sri(track->audio.samplerate); - extradata = av_malloc(5); + extradata = av_mallocz(5 + FF_INPUT_BUFFER_PADDING_SIZE); if (extradata == NULL) return AVERROR(ENOMEM); extradata[0] = (profile << 3) | ((sri&0x0E) >> 1); From 183e0eb5b9a8780b9879bd78b20ad9156d756a01 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Fri, 24 Feb 2012 16:12:18 -0800 Subject: [PATCH 019/154] matroska: don't overwrite string values until read/alloc was succesful. This prevents certain tags with a default value assigned to them (as per the EBML syntax elements) from ever being assigned a NULL value. Other parts of the code rely on these being non-NULL (i.e. they don't check for NULL before e.g. using the string in strcmp() or similar), and thus in effect this prevents crashes when reading of such specific tags fails, either because of low memory or because of targeted file corruption. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit cd40c31ee9ad2cca6f3635950b002fd46be07e98) Signed-off-by: Anton Khirnov --- libavformat/matroskadec.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/libavformat/matroskadec.c b/libavformat/matroskadec.c index 5b919449f5..1987b5095f 100644 --- a/libavformat/matroskadec.c +++ b/libavformat/matroskadec.c @@ -639,16 +639,19 @@ static int ebml_read_float(AVIOContext *pb, int size, double *num) */ static int ebml_read_ascii(AVIOContext *pb, int size, char **str) { - av_free(*str); + char *res; + /* EBML strings are usually not 0-terminated, so we allocate one * byte more, read the string and NULL-terminate it ourselves. */ - if (!(*str = av_malloc(size + 1))) + if (!(res = av_malloc(size + 1))) return AVERROR(ENOMEM); - if (avio_read(pb, (uint8_t *) *str, size) != size) { - av_freep(str); + if (avio_read(pb, (uint8_t *) res, size) != size) { + av_free(res); return AVERROR(EIO); } - (*str)[size] = '\0'; + (res)[size] = '\0'; + av_free(*str); + *str = res; return 0; } From d16653c3d437ff7843c111d9fffa3e8c3e186db7 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Wed, 18 Jan 2012 10:59:32 +0100 Subject: [PATCH 020/154] lavf: prevent infinite loops while flushing in avformat_find_stream_info If no data was seen for a stream decoder are returning 0 when fed with empty packets for flushing. We can stop flushing when the decoder does not return delayed delayed frames anymore. Changes try_decode_frame() return value to got_picture or negative error. CC: libav-stable@libav.org (cherry picked from commit b3461c29c1aee7d62eeb02a59d46593c60362679) Signed-off-by: Anton Khirnov --- libavformat/utils.c | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/libavformat/utils.c b/libavformat/utils.c index f2d55028f9..e6b4f40cf3 100644 --- a/libavformat/utils.c +++ b/libavformat/utils.c @@ -2130,6 +2130,7 @@ static int has_decode_delay_been_guessed(AVStream *st) st->info->nb_decoded_frames >= 6; } +/* returns 1 or 0 if or if not decoded data was returned, or a negative error */ static int try_decode_frame(AVStream *st, AVPacket *avpkt, AVDictionary **options) { AVCodec *codec; @@ -2179,6 +2180,7 @@ static int try_decode_frame(AVStream *st, AVPacket *avpkt, AVDictionary **option st->info->nb_decoded_frames++; pkt.data += ret; pkt.size -= ret; + ret = got_picture; } } return ret; @@ -2403,16 +2405,20 @@ int avformat_find_stream_info(AVFormatContext *ic, AVDictionary **options) st = ic->streams[i]; /* flush the decoders */ - while ((err = try_decode_frame(st, &empty_pkt, - (options && i < orig_nb_streams) ? - &options[i] : NULL)) >= 0) - if (has_codec_parameters(st->codec)) - break; + do { + err = try_decode_frame(st, &empty_pkt, + (options && i < orig_nb_streams) ? + &options[i] : NULL); + } while (err > 0 && !has_codec_parameters(st->codec)); - if (!has_codec_parameters(st->codec)){ + if (err < 0) { + av_log(ic, AV_LOG_WARNING, + "decoding for stream %d failed\n", st->index); + } else if (!has_codec_parameters(st->codec)){ char buf[256]; avcodec_string(buf, sizeof(buf), st->codec, 0); - av_log(ic, AV_LOG_WARNING, "Could not find codec parameters (%s)\n", buf); + av_log(ic, AV_LOG_WARNING, + "Could not find codec parameters (%s)\n", buf); } else { ret = 0; } From 04597e25952d399a350062c1824587c230cdd5b4 Mon Sep 17 00:00:00 2001 From: Alex Converse Date: Wed, 25 Jan 2012 16:12:42 -0800 Subject: [PATCH 021/154] smacker: Sanity check huffman tables found in the headers. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 9adf25c1cf78dbf1d71bf386c49dc74cb8a60df0) Signed-off-by: Anton Khirnov --- libavcodec/smacker.c | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/libavcodec/smacker.c b/libavcodec/smacker.c index 0c7c40560f..4714fa0346 100644 --- a/libavcodec/smacker.c +++ b/libavcodec/smacker.c @@ -128,12 +128,12 @@ static int smacker_decode_tree(GetBitContext *gb, HuffContext *hc, uint32_t pref */ static int smacker_decode_bigtree(GetBitContext *gb, HuffContext *hc, DBCtx *ctx) { + if (hc->current + 1 >= hc->length) { + av_log(NULL, AV_LOG_ERROR, "Tree size exceeded!\n"); + return -1; + } if(!get_bits1(gb)){ //Leaf int val, i1, i2, b1, b2; - if(hc->current >= hc->length){ - av_log(NULL, AV_LOG_ERROR, "Tree size exceeded!\n"); - return -1; - } b1 = get_bits_count(gb); i1 = ctx->v1->table ? get_vlc2(gb, ctx->v1->table, SMKTREE_BITS, 3) : 0; b1 = get_bits_count(gb) - b1; @@ -157,7 +157,7 @@ static int smacker_decode_bigtree(GetBitContext *gb, HuffContext *hc, DBCtx *ctx hc->values[hc->current++] = val; return 1; } else { //Node - int r = 0, t; + int r = 0, r_new, t; t = hc->current++; r = smacker_decode_bigtree(gb, hc, ctx); @@ -165,8 +165,10 @@ static int smacker_decode_bigtree(GetBitContext *gb, HuffContext *hc, DBCtx *ctx return r; hc->values[t] = SMK_NODE | r; r++; - r += smacker_decode_bigtree(gb, hc, ctx); - return r; + r_new = smacker_decode_bigtree(gb, hc, ctx); + if (r_new < 0) + return r_new; + return r + r_new; } } @@ -181,6 +183,7 @@ static int smacker_decode_header_tree(SmackVContext *smk, GetBitContext *gb, int VLC vlc[2]; int escapes[3]; DBCtx ctx; + int err = 0; if(size >= UINT_MAX>>4){ // (((size + 3) >> 2) + 3) << 2 must not overflow av_log(smk->avctx, AV_LOG_ERROR, "size too large\n"); @@ -254,7 +257,8 @@ static int smacker_decode_header_tree(SmackVContext *smk, GetBitContext *gb, int huff.current = 0; huff.values = av_mallocz(huff.length * sizeof(int)); - smacker_decode_bigtree(gb, &huff, &ctx); + if (smacker_decode_bigtree(gb, &huff, &ctx) < 0) + err = -1; skip_bits1(gb); if(ctx.last[0] == -1) ctx.last[0] = huff.current++; if(ctx.last[1] == -1) ctx.last[1] = huff.current++; @@ -273,7 +277,7 @@ static int smacker_decode_header_tree(SmackVContext *smk, GetBitContext *gb, int av_free(tmp2.lengths); av_free(tmp2.values); - return 0; + return err; } static int decode_header_trees(SmackVContext *smk) { From d19e3e19d67b50cb5614ead2e0f125678e1c257d Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Wed, 25 Jan 2012 15:49:54 +0100 Subject: [PATCH 022/154] vc1: prevent null pointer dereference on broken files CC: libav-stable@libav.org (cherry picked from commit 510ef04a461b3b54a762c6141ad880cbed85981f) Signed-off-by: Anton Khirnov --- libavcodec/vc1dec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavcodec/vc1dec.c b/libavcodec/vc1dec.c index fa952739bb..0425a87e41 100644 --- a/libavcodec/vc1dec.c +++ b/libavcodec/vc1dec.c @@ -5708,7 +5708,7 @@ static int vc1_decode_frame(AVCodecContext *avctx, void *data, if (!v->field_mode || v->second_field) s->end_mb_y = (i == n_slices ) ? mb_height : FFMIN(mb_height, slices[i].mby_start % mb_height); else - s->end_mb_y = (i == n_slices1 + 1) ? mb_height : FFMIN(mb_height, slices[i].mby_start % mb_height); + s->end_mb_y = (i <= n_slices1 + 1) ? mb_height : FFMIN(mb_height, slices[i].mby_start % mb_height); vc1_decode_blocks(v); if (i != n_slices) s->gb = slices[i].gb; From 7046ae55932f8fae83269871847cea9fd84c23f5 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Fri, 10 Feb 2012 10:51:43 -0800 Subject: [PATCH 023/154] tta: error out if samplerate is zero. Prevents a division by zero later on. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 7416d610362807848236ceff1bc6740dbc82842d) Signed-off-by: Anton Khirnov --- libavcodec/tta.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libavcodec/tta.c b/libavcodec/tta.c index 4656ce12e6..c8daff278c 100644 --- a/libavcodec/tta.c +++ b/libavcodec/tta.c @@ -224,6 +224,9 @@ static av_cold int tta_decode_init(AVCodecContext * avctx) if (s->channels == 0) { av_log(s->avctx, AV_LOG_ERROR, "Invalid number of channels\n"); return AVERROR_INVALIDDATA; + } else if (avctx->sample_rate == 0) { + av_log(s->avctx, AV_LOG_ERROR, "Invalid samplerate\n"); + return AVERROR_INVALIDDATA; } switch(s->bps) { From b68470707bf2e010136c6debd25051afdf198466 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Sat, 11 Feb 2012 08:42:28 -0800 Subject: [PATCH 024/154] swscale: enforce a minimum filtersize. At very small dimensions, this calculation could lead to zero-sized filters, which leads to uninitialized output, zero-sized allocations, loop overflows in SIMD that uses do{..}while(i++ --- libswscale/utils.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libswscale/utils.c b/libswscale/utils.c index b49f924244..9d72196e42 100644 --- a/libswscale/utils.c +++ b/libswscale/utils.c @@ -263,7 +263,7 @@ static int initFilter(int16_t **outFilter, int16_t **filterPos, int *outFilterSi if (xInc <= 1<<16) filterSize= 1 + sizeFactor; // upscale else filterSize= 1 + (sizeFactor*srcW + dstW - 1)/ dstW; - if (filterSize > srcW-2) filterSize=srcW-2; + filterSize = av_clip(filterSize, 1, srcW - 2); FF_ALLOC_OR_GOTO(NULL, filter, dstW*sizeof(*filter)*filterSize, fail); From cd9bdc639588067732b53bb47a01f7b9b902b9ef Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Wed, 22 Feb 2012 16:46:31 -0800 Subject: [PATCH 025/154] swscale: fix overflows in filterPos[] calculation for large sizes. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 19a65b5be47944c607a9e979edb098924d95f2e4) Signed-off-by: Anton Khirnov --- libswscale/utils.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/libswscale/utils.c b/libswscale/utils.c index 9d72196e42..2d7029e2f1 100644 --- a/libswscale/utils.c +++ b/libswscale/utils.c @@ -244,7 +244,7 @@ static int initFilter(int16_t **outFilter, int16_t **filterPos, int *outFilterSi xDstInSrc+= xInc; } } else { - int xDstInSrc; + int64_t xDstInSrc; int sizeFactor; if (flags&SWS_BICUBIC) sizeFactor= 4; @@ -809,8 +809,8 @@ int sws_init_context(SwsContext *c, SwsFilter *srcFilter, SwsFilter *dstFilter) if (!dstFilter) dstFilter= &dummyFilter; if (!srcFilter) srcFilter= &dummyFilter; - c->lumXInc= ((srcW<<16) + (dstW>>1))/dstW; - c->lumYInc= ((srcH<<16) + (dstH>>1))/dstH; + c->lumXInc= (((int64_t)srcW<<16) + (dstW>>1))/dstW; + c->lumYInc= (((int64_t)srcH<<16) + (dstH>>1))/dstH; c->dstFormatBpp = av_get_bits_per_pixel(&av_pix_fmt_descriptors[dstFormat]); c->srcFormatBpp = av_get_bits_per_pixel(&av_pix_fmt_descriptors[srcFormat]); c->vRounder= 4* 0x0001000100010001ULL; @@ -896,8 +896,8 @@ int sws_init_context(SwsContext *c, SwsFilter *srcFilter, SwsFilter *dstFilter) else c->canMMX2BeUsed=0; - c->chrXInc= ((c->chrSrcW<<16) + (c->chrDstW>>1))/c->chrDstW; - c->chrYInc= ((c->chrSrcH<<16) + (c->chrDstH>>1))/c->chrDstH; + c->chrXInc= (((int64_t)c->chrSrcW<<16) + (c->chrDstW>>1))/c->chrDstW; + c->chrYInc= (((int64_t)c->chrSrcH<<16) + (c->chrDstH>>1))/c->chrDstH; // match pixel 0 of the src to pixel 0 of dst and match pixel n-2 of src to pixel n-2 of dst // but only for the FAST_BILINEAR mode otherwise do correct scaling @@ -912,8 +912,8 @@ int sws_init_context(SwsContext *c, SwsFilter *srcFilter, SwsFilter *dstFilter) } //we don't use the x86 asm scaler if MMX is available else if (HAVE_MMX && cpu_flags & AV_CPU_FLAG_MMX) { - c->lumXInc = ((srcW-2)<<16)/(dstW-2) - 20; - c->chrXInc = ((c->chrSrcW-2)<<16)/(c->chrDstW-2) - 20; + c->lumXInc = ((int64_t)(srcW-2)<<16)/(dstW-2) - 20; + c->chrXInc = ((int64_t)(c->chrSrcW-2)<<16)/(c->chrDstW-2) - 20; } } From 0c60d5c59fe05de80fc45e097c61b6f5487431de Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Wed, 22 Feb 2012 16:48:38 -0800 Subject: [PATCH 026/154] swscale: take first/lastline over/underflows into account for MMX. Fixes crashes for extremely large resizes (several 100-fold). Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 1d8c4af396b6ed84c84b5ebf0bf1163c4a7a3017) Signed-off-by: Anton Khirnov --- libswscale/x86/swscale_mmx.c | 38 ++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/libswscale/x86/swscale_mmx.c b/libswscale/x86/swscale_mmx.c index 867a9f1244..0853e12c41 100644 --- a/libswscale/x86/swscale_mmx.c +++ b/libswscale/x86/swscale_mmx.c @@ -132,6 +132,44 @@ void updateMMXDitherTables(SwsContext *c, int dstY, int lumBufIndex, int chrBufI const int16_t **chrUSrcPtr= (const int16_t **) chrUPixBuf + chrBufIndex + firstChrSrcY - lastInChrBuf + vChrBufSize; const int16_t **alpSrcPtr= (CONFIG_SWSCALE_ALPHA && alpPixBuf) ? (const int16_t **) alpPixBuf + lumBufIndex + firstLumSrcY - lastInLumBuf + vLumBufSize : NULL; int i; + + if (firstLumSrcY < 0 || firstLumSrcY + vLumFilterSize > c->srcH) { + const int16_t **tmpY = (const int16_t **) lumPixBuf + 2 * vLumBufSize; + int neg = -firstLumSrcY, i, end = FFMIN(c->srcH - firstLumSrcY, vLumFilterSize); + for (i = 0; i < neg; i++) + tmpY[i] = lumSrcPtr[neg]; + for ( ; i < end; i++) + tmpY[i] = lumSrcPtr[i]; + for ( ; i < vLumFilterSize; i++) + tmpY[i] = tmpY[i-1]; + lumSrcPtr = tmpY; + + if (alpSrcPtr) { + const int16_t **tmpA = (const int16_t **) alpPixBuf + 2 * vLumBufSize; + for (i = 0; i < neg; i++) + tmpA[i] = alpSrcPtr[neg]; + for ( ; i < end; i++) + tmpA[i] = alpSrcPtr[i]; + for ( ; i < vLumFilterSize; i++) + tmpA[i] = tmpA[i - 1]; + alpSrcPtr = tmpA; + } + } + if (firstChrSrcY < 0 || firstChrSrcY + vChrFilterSize > c->chrSrcH) { + const int16_t **tmpU = (const int16_t **) chrUPixBuf + 2 * vChrBufSize; + int neg = -firstChrSrcY, i, end = FFMIN(c->chrSrcH - firstChrSrcY, vChrFilterSize); + for (i = 0; i < neg; i++) { + tmpU[i] = chrUSrcPtr[neg]; + } + for ( ; i < end; i++) { + tmpU[i] = chrUSrcPtr[i]; + } + for ( ; i < vChrFilterSize; i++) { + tmpU[i] = tmpU[i - 1]; + } + chrUSrcPtr = tmpU; + } + if (flags & SWS_ACCURATE_RND) { int s= APCK_SIZE / 8; for (i=0; i Date: Tue, 14 Feb 2012 12:40:19 -0800 Subject: [PATCH 027/154] vc1: prevent using last_frame as a reference for I/P first frame. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit ae591aeea58d64399b8281be31dacec0de85ae04) Signed-off-by: Anton Khirnov --- libavcodec/vc1dec.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/libavcodec/vc1dec.c b/libavcodec/vc1dec.c index 0425a87e41..3869d92518 100644 --- a/libavcodec/vc1dec.c +++ b/libavcodec/vc1dec.c @@ -478,7 +478,10 @@ static void vc1_mc_1mv(VC1Context *v, int dir) int dxy, mx, my, uvmx, uvmy, src_x, src_y, uvsrc_x, uvsrc_y; int off, off_uv; int v_edge_pos = s->v_edge_pos >> v->field_mode; - if (!v->field_mode && !v->s.last_picture.f.data[0]) + + if ((!v->field_mode || + (v->ref_field_type[dir] == 1 && v->cur_field_type == 1)) && + !v->s.last_picture.f.data[0]) return; mx = s->mv[dir][0][0]; @@ -690,7 +693,9 @@ static void vc1_mc_4mv_luma(VC1Context *v, int n, int dir) int fieldmv = (v->fcm == ILACE_FRAME) ? v->blk_mv_type[s->block_index[n]] : 0; int v_edge_pos = s->v_edge_pos >> v->field_mode; - if (!v->field_mode && !v->s.last_picture.f.data[0]) + if ((!v->field_mode || + (v->ref_field_type[dir] == 1 && v->cur_field_type == 1)) && + !v->s.last_picture.f.data[0]) return; mx = s->mv[dir][n][0]; @@ -946,6 +951,8 @@ static void vc1_mc_4mv_chroma(VC1Context *v, int dir) if (dominant) chroma_ref_type = !v->cur_field_type; } + if (v->field_mode && chroma_ref_type == 1 && v->cur_field_type == 1 && !v->s.last_picture.f.data[0]) + return; s->current_picture.f.motion_val[1][s->block_index[0] + v->blocks_off][0] = tx; s->current_picture.f.motion_val[1][s->block_index[0] + v->blocks_off][1] = ty; uvmx = (tx + ((tx & 3) == 3)) >> 1; From cfd7d166e2ae68302329c059afa7c4778a70e9b5 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Fri, 17 Feb 2012 12:10:33 -0800 Subject: [PATCH 028/154] cook: prevent div-by-zero if channels is zero. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 941fc1ea1ed7f7d99a8b9e2607b41f2f2820394a) Signed-off-by: Anton Khirnov --- libavcodec/cook.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libavcodec/cook.c b/libavcodec/cook.c index d2ed819b83..dc4c2ab170 100644 --- a/libavcodec/cook.c +++ b/libavcodec/cook.c @@ -1078,6 +1078,10 @@ static av_cold int cook_decode_init(AVCodecContext *avctx) q->sample_rate = avctx->sample_rate; q->nb_channels = avctx->channels; q->bit_rate = avctx->bit_rate; + if (!q->nb_channels) { + av_log(avctx, AV_LOG_ERROR, "Invalid number of channels\n"); + return AVERROR_INVALIDDATA; + } /* Initialize RNG. */ av_lfg_init(&q->random_state, 0); From 5ab9294a8db5b3a796871e403b1a779a413a494c Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Fri, 17 Feb 2012 12:28:26 -0800 Subject: [PATCH 029/154] als: prevent infinite loop in zero_remaining(). Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit af468015d972c0dec5c8c37b2685ffa5cbe4ae87) Signed-off-by: Anton Khirnov --- libavcodec/alsdec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavcodec/alsdec.c b/libavcodec/alsdec.c index dc4961c9ba..26496bf0f1 100644 --- a/libavcodec/alsdec.c +++ b/libavcodec/alsdec.c @@ -1011,7 +1011,7 @@ static void zero_remaining(unsigned int b, unsigned int b_max, { unsigned int count = 0; - while (b < b_max) + for (; b < b_max; b++) count += div_blocks[b]; if (count) From 27558bd87e7e67b83ddefb9176f1729c2291c7a0 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Fri, 17 Feb 2012 15:00:47 -0800 Subject: [PATCH 030/154] huffyuv: error out on bit overrun. On EOF, get_bits() will continuously return 0, causing an infinite loop. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 84c202cc37024bd78261e4222e46631ea73c48dd) Signed-off-by: Anton Khirnov --- libavcodec/huffyuv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavcodec/huffyuv.c b/libavcodec/huffyuv.c index 57b5f32fc8..efa87de802 100644 --- a/libavcodec/huffyuv.c +++ b/libavcodec/huffyuv.c @@ -184,7 +184,7 @@ static int read_len_table(uint8_t *dst, GetBitContext *gb){ if(repeat==0) repeat= get_bits(gb, 8); //printf("%d %d\n", val, repeat); - if(i+repeat > 256) { + if(i+repeat > 256 || get_bits_left(gb) < 0) { av_log(NULL, AV_LOG_ERROR, "Error reading huffman table\n"); return -1; } From 95a9d44dc3121a93c68087dddd7b9b49d34bf930 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Fri, 17 Feb 2012 15:20:27 -0800 Subject: [PATCH 031/154] mp3on4: require a minimum framesize. If bufsize < headersize, init_get_bits() will be called with a negative number, causing it to fail and any subsequent call to get_bits() will crash because it reads from a NULL pointer. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 3e13005cac6e076053276b515f5fcf59a3f4b65d) Signed-off-by: Anton Khirnov --- libavcodec/mpegaudiodec.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libavcodec/mpegaudiodec.c b/libavcodec/mpegaudiodec.c index a83b1621fd..860c0c3d73 100644 --- a/libavcodec/mpegaudiodec.c +++ b/libavcodec/mpegaudiodec.c @@ -1921,6 +1921,10 @@ static int decode_frame_mp3on4(AVCodecContext *avctx, void *data, m = s->mp3decctx[fr]; assert(m != NULL); + if (fsize < HEADER_SIZE) { + av_log(avctx, AV_LOG_ERROR, "Frame size smaller than header size\n"); + return AVERROR_INVALIDDATA; + } header = (AV_RB32(buf) & 0x000fffff) | s->syncword; // patch header if (ff_mpa_check_header(header) < 0) // Bad header, discard block From 5c365dc9792a6a91637498e2ee1fdcb90c9c7640 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Fri, 17 Feb 2012 15:51:27 -0800 Subject: [PATCH 032/154] aiff: don't skip block_align==0 check on COMM-after-SSND files. This prevents SIGFPEs when using block_align for divisions. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 32a659c758bf2ddd8ad48f18c06fa77444341286) Signed-off-by: Anton Khirnov --- libavformat/aiffdec.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libavformat/aiffdec.c b/libavformat/aiffdec.c index 0e69d02c8c..88e1e68bfe 100644 --- a/libavformat/aiffdec.c +++ b/libavformat/aiffdec.c @@ -264,12 +264,12 @@ static int aiff_read_header(AVFormatContext *s, } } +got_sound: if (!st->codec->block_align) { - av_log(s, AV_LOG_ERROR, "could not find COMM tag\n"); + av_log(s, AV_LOG_ERROR, "could not find COMM tag or invalid block_align value\n"); return -1; } -got_sound: /* Now positioned, get the sound data start and end */ avpriv_set_pts_info(st, 64, 1, st->codec->sample_rate); st->start_time = 0; From f947e965beb858b67ab6e49f9e24e8d12d9b5a7d Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Fri, 17 Feb 2012 12:21:18 -0800 Subject: [PATCH 033/154] asf: prevent packet_size_left from going negative if hdrlen > pktlen. This prevents failed assertions further down in the packet processing where we require non-negative values for packet_size_left. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 41afac7f7a67c634c86b1d17fc930e9183d4aaa0) Signed-off-by: Anton Khirnov --- libavformat/asfdec.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/libavformat/asfdec.c b/libavformat/asfdec.c index 91d285e8b5..eb93f14ecf 100644 --- a/libavformat/asfdec.c +++ b/libavformat/asfdec.c @@ -789,6 +789,13 @@ static int ff_asf_get_packet(AVFormatContext *s, AVIOContext *pb) asf->packet_segments = 1; asf->packet_segsizetype = 0x80; } + if (rsize > packet_length - padsize) { + asf->packet_size_left = 0; + av_log(s, AV_LOG_ERROR, + "invalid packet header length %d for pktlen %d-%d at %"PRId64"\n", + rsize, packet_length, padsize, avio_tell(pb)); + return -1; + } asf->packet_size_left = packet_length - padsize - rsize; if (packet_length < asf->hdr.min_pktsize) padsize += asf->hdr.min_pktsize - packet_length; From bba43a1ea07392f14c508aeff2ee13a4cfc425b5 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Fri, 17 Feb 2012 16:27:36 -0800 Subject: [PATCH 034/154] mjpegb: don't return 0 at the end of frame decoding. Return 0 indicates "please return the same data again", i.e. it causes an infinite loop. Instead, return that we consumed the buffer if we finished decoding succesfully, or return an error if an error occurred. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 74699ac8c8b562e9f8d26e21482b89585365774a) Signed-off-by: Anton Khirnov --- libavcodec/mjpegbdec.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libavcodec/mjpegbdec.c b/libavcodec/mjpegbdec.c index 4ad17ab3ce..9f71f508ae 100644 --- a/libavcodec/mjpegbdec.c +++ b/libavcodec/mjpegbdec.c @@ -66,7 +66,7 @@ read_header: if (get_bits_long(&hgb, 32) != MKBETAG('m','j','p','g')) { av_log(avctx, AV_LOG_WARNING, "not mjpeg-b (bad fourcc)\n"); - return 0; + return AVERROR_INVALIDDATA; } field_size = get_bits_long(&hgb, 32); /* field size */ @@ -146,7 +146,7 @@ read_header: picture->quality*= FF_QP2LAMBDA; } - return buf_ptr - buf; + return buf_size; } AVCodec ff_mjpegb_decoder = { From fe710f2074a711b5b07b76fe9ecf11b4068b32ef Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Fri, 17 Feb 2012 16:57:00 -0800 Subject: [PATCH 035/154] wma: don't return 0 on invalid packets. Return 0 means "please return the same data again", i.e. it causes an infinite loop. Instead, return an error. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 9d3050d3e95e307ebc34a943484c7add838d1220) Signed-off-by: Anton Khirnov --- libavcodec/wmadec.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/libavcodec/wmadec.c b/libavcodec/wmadec.c index 5600f9ba90..afc0658eac 100644 --- a/libavcodec/wmadec.c +++ b/libavcodec/wmadec.c @@ -817,8 +817,12 @@ static int wma_decode_superframe(AVCodecContext *avctx, void *data, s->last_superframe_len = 0; return 0; } - if (buf_size < s->block_align) - return 0; + if (buf_size < s->block_align) { + av_log(avctx, AV_LOG_ERROR, + "Input packet size too small (%d < %d)\n", + buf_size, s->block_align); + return AVERROR_INVALIDDATA; + } buf_size = s->block_align; init_get_bits(&s->gb, buf, buf_size*8); From 8011a29fa8875aa4de54199bdfcd4e5331d532dd Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Fri, 17 Feb 2012 14:18:22 -0800 Subject: [PATCH 036/154] vc1parse: call vc1_init_common(). The parser uses VLC tables initialized in vc1_common_init(), therefore we should call this function on parser init also. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit c742ab4e81bb9dcabfdab006d6b8b09a5808c4ce) Conflicts: libavcodec/vc1.h Signed-off-by: Anton Khirnov --- libavcodec/vc1.h | 1 + libavcodec/vc1_parser.c | 2 +- libavcodec/vc1dec.c | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/libavcodec/vc1.h b/libavcodec/vc1.h index 6096077660..5ce0cb53cb 100644 --- a/libavcodec/vc1.h +++ b/libavcodec/vc1.h @@ -447,5 +447,6 @@ int vc1_decode_entry_point(AVCodecContext *avctx, VC1Context *v, GetBitContext * int vc1_parse_frame_header (VC1Context *v, GetBitContext *gb); int vc1_parse_frame_header_adv(VC1Context *v, GetBitContext *gb); +int ff_vc1_init_common(VC1Context *v); #endif /* AVCODEC_VC1_H */ diff --git a/libavcodec/vc1_parser.c b/libavcodec/vc1_parser.c index 0cc5ea0fa8..cdea0d7a80 100644 --- a/libavcodec/vc1_parser.c +++ b/libavcodec/vc1_parser.c @@ -188,7 +188,7 @@ static int vc1_parse_init(AVCodecParserContext *s) { VC1ParseContext *vpc = s->priv_data; vpc->v.s.slice_context_count = 1; - return 0; + return ff_vc1_init_common(&vpc->v); } AVCodecParser ff_vc1_parser = { diff --git a/libavcodec/vc1dec.c b/libavcodec/vc1dec.c index 3869d92518..3e84464135 100644 --- a/libavcodec/vc1dec.c +++ b/libavcodec/vc1dec.c @@ -67,7 +67,7 @@ static const int offset_table2[9] = { 0, 1, 3, 7, 15, 31, 63, 127, 255 }; * @param v The VC1Context to initialize * @return Status */ -static int vc1_init_common(VC1Context *v) +int ff_vc1_init_common(VC1Context *v) { static int done = 0; int i = 0; @@ -5273,7 +5273,7 @@ static av_cold int vc1_decode_init(AVCodecContext *avctx) avctx->idct_algo = FF_IDCT_WMV2; } - if (vc1_init_common(v) < 0) + if (ff_vc1_init_common(v) < 0) return -1; ff_vc1dsp_init(&v->vc1dsp); From 62beae313a4f91e8ff4e8dc0b2ec78baaa804b32 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Tue, 21 Feb 2012 16:34:08 +0100 Subject: [PATCH 037/154] avplay: fix -threads option The AVOptions based default to threads auto in 2473a45c8 works only if avplay does not use custom option handling for -threads. CC: (cherry picked from commit e48a70e6da02cd5426b6340af70410bdfe27dfa7) Signed-off-by: Anton Khirnov --- avplay.c | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/avplay.c b/avplay.c index 432afc11b3..57fb864ea8 100644 --- a/avplay.c +++ b/avplay.c @@ -242,7 +242,6 @@ static int64_t duration = AV_NOPTS_VALUE; static int debug = 0; static int debug_mv = 0; static int step = 0; -static int thread_count = 1; static int workaround_bugs = 1; static int fast = 0; static int genpts = 0; @@ -2189,7 +2188,6 @@ static int stream_component_open(VideoState *is, int stream_index) avctx->skip_loop_filter = skip_loop_filter; avctx->error_recognition = error_recognition; avctx->error_concealment = error_concealment; - avctx->thread_count = thread_count; if (lowres) avctx->flags |= CODEC_FLAG_EMU_EDGE; if (fast) avctx->flags2 |= CODEC_FLAG2_FAST; @@ -2954,15 +2952,6 @@ static int opt_vismv(const char *opt, const char *arg) return 0; } -static int opt_thread_count(const char *opt, const char *arg) -{ - thread_count = parse_number_or_die(opt, arg, OPT_INT64, 0, INT_MAX); -#if !HAVE_THREADS - fprintf(stderr, "Warning: not compiled with thread support, using thread emulation\n"); -#endif - return 0; -} - static const OptionDef options[] = { #include "cmdutils_common_opts.h" { "x", HAS_ARG, { (void*)opt_width }, "force displayed width", "width" }, @@ -2995,7 +2984,6 @@ static const OptionDef options[] = { { "er", OPT_INT | HAS_ARG | OPT_EXPERT, { (void*)&error_recognition }, "set error detection threshold (0-4)", "threshold" }, { "ec", OPT_INT | HAS_ARG | OPT_EXPERT, { (void*)&error_concealment }, "set error concealment options", "bit_mask" }, { "sync", HAS_ARG | OPT_EXPERT, { (void*)opt_sync }, "set audio-video sync. type (type=audio/video/ext)", "type" }, - { "threads", HAS_ARG | OPT_EXPERT, { (void*)opt_thread_count }, "thread count", "count" }, { "autoexit", OPT_BOOL | OPT_EXPERT, { (void*)&autoexit }, "exit at the end", "" }, { "exitonkeydown", OPT_BOOL | OPT_EXPERT, { (void*)&exit_on_keydown }, "exit on key down", "" }, { "exitonmousedown", OPT_BOOL | OPT_EXPERT, { (void*)&exit_on_mousedown }, "exit on mouse down", "" }, From 0312969b9ea7fa7027bca665bfded88690c4caa0 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Tue, 21 Feb 2012 10:36:27 -0800 Subject: [PATCH 038/154] rmdec: when using INT4 deinterleaving, error out if sub_packet_h <= 1. We read sub_packet_h / 2 packets per line of data (during deinterleaving), which equals zero if sub_packet_h <= 1, thus causing us to not read any data, leading to an infinite loop. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit e30b3e59a4f3004337cb1623b2aac988ce52b93f) Signed-off-by: Anton Khirnov --- libavformat/rmdec.c | 1 + 1 file changed, 1 insertion(+) diff --git a/libavformat/rmdec.c b/libavformat/rmdec.c index 75e4833c4c..3d922530e5 100644 --- a/libavformat/rmdec.c +++ b/libavformat/rmdec.c @@ -265,6 +265,7 @@ static int rm_read_audio_stream_info(AVFormatContext *s, AVIOContext *pb, switch (ast->deint_id) { case DEINT_ID_INT4: if (ast->coded_framesize > ast->audio_framesize || + sub_packet_h <= 1 || ast->coded_framesize * sub_packet_h > (2 + (sub_packet_h & 1)) * ast->audio_framesize) return AVERROR_INVALIDDATA; break; From 8e3dc37bc01950915dcdab473fc2694fc3670a54 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Wed, 22 Feb 2012 12:19:52 -0800 Subject: [PATCH 039/154] truemotion2: error out if the huffman tree has no nodes. This prevents crashers and errors further down when reading nodes in the empty tree. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 2b83e8b7005d531bc78b0fd4f699e9faa54ce9bb) Signed-off-by: Anton Khirnov --- libavcodec/truemotion2.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavcodec/truemotion2.c b/libavcodec/truemotion2.c index 4045342ffa..29d2e4d057 100644 --- a/libavcodec/truemotion2.c +++ b/libavcodec/truemotion2.c @@ -132,7 +132,7 @@ static int tm2_build_huff_table(TM2Context *ctx, TM2Codes *code) huff.val_bits, huff.max_bits); return -1; } - if((huff.nodes < 0) || (huff.nodes > 0x10000)) { + if((huff.nodes <= 0) || (huff.nodes > 0x10000)) { av_log(ctx->avctx, AV_LOG_ERROR, "Incorrect number of Huffman tree nodes: %i\n", huff.nodes); return -1; } From 4f48417fe768a2d0d1852489463530a9a889fe76 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Thu, 23 Feb 2012 11:53:27 -0800 Subject: [PATCH 040/154] swf: check return values for av_get/new_packet(). Prevents crashers when using the packet if allocation failed. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 31632e73f47d25e2077fce729571259ee6354854) Signed-off-by: Anton Khirnov --- libavformat/swfdec.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/libavformat/swfdec.c b/libavformat/swfdec.c index 1fc301b696..6966176a34 100644 --- a/libavformat/swfdec.c +++ b/libavformat/swfdec.c @@ -84,7 +84,7 @@ static int swf_read_packet(AVFormatContext *s, AVPacket *pkt) SWFContext *swf = s->priv_data; AVIOContext *pb = s->pb; AVStream *vst = NULL, *ast = NULL, *st = 0; - int tag, len, i, frame, v; + int tag, len, i, frame, v, res; for(;;) { uint64_t pos = avio_tell(pb); @@ -150,7 +150,8 @@ static int swf_read_packet(AVFormatContext *s, AVPacket *pkt) st = s->streams[i]; if (st->codec->codec_type == AVMEDIA_TYPE_VIDEO && st->id == ch_id) { frame = avio_rl16(pb); - av_get_packet(pb, pkt, len-2); + if ((res = av_get_packet(pb, pkt, len-2)) < 0) + return res; pkt->pos = pos; pkt->pts = frame; pkt->stream_index = st->index; @@ -163,9 +164,11 @@ static int swf_read_packet(AVFormatContext *s, AVPacket *pkt) if (st->codec->codec_type == AVMEDIA_TYPE_AUDIO && st->id == -1) { if (st->codec->codec_id == CODEC_ID_MP3) { avio_skip(pb, 4); - av_get_packet(pb, pkt, len-4); + if ((res = av_get_packet(pb, pkt, len-4)) < 0) + return res; } else { // ADPCM, PCM - av_get_packet(pb, pkt, len); + if ((res = av_get_packet(pb, pkt, len)) < 0) + return res; } pkt->pos = pos; pkt->stream_index = st->index; @@ -190,7 +193,8 @@ static int swf_read_packet(AVFormatContext *s, AVPacket *pkt) st = vst; } avio_rl16(pb); /* BITMAP_ID */ - av_new_packet(pkt, len-2); + if ((res = av_new_packet(pkt, len-2)) < 0) + return res; avio_read(pb, pkt->data, 4); if (AV_RB32(pkt->data) == 0xffd8ffd9 || AV_RB32(pkt->data) == 0xffd9ffd8) { From 424b6edd1944cf02261109edb5913417cf8e5dfb Mon Sep 17 00:00:00 2001 From: Alex Converse Date: Thu, 23 Feb 2012 10:47:50 -0800 Subject: [PATCH 041/154] tiff: Prevent overreads in the type_sizes array. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 447363870f2f91e125e07ac2d0820359a5d86b06) Signed-off-by: Anton Khirnov --- libavcodec/tiff.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/libavcodec/tiff.c b/libavcodec/tiff.c index a88d0f988b..6810f81b35 100644 --- a/libavcodec/tiff.c +++ b/libavcodec/tiff.c @@ -289,6 +289,11 @@ static int tiff_decode_tag(TiffContext *s, const uint8_t *start, const uint8_t * count = tget_long(&buf, s->le); off = tget_long(&buf, s->le); + if (type == 0 || type >= FF_ARRAY_ELEMS(type_sizes)) { + av_log(s->avctx, AV_LOG_DEBUG, "Unknown tiff type (%u) encountered\n", type); + return 0; + } + if(count == 1){ switch(type){ case TIFF_BYTE: @@ -310,10 +315,12 @@ static int tiff_decode_tag(TiffContext *s, const uint8_t *start, const uint8_t * value = -1; buf = start + off; } - }else if(type_sizes[type] * count <= 4){ - buf -= 4; - }else{ - buf = start + off; + } else { + if (count <= 4 && type_sizes[type] * count <= 4) { + buf -= 4; + } else { + buf = start + off; + } } if(buf && (buf < start || buf > end_buf)){ From bf6d1a1ca792e4207e5d9b71c5020befb2296ae3 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Thu, 23 Feb 2012 12:22:40 -0800 Subject: [PATCH 042/154] mjpeg: abort decoding if packet is too large. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit ab492ca2ab105aeb24d955f3f03756bdb3139ee1) Signed-off-by: Anton Khirnov --- libavcodec/mjpegdec.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libavcodec/mjpegdec.c b/libavcodec/mjpegdec.c index 2ae502ddeb..49d334bb7e 100644 --- a/libavcodec/mjpegdec.c +++ b/libavcodec/mjpegdec.c @@ -1466,6 +1466,10 @@ int ff_mjpeg_decode_frame(AVCodecContext *avctx, void *data, int *data_size, /* EOF */ if (start_code < 0) { goto the_end; + } else if (unescaped_buf_size > (1U<<29)) { + av_log(avctx, AV_LOG_ERROR, "MJPEG packet 0x%x too big (0x%x/0x%x), corrupt data?\n", + start_code, unescaped_buf_ptr, buf_size); + return AVERROR_INVALIDDATA; } else { av_log(avctx, AV_LOG_DEBUG, "marker=%x avail_size_in_buf=%td\n", start_code, buf_end - buf_ptr); From 19f4943d12968a6dfb7c2915da191489dc614b87 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Thu, 23 Feb 2012 16:09:36 -0800 Subject: [PATCH 043/154] lcl: error out if uncompressed input buffer is smaller than framesize. This prevents crashes when trying to read beyond the end of the buffer while decoding frame data. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit be129271eac04f91393bf42a490ec631e1a9abea) Signed-off-by: Anton Khirnov --- libavcodec/lcldec.c | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/libavcodec/lcldec.c b/libavcodec/lcldec.c index b66a3ce65b..5b18418169 100644 --- a/libavcodec/lcldec.c +++ b/libavcodec/lcldec.c @@ -223,8 +223,29 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, AVPac len = mszh_dlen; } break; - case COMP_MSZH_NOCOMP: + case COMP_MSZH_NOCOMP: { + int bppx2; + switch (c->imgtype) { + case IMGTYPE_YUV111: + case IMGTYPE_RGB24: + bppx2 = 6; + break; + case IMGTYPE_YUV422: + case IMGTYPE_YUV211: + bppx2 = 4; + break; + case IMGTYPE_YUV411: + case IMGTYPE_YUV420: + bppx2 = 3; + break; + default: + bppx2 = 0; // will error out below + break; + } + if (len < ((width * height * bppx2) >> 1)) + return AVERROR_INVALIDDATA; break; + } default: av_log(avctx, AV_LOG_ERROR, "BUG! Unknown MSZH compression in frame decoder.\n"); return -1; From e537dc230b2e123be8aebdaeee5a7d7787328b0b Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Thu, 29 Dec 2011 09:07:32 -0800 Subject: [PATCH 044/154] kgv1: use avctx->get/release_buffer(). Also fixes crashes on corrupt bitstreams. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 33cd32b389864f2437c94e6fd7dc109ff5f0ed06) Signed-off-by: Anton Khirnov --- libavcodec/kgv1dec.c | 64 +++++++++++++++++++++++++++----------------- 1 file changed, 39 insertions(+), 25 deletions(-) diff --git a/libavcodec/kgv1dec.c b/libavcodec/kgv1dec.c index 2d6fa73fc2..4526bf9658 100644 --- a/libavcodec/kgv1dec.c +++ b/libavcodec/kgv1dec.c @@ -30,10 +30,17 @@ typedef struct { AVCodecContext *avctx; - AVFrame pic; - uint16_t *prev, *cur; + AVFrame prev, cur; } KgvContext; +static void decode_flush(AVCodecContext *avctx) +{ + KgvContext * const c = avctx->priv_data; + + if (c->prev.data[0]) + avctx->release_buffer(avctx, &c->prev); +} + static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, AVPacket *avpkt) { const uint8_t *buf = avpkt->data; @@ -42,7 +49,7 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, AVPac int offsets[7]; uint16_t *out, *prev; int outcnt = 0, maxcnt; - int w, h, i; + int w, h, i, res; if (avpkt->size < 2) return -1; @@ -59,15 +66,15 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, AVPac maxcnt = w * h; - out = av_realloc(c->cur, w * h * 2); - if (!out) - return -1; - c->cur = out; - - prev = av_realloc(c->prev, w * h * 2); - if (!prev) - return -1; - c->prev = prev; + c->cur.reference = 3; + if ((res = avctx->get_buffer(avctx, &c->cur)) < 0) + return res; + out = (uint16_t *) c->cur.data[0]; + if (c->prev.data[0]) { + prev = (uint16_t *) c->prev.data[0]; + } else { + prev = NULL; + } for (i = 0; i < 7; i++) offsets[i] = -1; @@ -80,6 +87,7 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, AVPac out[outcnt++] = code; // rgb555 pixel coded directly } else { int count; + int inp_off; uint16_t *inp; if ((code & 0x6000) == 0x6000) { @@ -101,7 +109,14 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, AVPac if (maxcnt - start < count) break; - inp = prev + start; + if (!prev) { + av_log(avctx, AV_LOG_ERROR, + "Frame reference does not exist\n"); + break; + } + + inp = prev; + inp_off = start; } else { // copy from earlier in this frame int offset = (code & 0x1FFF) + 1; @@ -119,27 +134,28 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, AVPac if (outcnt < offset) break; - inp = out + outcnt - offset; + inp = out; + inp_off = outcnt - offset; } if (maxcnt - outcnt < count) break; - for (i = 0; i < count; i++) + for (i = inp_off; i < count + inp_off; i++) { out[outcnt++] = inp[i]; + } } } if (outcnt - maxcnt) av_log(avctx, AV_LOG_DEBUG, "frame finished with %d diff\n", outcnt - maxcnt); - c->pic.data[0] = (uint8_t *)c->cur; - c->pic.linesize[0] = w * 2; - *data_size = sizeof(AVFrame); - *(AVFrame*)data = c->pic; + *(AVFrame*)data = c->cur; - FFSWAP(uint16_t *, c->cur, c->prev); + if (c->prev.data[0]) + avctx->release_buffer(avctx, &c->prev); + FFSWAP(AVFrame, c->cur, c->prev); return avpkt->size; } @@ -150,17 +166,14 @@ static av_cold int decode_init(AVCodecContext *avctx) c->avctx = avctx; avctx->pix_fmt = PIX_FMT_RGB555; + avctx->flags |= CODEC_FLAG_EMU_EDGE; return 0; } static av_cold int decode_end(AVCodecContext *avctx) { - KgvContext * const c = avctx->priv_data; - - av_freep(&c->cur); - av_freep(&c->prev); - + decode_flush(avctx); return 0; } @@ -172,5 +185,6 @@ AVCodec ff_kgv1_decoder = { .init = decode_init, .close = decode_end, .decode = decode_frame, + .flush = decode_flush, .long_name = NULL_IF_CONFIG_SMALL("Kega Game Video"), }; From a0473085f3e2300908b1bf7ecf2ed7177eef0d4f Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Fri, 24 Feb 2012 16:27:53 -0800 Subject: [PATCH 045/154] kgv1: release reference picture on size change. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 6c4c27adb61b2881a94ce5c7d97ee1c8adadb5fe) Signed-off-by: Anton Khirnov --- libavcodec/kgv1dec.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libavcodec/kgv1dec.c b/libavcodec/kgv1dec.c index 4526bf9658..c4c3dac016 100644 --- a/libavcodec/kgv1dec.c +++ b/libavcodec/kgv1dec.c @@ -61,8 +61,11 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, AVPac if (av_image_check_size(w, h, 0, avctx)) return -1; - if (w != avctx->width || h != avctx->height) + if (w != avctx->width || h != avctx->height) { + if (c->prev.data[0]) + avctx->release_buffer(avctx, &c->prev); avcodec_set_dimensions(avctx, w, h); + } maxcnt = w * h; From 0d30e2c6f28dc0ae1bcb9bb40b26aedb5b5ce731 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Fri, 24 Feb 2012 14:11:04 -0800 Subject: [PATCH 046/154] fraps: release reference buffer on pix_fmt change. Prevents crash when trying to copy from a non-existing plane in e.g. a RGB32 reference image to a YUV420P target image Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 830f70442a87a31f7c75565e9380e3caf8333b8a) Signed-off-by: Anton Khirnov --- libavcodec/fraps.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/libavcodec/fraps.c b/libavcodec/fraps.c index 1444eda979..d887cde0fc 100644 --- a/libavcodec/fraps.c +++ b/libavcodec/fraps.c @@ -138,7 +138,7 @@ static int decode_frame(AVCodecContext *avctx, uint32_t *luma1,*luma2,*cb,*cr; uint32_t offs[4]; int i, j, is_chroma, planes; - + enum PixelFormat pix_fmt; header = AV_RL32(buf); version = header & 0xff; @@ -155,12 +155,16 @@ static int decode_frame(AVCodecContext *avctx, if (header_size == 8) buf+=4; + pix_fmt = version & 1 ? PIX_FMT_BGR24 : PIX_FMT_YUVJ420P; + if (avctx->pix_fmt != pix_fmt && f->data[0]) { + avctx->release_buffer(avctx, f); + } + avctx->pix_fmt = pix_fmt; + switch(version) { case 0: default: /* Fraps v0 is a reordered YUV420 */ - avctx->pix_fmt = PIX_FMT_YUVJ420P; - if ( (buf_size != avctx->width*avctx->height*3/2+header_size) && (buf_size != header_size) ) { av_log(avctx, AV_LOG_ERROR, @@ -208,8 +212,6 @@ static int decode_frame(AVCodecContext *avctx, case 1: /* Fraps v1 is an upside-down BGR24 */ - avctx->pix_fmt = PIX_FMT_BGR24; - if ( (buf_size != avctx->width*avctx->height*3+header_size) && (buf_size != header_size) ) { av_log(avctx, AV_LOG_ERROR, @@ -244,7 +246,6 @@ static int decode_frame(AVCodecContext *avctx, * Fraps v2 is Huffman-coded YUV420 planes * Fraps v4 is virtually the same */ - avctx->pix_fmt = PIX_FMT_YUVJ420P; planes = 3; f->reference = 1; f->buffer_hints = FF_BUFFER_HINTS_VALID | @@ -287,7 +288,6 @@ static int decode_frame(AVCodecContext *avctx, case 3: case 5: /* Virtually the same as version 4, but is for RGB24 */ - avctx->pix_fmt = PIX_FMT_BGR24; planes = 3; f->reference = 1; f->buffer_hints = FF_BUFFER_HINTS_VALID | From abe35728786d79cd8230dffe41205b28ad6b7678 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Wed, 22 Feb 2012 11:33:24 -0800 Subject: [PATCH 047/154] rm: prevent infinite loops for index parsing. Specifically, prevent jumping back in the file for the next index, since this can lead to infinite loops where we jump between indexes referring to each other, and don't read indexes that don't fit in the file. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit aac07a7a4c2c7a4a29cf6dbc88c1b9fdd191b99d) Signed-off-by: Reinhard Tartler --- libavformat/rmdec.c | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/libavformat/rmdec.c b/libavformat/rmdec.c index 3d922530e5..405162e8ca 100644 --- a/libavformat/rmdec.c +++ b/libavformat/rmdec.c @@ -370,8 +370,19 @@ static int rm_read_index(AVFormatContext *s) st = s->streams[n]; break; } - if (n == s->nb_streams) + if (n == s->nb_streams) { + av_log(s, AV_LOG_ERROR, + "Invalid stream index %d for index at pos %"PRId64"\n", + str_id, avio_tell(pb)); goto skip; + } else if ((avio_size(pb) - avio_tell(pb)) / 14 < n_pkts) { + av_log(s, AV_LOG_ERROR, + "Nr. of packets in packet index for stream index %d " + "exceeds filesize (%"PRId64" at %"PRId64" = %d)\n", + str_id, avio_size(pb), avio_tell(pb), + (avio_size(pb) - avio_tell(pb)) / 14); + goto skip; + } for (n = 0; n < n_pkts; n++) { avio_skip(pb, 2); @@ -383,9 +394,12 @@ static int rm_read_index(AVFormatContext *s) } skip: - if (next_off && avio_tell(pb) != next_off && - avio_seek(pb, next_off, SEEK_SET) < 0) + if (next_off && avio_tell(pb) < next_off && + avio_seek(pb, next_off, SEEK_SET) < 0) { + av_log(s, AV_LOG_ERROR, + "Non-linear index detected, not supported\n"); return -1; + } } while (next_off); return 0; From 0f839cff6bf4569393cd0594f0f300af1c488723 Mon Sep 17 00:00:00 2001 From: Reinhard Tartler Date: Sun, 26 Feb 2012 10:50:45 +0100 Subject: [PATCH 048/154] Fix parser not to clobber has_b_frames when extradata is set. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Because in contrast to the decoder, the parser does not setup low_delay. The code in parse_nal_units would always end up setting has_b_frames to "1", except when stream is explicitly marked as low delay. Since the parser itself would create 'extradata', simply reopening the parser would cause this. This happens for instance in estimate_timings_from_pts(), which causes the parser to be reopened on the same stream. This fixes Libav #22 and FFmpeg (trac) #360 CC: libav-stable@libav.org Based on a patch by Reimar Döffinger (commit 31ac0ac29b6bba744493f7d1040757a3f51b9ad7) Comments and description adapted by Reinhard Tartler. Signed-off-by: Reinhard Tartler (cherry picked from commit 790a367d9ecd04360f78616765ee723f3fe65645) Signed-off-by: Reinhard Tartler --- libavcodec/h264_parser.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/libavcodec/h264_parser.c b/libavcodec/h264_parser.c index bcaa04a115..48215c5ada 100644 --- a/libavcodec/h264_parser.c +++ b/libavcodec/h264_parser.c @@ -251,6 +251,13 @@ static int h264_parse(AVCodecParserContext *s, h->got_first = 1; if (avctx->extradata_size) { h->s.avctx = avctx; + // must be done like in the decoder. + // otherwise opening the parser, creating extradata, + // and then closing and opening again + // will cause has_b_frames to be always set. + // NB: estimate_timings_from_pts behaves exactly like this. + if (!avctx->has_b_frames) + h->s.low_delay = 1; ff_h264_decode_extradata(h); } } From 2510e1476e9a8bfcca0fe4e85a1380482aed0ab3 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Tue, 10 Jan 2012 17:01:26 -0800 Subject: [PATCH 049/154] vorbis: fix overflows in floor1[] vector and inverse db table index. (cherry picked from commit 24947d4988012f1f0fd467c83418615adc11c3e8) Signed-off-by: Reinhard Tartler --- libavcodec/vorbis.c | 19 +++++++++---------- libavcodec/vorbisdec.c | 10 +++++----- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/libavcodec/vorbis.c b/libavcodec/vorbis.c index 0b26870421..52ded8b0a8 100644 --- a/libavcodec/vorbis.c +++ b/libavcodec/vorbis.c @@ -152,7 +152,7 @@ void ff_vorbis_ready_floor1_list(vorbis_floor1_entry * list, int values) } } -static inline void render_line_unrolled(intptr_t x, uint8_t y, int x1, +static inline void render_line_unrolled(intptr_t x, int y, int x1, intptr_t sy, int ady, int adx, float *buf) { @@ -164,30 +164,30 @@ static inline void render_line_unrolled(intptr_t x, uint8_t y, int x1, if (err >= 0) { err += ady - adx; y += sy; - buf[x++] = ff_vorbis_floor1_inverse_db_table[y]; + buf[x++] = ff_vorbis_floor1_inverse_db_table[av_clip_uint8(y)]; } - buf[x] = ff_vorbis_floor1_inverse_db_table[y]; + buf[x] = ff_vorbis_floor1_inverse_db_table[av_clip_uint8(y)]; } if (x <= 0) { if (err + ady >= 0) y += sy; - buf[x] = ff_vorbis_floor1_inverse_db_table[y]; + buf[x] = ff_vorbis_floor1_inverse_db_table[av_clip_uint8(y)]; } } -static void render_line(int x0, uint8_t y0, int x1, int y1, float *buf) +static void render_line(int x0, int y0, int x1, int y1, float *buf) { int dy = y1 - y0; int adx = x1 - x0; int ady = FFABS(dy); int sy = dy < 0 ? -1 : 1; - buf[x0] = ff_vorbis_floor1_inverse_db_table[y0]; + buf[x0] = ff_vorbis_floor1_inverse_db_table[av_clip_uint8(y0)]; if (ady*2 <= adx) { // optimized common case render_line_unrolled(x0, y0, x1, sy, ady, adx, buf); } else { int base = dy / adx; int x = x0; - uint8_t y = y0; + int y = y0; int err = -adx; ady -= FFABS(base) * adx; while (++x < x1) { @@ -197,7 +197,7 @@ static void render_line(int x0, uint8_t y0, int x1, int y1, float *buf) err -= adx; y += sy; } - buf[x] = ff_vorbis_floor1_inverse_db_table[y]; + buf[x] = ff_vorbis_floor1_inverse_db_table[av_clip_uint8(y)]; } } } @@ -206,8 +206,7 @@ void ff_vorbis_floor1_render_list(vorbis_floor1_entry * list, int values, uint16_t *y_list, int *flag, int multiplier, float *out, int samples) { - int lx, i; - uint8_t ly; + int lx, ly, i; lx = 0; ly = y_list[0] * multiplier; for (i = 1; i < values; i++) { diff --git a/libavcodec/vorbisdec.c b/libavcodec/vorbisdec.c index bb69fed254..22a2cf7e8a 100644 --- a/libavcodec/vorbisdec.c +++ b/libavcodec/vorbisdec.c @@ -1244,20 +1244,20 @@ static int vorbis_floor1_decode(vorbis_context *vc, floor1_flag[i] = 1; if (val >= room) { if (highroom > lowroom) { - floor1_Y_final[i] = val - lowroom + predicted; + floor1_Y_final[i] = av_clip_uint16(val - lowroom + predicted); } else { - floor1_Y_final[i] = predicted - val + highroom - 1; + floor1_Y_final[i] = av_clip_uint16(predicted - val + highroom - 1); } } else { if (val & 1) { - floor1_Y_final[i] = predicted - (val + 1) / 2; + floor1_Y_final[i] = av_clip_uint16(predicted - (val + 1) / 2); } else { - floor1_Y_final[i] = predicted + val / 2; + floor1_Y_final[i] = av_clip_uint16(predicted + val / 2); } } } else { floor1_flag[i] = 0; - floor1_Y_final[i] = predicted; + floor1_Y_final[i] = av_clip_uint16(predicted); } av_dlog(NULL, " Decoded floor(%d) = %u / val %u\n", From 9dbd437da2bafbec540e38cb51bc7ce2b0101ee5 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Tue, 28 Feb 2012 10:22:28 -0800 Subject: [PATCH 050/154] Indeo3: fix crashes on corrupt bitstreams. Splits at borders of cells are invalid, since it leaves one of the cells with a width/height of zero. Also, propagate errors on buffer allocation failures, so we don't continue decoding (which crashes). Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit fc9bc08dca9ac32526251e19fcf738d23b8c68d1) Signed-off-by: Reinhard Tartler --- libavcodec/indeo3.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/libavcodec/indeo3.c b/libavcodec/indeo3.c index d2b01f469a..55b4ec7a7a 100644 --- a/libavcodec/indeo3.c +++ b/libavcodec/indeo3.c @@ -724,6 +724,8 @@ static int parse_bintree(Indeo3DecodeContext *ctx, AVCodecContext *avctx, SPLIT_CELL(ref_cell->height, curr_cell.height); ref_cell->ypos += curr_cell.height; ref_cell->height -= curr_cell.height; + if (ref_cell->height <= 0 || curr_cell.height <= 0) + return AVERROR_INVALIDDATA; } else if (code == V_SPLIT) { if (curr_cell.width > strip_width) { /* split strip */ @@ -732,6 +734,8 @@ static int parse_bintree(Indeo3DecodeContext *ctx, AVCodecContext *avctx, SPLIT_CELL(ref_cell->width, curr_cell.width); ref_cell->xpos += curr_cell.width; ref_cell->width -= curr_cell.width; + if (ref_cell->width <= 0 || curr_cell.width <= 0) + return AVERROR_INVALIDDATA; } while (1) { /* loop until return */ @@ -887,13 +891,16 @@ static int decode_frame_headers(Indeo3DecodeContext *ctx, AVCodecContext *avctx, return AVERROR_INVALIDDATA; if (width != ctx->width || height != ctx->height) { + int res; + av_dlog(avctx, "Frame dimensions changed!\n"); ctx->width = width; ctx->height = height; free_frame_buffers(ctx); - allocate_frame_buffers(ctx, avctx); + if ((res = allocate_frame_buffers(ctx, avctx)) < 0) + return res; avcodec_set_dimensions(avctx, width, height); } From 71a939fee47d8b59ba1258b481322d16378e556f Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Tue, 28 Feb 2012 11:35:36 -0800 Subject: [PATCH 051/154] oma: don't read beyond end of leaf_table. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 934cd18a43151ba4b819d9270d539cdb26f6e079) Signed-off-by: Reinhard Tartler --- libavformat/omadec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavformat/omadec.c b/libavformat/omadec.c index 0beed7165d..cc37397010 100644 --- a/libavformat/omadec.c +++ b/libavformat/omadec.c @@ -231,7 +231,7 @@ static int decrypt_init(AVFormatContext *s, ID3v2ExtraMeta *em, uint8_t *header) rprobe(s, gdata, oc->r_val) < 0 && nprobe(s, gdata, oc->n_val) < 0) { int i; - for (i = 0; i < sizeof(leaf_table); i += 2) { + for (i = 0; i < FF_ARRAY_ELEMS(leaf_table); i += 2) { uint8_t buf[16]; AV_WL64(buf, leaf_table[i]); AV_WL64(&buf[8], leaf_table[i+1]); From 083a8a00373b12dc06b8ae4c49eec61fb5e55f4b Mon Sep 17 00:00:00 2001 From: Alex Converse Date: Wed, 25 Jan 2012 13:39:24 -0800 Subject: [PATCH 052/154] mjpegbdec: Fix overflow in SOS. Based in part by a fix from Michael Niedermayer Fixes CVE-2011-3947 Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind (cherry picked from commit b57d262412204e54a7ef8fa1b23ff4dcede622e5) Signed-off-by: Reinhard Tartler --- libavcodec/mjpegbdec.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/libavcodec/mjpegbdec.c b/libavcodec/mjpegbdec.c index 9f71f508ae..10c5addca0 100644 --- a/libavcodec/mjpegbdec.c +++ b/libavcodec/mjpegbdec.c @@ -59,6 +59,9 @@ read_header: s->restart_count = 0; s->mjpb_skiptosod = 0; + if (buf_end - buf_ptr >= 1 << 28) + return AVERROR_INVALIDDATA; + init_get_bits(&hgb, buf_ptr, /*buf_size*/(buf_end - buf_ptr)*8); skip_bits(&hgb, 32); /* reserved zeros */ @@ -111,8 +114,8 @@ read_header: av_log(avctx, AV_LOG_DEBUG, "sod offs: 0x%x\n", sod_offs); if (sos_offs) { -// init_get_bits(&s->gb, buf+sos_offs, (buf_end - (buf+sos_offs))*8); - init_get_bits(&s->gb, buf_ptr+sos_offs, field_size*8); + init_get_bits(&s->gb, buf_ptr + sos_offs, + 8 * FFMIN(field_size, buf_end - buf_ptr - sos_offs)); s->mjpb_skiptosod = (sod_offs - sos_offs - show_bits(&s->gb, 16)); s->start_code = SOS; if (ff_mjpeg_decode_sos(s, NULL, NULL) < 0 && From a1556d37b85328fda3c4010bc2f49e1a93273128 Mon Sep 17 00:00:00 2001 From: Paul B Mahol Date: Sun, 29 Jan 2012 20:09:22 +0000 Subject: [PATCH 053/154] avutil: make intfloat api public The functions are already av_ prefixed and intfloat header is already provided. Install libavutil/intfloat.h Signed-off-by: Paul B Mahol Signed-off-by: Anton Khirnov (cherry picked from commit 8b933129b932f523a746e921a0a20b8dd8816971) Conflicts: doc/APIchanges Signed-off-by: Anton Khirnov --- doc/APIchanges | 4 ++++ libavutil/Makefile | 1 + libavutil/avutil.h | 2 +- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/doc/APIchanges b/doc/APIchanges index 1e326cac3f..b2ee01bd4b 100644 --- a/doc/APIchanges +++ b/doc/APIchanges @@ -13,6 +13,10 @@ libavutil: 2011-04-18 API changes, most recent first: +2012-02-29 - xxxxxxx - lavu 51.22.0 - intfloat.h + Add a new installed header libavutil/intfloat.h with int/float punning + functions. + 2012-02-17 - xxxxxxx - lavc 53.35.0 Add avcodec_is_open() function. diff --git a/libavutil/Makefile b/libavutil/Makefile index 6896846081..4bbe2575bb 100644 --- a/libavutil/Makefile +++ b/libavutil/Makefile @@ -17,6 +17,7 @@ HEADERS = adler32.h \ fifo.h \ file.h \ imgutils.h \ + intfloat.h \ intfloat_readwrite.h \ intreadwrite.h \ lfg.h \ diff --git a/libavutil/avutil.h b/libavutil/avutil.h index f0be5c110a..0e62b4a13f 100644 --- a/libavutil/avutil.h +++ b/libavutil/avutil.h @@ -154,7 +154,7 @@ */ #define LIBAVUTIL_VERSION_MAJOR 51 -#define LIBAVUTIL_VERSION_MINOR 21 +#define LIBAVUTIL_VERSION_MINOR 22 #define LIBAVUTIL_VERSION_MICRO 0 #define LIBAVUTIL_VERSION_INT AV_VERSION_INT(LIBAVUTIL_VERSION_MAJOR, \ From 2ad77c60ef862baa2afcdcb7e6f43dedabab38ef Mon Sep 17 00:00:00 2001 From: Anton Khirnov Date: Fri, 27 Jan 2012 13:33:09 +0100 Subject: [PATCH 054/154] lavf: add functions for accessing the fourcc<->CodecID mapping tables. Fixes bug 212. (cherry picked from commit dd6d3b0e025cb2a16022665dbb8ab1be18dc05e8) Conflicts: doc/APIchanges Signed-off-by: Anton Khirnov --- doc/APIchanges | 3 +++ libavformat/Makefile | 54 +++++++++++++++++++++--------------------- libavformat/avformat.h | 24 +++++++++++++++++++ libavformat/utils.c | 9 +++++++ libavformat/version.h | 2 +- 5 files changed, 64 insertions(+), 28 deletions(-) diff --git a/doc/APIchanges b/doc/APIchanges index b2ee01bd4b..58186a082c 100644 --- a/doc/APIchanges +++ b/doc/APIchanges @@ -13,6 +13,9 @@ libavutil: 2011-04-18 API changes, most recent first: +2012-02-29 - xxxxxxx - lavf 53.21.0 + Add avformat_get_riff_video_tags() and avformat_get_riff_audio_tags(). + 2012-02-29 - xxxxxxx - lavu 51.22.0 - intfloat.h Add a new installed header libavutil/intfloat.h with int/float punning functions. diff --git a/libavformat/Makefile b/libavformat/Makefile index 2a2a946104..c850bf4493 100644 --- a/libavformat/Makefile +++ b/libavformat/Makefile @@ -10,6 +10,7 @@ OBJS = allformats.o \ metadata.o \ options.o \ os_support.o \ + riff.o \ sdp.o \ seek.o \ utils.o \ @@ -25,8 +26,8 @@ OBJS-$(CONFIG_ADX_DEMUXER) += adxdec.o OBJS-$(CONFIG_ADX_MUXER) += rawenc.o OBJS-$(CONFIG_ADTS_MUXER) += adtsenc.o OBJS-$(CONFIG_AEA_DEMUXER) += aea.o pcm.o -OBJS-$(CONFIG_AIFF_DEMUXER) += aiffdec.o riff.o pcm.o -OBJS-$(CONFIG_AIFF_MUXER) += aiffenc.o riff.o +OBJS-$(CONFIG_AIFF_DEMUXER) += aiffdec.o pcm.o +OBJS-$(CONFIG_AIFF_MUXER) += aiffenc.o OBJS-$(CONFIG_AMR_DEMUXER) += amr.o OBJS-$(CONFIG_AMR_MUXER) += amr.o OBJS-$(CONFIG_ANM_DEMUXER) += anm.o @@ -34,14 +35,14 @@ OBJS-$(CONFIG_APC_DEMUXER) += apc.o OBJS-$(CONFIG_APE_DEMUXER) += ape.o apetag.o OBJS-$(CONFIG_APPLEHTTP_DEMUXER) += applehttp.o OBJS-$(CONFIG_ASF_DEMUXER) += asfdec.o asf.o asfcrypt.o \ - riff.o avlanguage.o -OBJS-$(CONFIG_ASF_MUXER) += asfenc.o asf.o riff.o + avlanguage.o +OBJS-$(CONFIG_ASF_MUXER) += asfenc.o asf.o OBJS-$(CONFIG_ASS_DEMUXER) += assdec.o OBJS-$(CONFIG_ASS_MUXER) += assenc.o OBJS-$(CONFIG_AU_DEMUXER) += au.o pcm.o OBJS-$(CONFIG_AU_MUXER) += au.o -OBJS-$(CONFIG_AVI_DEMUXER) += avidec.o riff.o -OBJS-$(CONFIG_AVI_MUXER) += avienc.o riff.o +OBJS-$(CONFIG_AVI_DEMUXER) += avidec.o +OBJS-$(CONFIG_AVI_MUXER) += avienc.o OBJS-$(CONFIG_AVISYNTH) += avisynth.o OBJS-$(CONFIG_AVM2_MUXER) += swfenc.o OBJS-$(CONFIG_AVS_DEMUXER) += avs.o vocdec.o voc.o @@ -51,7 +52,7 @@ OBJS-$(CONFIG_BINK_DEMUXER) += bink.o OBJS-$(CONFIG_BMV_DEMUXER) += bmv.o OBJS-$(CONFIG_C93_DEMUXER) += c93.o vocdec.o voc.o OBJS-$(CONFIG_CAF_DEMUXER) += cafdec.o caf.o mov.o mov_chan.o \ - riff.o isom.o + isom.o OBJS-$(CONFIG_CAVSVIDEO_DEMUXER) += cavsvideodec.o rawdec.o OBJS-$(CONFIG_CAVSVIDEO_MUXER) += rawenc.o OBJS-$(CONFIG_CDG_DEMUXER) += cdg.o @@ -68,7 +69,7 @@ OBJS-$(CONFIG_DTS_DEMUXER) += dtsdec.o rawdec.o OBJS-$(CONFIG_DTS_MUXER) += rawenc.o OBJS-$(CONFIG_DV_DEMUXER) += dv.o OBJS-$(CONFIG_DV_MUXER) += dvenc.o -OBJS-$(CONFIG_DXA_DEMUXER) += dxa.o riff.o +OBJS-$(CONFIG_DXA_DEMUXER) += dxa.o OBJS-$(CONFIG_EA_CDATA_DEMUXER) += eacdata.o OBJS-$(CONFIG_EA_DEMUXER) += electronicarts.o OBJS-$(CONFIG_EAC3_DEMUXER) += ac3dec.o rawdec.o @@ -112,7 +113,7 @@ OBJS-$(CONFIG_INGENIENT_DEMUXER) += ingenientdec.o rawdec.o OBJS-$(CONFIG_IPMOVIE_DEMUXER) += ipmovie.o OBJS-$(CONFIG_ISS_DEMUXER) += iss.o OBJS-$(CONFIG_IV8_DEMUXER) += iv8.o -OBJS-$(CONFIG_IVF_DEMUXER) += ivfdec.o riff.o +OBJS-$(CONFIG_IVF_DEMUXER) += ivfdec.o OBJS-$(CONFIG_IVF_MUXER) += ivfenc.o OBJS-$(CONFIG_JV_DEMUXER) += jvdec.o OBJS-$(CONFIG_LATM_DEMUXER) += rawdec.o @@ -122,9 +123,9 @@ OBJS-$(CONFIG_LXF_DEMUXER) += lxfdec.o OBJS-$(CONFIG_M4V_DEMUXER) += m4vdec.o rawdec.o OBJS-$(CONFIG_M4V_MUXER) += rawenc.o OBJS-$(CONFIG_MATROSKA_DEMUXER) += matroskadec.o matroska.o \ - riff.o isom.o rmdec.o rm.o + isom.o rmdec.o rm.o OBJS-$(CONFIG_MATROSKA_MUXER) += matroskaenc.o matroska.o \ - riff.o isom.o avc.o \ + isom.o avc.o \ flacenc_header.o avlanguage.o OBJS-$(CONFIG_MD5_MUXER) += md5enc.o OBJS-$(CONFIG_MJPEG_DEMUXER) += rawdec.o @@ -133,9 +134,9 @@ OBJS-$(CONFIG_MLP_DEMUXER) += rawdec.o OBJS-$(CONFIG_MLP_MUXER) += rawenc.o OBJS-$(CONFIG_MM_DEMUXER) += mm.o OBJS-$(CONFIG_MMF_DEMUXER) += mmf.o pcm.o -OBJS-$(CONFIG_MMF_MUXER) += mmf.o riff.o -OBJS-$(CONFIG_MOV_DEMUXER) += mov.o riff.o isom.o mov_chan.o -OBJS-$(CONFIG_MOV_MUXER) += movenc.o riff.o isom.o avc.o \ +OBJS-$(CONFIG_MMF_MUXER) += mmf.o +OBJS-$(CONFIG_MOV_DEMUXER) += mov.o isom.o mov_chan.o +OBJS-$(CONFIG_MOV_MUXER) += movenc.o isom.o avc.o \ movenchint.o rtpenc_chain.o \ mov_chan.o OBJS-$(CONFIG_MP2_MUXER) += mp3enc.o rawenc.o @@ -164,9 +165,9 @@ OBJS-$(CONFIG_MXG_DEMUXER) += mxg.o OBJS-$(CONFIG_NC_DEMUXER) += ncdec.o OBJS-$(CONFIG_NSV_DEMUXER) += nsvdec.o OBJS-$(CONFIG_NULL_MUXER) += nullenc.o -OBJS-$(CONFIG_NUT_DEMUXER) += nutdec.o nut.o riff.o -OBJS-$(CONFIG_NUT_MUXER) += nutenc.o nut.o riff.o -OBJS-$(CONFIG_NUV_DEMUXER) += nuv.o riff.o +OBJS-$(CONFIG_NUT_DEMUXER) += nutdec.o nut.o +OBJS-$(CONFIG_NUT_MUXER) += nutenc.o nut.o +OBJS-$(CONFIG_NUV_DEMUXER) += nuv.o OBJS-$(CONFIG_OGG_DEMUXER) += oggdec.o \ oggparsecelt.o \ oggparsedirac.o \ @@ -176,7 +177,6 @@ OBJS-$(CONFIG_OGG_DEMUXER) += oggdec.o \ oggparsespeex.o \ oggparsetheora.o \ oggparsevorbis.o \ - riff.o \ vorbiscomment.o OBJS-$(CONFIG_OGG_MUXER) += oggenc.o \ vorbiscomment.o @@ -301,28 +301,28 @@ OBJS-$(CONFIG_VMD_DEMUXER) += sierravmd.o OBJS-$(CONFIG_VOC_DEMUXER) += vocdec.o voc.o OBJS-$(CONFIG_VOC_MUXER) += vocenc.o voc.o OBJS-$(CONFIG_VQF_DEMUXER) += vqf.o -OBJS-$(CONFIG_W64_DEMUXER) += wav.o riff.o pcm.o -OBJS-$(CONFIG_WAV_DEMUXER) += wav.o riff.o pcm.o -OBJS-$(CONFIG_WAV_MUXER) += wav.o riff.o +OBJS-$(CONFIG_W64_DEMUXER) += wav.o pcm.o +OBJS-$(CONFIG_WAV_DEMUXER) += wav.o pcm.o +OBJS-$(CONFIG_WAV_MUXER) += wav.o OBJS-$(CONFIG_WC3_DEMUXER) += wc3movie.o OBJS-$(CONFIG_WEBM_MUXER) += matroskaenc.o matroska.o \ - riff.o isom.o avc.o \ + isom.o avc.o \ flacenc_header.o avlanguage.o OBJS-$(CONFIG_WSAUD_DEMUXER) += westwood.o OBJS-$(CONFIG_WSVQA_DEMUXER) += westwood.o OBJS-$(CONFIG_WTV_DEMUXER) += wtv.o asfdec.o asf.o asfcrypt.o \ - avlanguage.o mpegts.o isom.o riff.o + avlanguage.o mpegts.o isom.o OBJS-$(CONFIG_WV_DEMUXER) += wv.o apetag.o OBJS-$(CONFIG_XA_DEMUXER) += xa.o -OBJS-$(CONFIG_XMV_DEMUXER) += xmv.o riff.o -OBJS-$(CONFIG_XWMA_DEMUXER) += xwma.o riff.o +OBJS-$(CONFIG_XMV_DEMUXER) += xmv.o +OBJS-$(CONFIG_XWMA_DEMUXER) += xwma.o OBJS-$(CONFIG_YOP_DEMUXER) += yop.o OBJS-$(CONFIG_YUV4MPEGPIPE_MUXER) += yuv4mpeg.o OBJS-$(CONFIG_YUV4MPEGPIPE_DEMUXER) += yuv4mpeg.o # external libraries -OBJS-$(CONFIG_LIBNUT_DEMUXER) += libnut.o riff.o -OBJS-$(CONFIG_LIBNUT_MUXER) += libnut.o riff.o +OBJS-$(CONFIG_LIBNUT_DEMUXER) += libnut.o +OBJS-$(CONFIG_LIBNUT_MUXER) += libnut.o # protocols I/O OBJS+= avio.o aviobuf.o diff --git a/libavformat/avformat.h b/libavformat/avformat.h index 71aed80305..22a89d3cd5 100644 --- a/libavformat/avformat.h +++ b/libavformat/avformat.h @@ -1995,6 +1995,30 @@ int av_match_ext(const char *filename, const char *extensions); */ int avformat_query_codec(AVOutputFormat *ofmt, enum CodecID codec_id, int std_compliance); +/** + * @defgroup riff_fourcc RIFF FourCCs + * @{ + * Get the tables mapping RIFF FourCCs to libavcodec CodecIDs. The tables are + * meant to be passed to av_codec_get_id()/av_codec_get_tag() as in the + * following code: + * @code + * uint32_t tag = MKTAG('H', '2', '6', '4'); + * const struct AVCodecTag *table[] = { avformat_get_riff_video_tags(), 0 }; + * enum CodecID id = av_codec_get_id(table, tag); + * @endcode + */ +/** + * @return the table mapping RIFF FourCCs for video to libavcodec CodecID. + */ +const struct AVCodecTag *avformat_get_riff_video_tags(void); +/** + * @return the table mapping RIFF FourCCs for audio to CodecID. + */ +const struct AVCodecTag *avformat_get_riff_audio_tags(void); +/** + * @} + */ + /** * @} */ diff --git a/libavformat/utils.c b/libavformat/utils.c index e6b4f40cf3..0c355cee60 100644 --- a/libavformat/utils.c +++ b/libavformat/utils.c @@ -4107,3 +4107,12 @@ int ff_add_param_change(AVPacket *pkt, int32_t channels, } return 0; } + +const struct AVCodecTag *avformat_get_riff_video_tags(void) +{ + return ff_codec_bmp_tags; +} +const struct AVCodecTag *avformat_get_riff_audio_tags(void) +{ + return ff_codec_wav_tags; +} diff --git a/libavformat/version.h b/libavformat/version.h index cd774fbe65..009a60b1ad 100644 --- a/libavformat/version.h +++ b/libavformat/version.h @@ -30,7 +30,7 @@ #include "libavutil/avutil.h" #define LIBAVFORMAT_VERSION_MAJOR 53 -#define LIBAVFORMAT_VERSION_MINOR 20 +#define LIBAVFORMAT_VERSION_MINOR 21 #define LIBAVFORMAT_VERSION_MICRO 0 #define LIBAVFORMAT_VERSION_INT AV_VERSION_INT(LIBAVFORMAT_VERSION_MAJOR, \ From 1c63d613721f9fb05dcf1646d00aabf5f63695eb Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Fri, 17 Feb 2012 12:21:22 -0800 Subject: [PATCH 055/154] asf: error out on ridiculously large minpktsize values. They cause various issues further down in demuxing. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 6e57a02b9f639af53acfa9fc742c1341400818f8) Signed-off-by: Reinhard Tartler --- libavformat/asfdec.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/libavformat/asfdec.c b/libavformat/asfdec.c index eb93f14ecf..1fbe79bf5f 100644 --- a/libavformat/asfdec.c +++ b/libavformat/asfdec.c @@ -202,6 +202,8 @@ static int asf_read_file_properties(AVFormatContext *s, int64_t size) asf->hdr.flags = avio_rl32(pb); asf->hdr.min_pktsize = avio_rl32(pb); asf->hdr.max_pktsize = avio_rl32(pb); + if (asf->hdr.min_pktsize >= (1U<<29)) + return AVERROR_INVALIDDATA; asf->hdr.max_bitrate = avio_rl32(pb); s->packet_size = asf->hdr.max_pktsize; @@ -616,7 +618,9 @@ static int asf_read_header(AVFormatContext *s, AVFormatParameters *ap) if (gsize < 24) return -1; if (!ff_guidcmp(&g, &ff_asf_file_header)) { - asf_read_file_properties(s, gsize); + int ret = asf_read_file_properties(s, gsize); + if (ret < 0) + return ret; } else if (!ff_guidcmp(&g, &ff_asf_stream_header)) { asf_read_stream_properties(s, gsize); } else if (!ff_guidcmp(&g, &ff_asf_comment_header)) { From 40ccc811461c2c5f7999200315f9e2a563807147 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Tue, 28 Feb 2012 16:13:46 -0800 Subject: [PATCH 056/154] asf: don't seek back on EOF. Seeking back on EOF will reset the EOF flag, causing us to re-enter the loop to find the next marker in the ASF file, thus potentially causing an infinite loop. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit bb6d5411e1e1a8e0608b1af1c4addee654dcbac5) Signed-off-by: Reinhard Tartler --- libavformat/asfdec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavformat/asfdec.c b/libavformat/asfdec.c index 1fbe79bf5f..969ab28875 100644 --- a/libavformat/asfdec.c +++ b/libavformat/asfdec.c @@ -761,7 +761,7 @@ static int ff_asf_get_packet(AVFormatContext *s, AVIOContext *pb) c= avio_r8(pb); d= avio_r8(pb); rsize+=3; - }else{ + } else if (!pb->eof_reached) { avio_seek(pb, -1, SEEK_CUR); //FIXME } From b2dcac7141a2fb72074679efbefcb4d8bef24c41 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Thu, 23 Feb 2012 11:19:33 -0800 Subject: [PATCH 057/154] vp56: error out on invalid stream dimensions. Prevents crashes when playing corrupt vp5/6 streams. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 8bc396fc0e8769a056375c1c211f389ce0e3ecc5) Signed-off-by: Reinhard Tartler --- libavcodec/vp5.c | 5 +++++ libavcodec/vp6.c | 6 +++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/libavcodec/vp5.c b/libavcodec/vp5.c index 56f667cb63..1c6eaa9d42 100644 --- a/libavcodec/vp5.c +++ b/libavcodec/vp5.c @@ -57,6 +57,11 @@ static int vp5_parse_header(VP56Context *s, const uint8_t *buf, int buf_size, } rows = vp56_rac_gets(c, 8); /* number of stored macroblock rows */ cols = vp56_rac_gets(c, 8); /* number of stored macroblock cols */ + if (!rows || !cols) { + av_log(s->avctx, AV_LOG_ERROR, "Invalid size %dx%d\n", + cols << 4, rows << 4); + return 0; + } vp56_rac_gets(c, 8); /* number of displayed macroblock rows */ vp56_rac_gets(c, 8); /* number of displayed macroblock cols */ vp56_rac_gets(c, 2); diff --git a/libavcodec/vp6.c b/libavcodec/vp6.c index 9433983be3..e4783c6e84 100644 --- a/libavcodec/vp6.c +++ b/libavcodec/vp6.c @@ -77,6 +77,10 @@ static int vp6_parse_header(VP56Context *s, const uint8_t *buf, int buf_size, cols = buf[3]; /* number of stored macroblock cols */ /* buf[4] is number of displayed macroblock rows */ /* buf[5] is number of displayed macroblock cols */ + if (!rows || !cols) { + av_log(s->avctx, AV_LOG_ERROR, "Invalid size %dx%d\n", cols << 4, rows << 4); + return 0; + } if (!s->macroblocks || /* first frame */ 16*cols != s->avctx->coded_width || @@ -97,7 +101,7 @@ static int vp6_parse_header(VP56Context *s, const uint8_t *buf, int buf_size, vrt_shift = 5; s->sub_version = sub_version; } else { - if (!s->sub_version) + if (!s->sub_version || !s->avctx->coded_width || !s->avctx->coded_height) return 0; if (separated_coeff || !s->filter_header) { From 5f896773e07126dd66f5b83e604e99adb30617cb Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Tue, 28 Feb 2012 18:21:31 -0800 Subject: [PATCH 058/154] swscale: fix another integer overflow. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 791de61bbb0d2bceb1037597b310e2a4a94494fd) Signed-off-by: Reinhard Tartler --- libswscale/utils.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libswscale/utils.c b/libswscale/utils.c index 2d7029e2f1..51bc3842dc 100644 --- a/libswscale/utils.c +++ b/libswscale/utils.c @@ -1013,7 +1013,7 @@ int sws_init_context(SwsContext *c, SwsFilter *srcFilter, SwsFilter *dstFilter) c->vLumBufSize= c->vLumFilterSize; c->vChrBufSize= c->vChrFilterSize; for (i=0; ichrDstH / dstH; + int chrI = (int64_t) i * c->chrDstH / dstH; int nextSlice= FFMAX(c->vLumFilterPos[i ] + c->vLumFilterSize - 1, ((c->vChrFilterPos[chrI] + c->vChrFilterSize - 1)<chrSrcVSubSample)); From e904e9b7204b6ebd3433dd49a6c978ffb293cbdc Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Tue, 28 Feb 2012 19:00:39 -0800 Subject: [PATCH 059/154] qtrle: return error on decode_init() failure. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit e54ae60e46f737b8e9a96548971091f7ab6b8f7c) Signed-off-by: Reinhard Tartler --- libavcodec/qtrle.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavcodec/qtrle.c b/libavcodec/qtrle.c index 0c74798226..6e7b3c2a43 100644 --- a/libavcodec/qtrle.c +++ b/libavcodec/qtrle.c @@ -407,7 +407,7 @@ static av_cold int qtrle_decode_init(AVCodecContext *avctx) default: av_log (avctx, AV_LOG_ERROR, "Unsupported colorspace: %d bits/sample?\n", avctx->bits_per_coded_sample); - break; + return AVERROR_INVALIDDATA; } s->frame.data[0] = NULL; From 4493af756b8f8346b1e7671b487afc34c72bc16e Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Tue, 28 Feb 2012 17:04:33 -0800 Subject: [PATCH 060/154] rpza: error out on buffer overreads. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 78e9852a2e3b198ecd69ffa0deab3fa22a8e5378) Signed-off-by: Reinhard Tartler --- libavcodec/rpza.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libavcodec/rpza.c b/libavcodec/rpza.c index 7350ef2c4a..59c3a7b3a7 100644 --- a/libavcodec/rpza.c +++ b/libavcodec/rpza.c @@ -183,6 +183,8 @@ static void rpza_decode_stream(RpzaContext *s) color4[1] |= ((11 * ta + 21 * tb) >> 5); color4[2] |= ((21 * ta + 11 * tb) >> 5); + if (s->size - stream_ptr < n_blocks * 4) + return; while (n_blocks--) { block_ptr = row_ptr + pixel_ptr; for (pixel_y = 0; pixel_y < 4; pixel_y++) { @@ -200,6 +202,8 @@ static void rpza_decode_stream(RpzaContext *s) /* Fill block with 16 colors */ case 0x00: + if (s->size - stream_ptr < 16) + return; block_ptr = row_ptr + pixel_ptr; for (pixel_y = 0; pixel_y < 4; pixel_y++) { for (pixel_x = 0; pixel_x < 4; pixel_x++){ From 1dd1ee00d54ba2a9f5d8ae2e82a22891300b6807 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Tue, 28 Feb 2012 19:00:48 -0800 Subject: [PATCH 061/154] vmnc: return error on decode_init() failure. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 07a180972fb369bb59bf6d4f8edb4598c51e80d2) Signed-off-by: Reinhard Tartler --- libavcodec/vmnc.c | 1 + 1 file changed, 1 insertion(+) diff --git a/libavcodec/vmnc.c b/libavcodec/vmnc.c index a72c507c3c..ca0ba82dcd 100644 --- a/libavcodec/vmnc.c +++ b/libavcodec/vmnc.c @@ -483,6 +483,7 @@ static av_cold int decode_init(AVCodecContext *avctx) break; default: av_log(avctx, AV_LOG_ERROR, "Unsupported bitdepth %i\n", c->bpp); + return AVERROR_INVALIDDATA; } return 0; From a63f3f714c014b3fcaffd45943bc089167b3fe61 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Thu, 1 Mar 2012 09:41:22 -0800 Subject: [PATCH 062/154] huffyuv: do not abort on unknown pix_fmt; instead, return an error. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 63c9de6469005974288f4e4d89fc79a590e38c06) Signed-off-by: Reinhard Tartler --- libavcodec/huffyuv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavcodec/huffyuv.c b/libavcodec/huffyuv.c index efa87de802..412fe4b68d 100644 --- a/libavcodec/huffyuv.c +++ b/libavcodec/huffyuv.c @@ -514,7 +514,7 @@ s->bgr32=1; } break; default: - assert(0); + return AVERROR_INVALIDDATA; } alloc_temp(s); From 750f5baf3036d5a4c488a60d1cd6e872e4a871c4 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Thu, 1 Mar 2012 11:56:05 -0800 Subject: [PATCH 063/154] h264: error out on invalid bitdepth. Fixes invalid reads while initializing the dequant tables, which uses the bit depth to determine the QP table size. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 0ce4fe482c27abfa7eac503a52fdc50b70ccd871) Signed-off-by: Reinhard Tartler --- libavcodec/h264.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/libavcodec/h264.c b/libavcodec/h264.c index e92acbd7a8..449c634cfe 100644 --- a/libavcodec/h264.c +++ b/libavcodec/h264.c @@ -2707,11 +2707,6 @@ static int decode_slice_header(H264Context *h, H264Context *h0){ s->avctx->level = h->sps.level_idc; s->avctx->refs = h->sps.ref_frame_count; - if(h == h0 && h->dequant_coeff_pps != pps_id){ - h->dequant_coeff_pps = pps_id; - init_dequant_tables(h); - } - s->mb_width= h->sps.mb_width; s->mb_height= h->sps.mb_height * (2 - h->sps.frame_mbs_only_flag); @@ -2786,7 +2781,7 @@ static int decode_slice_header(H264Context *h, H264Context *h0){ else s->avctx->pix_fmt = PIX_FMT_YUV420P10; break; - default: + case 8: if (CHROMA444){ if (s->avctx->colorspace == AVCOL_SPC_RGB) { s->avctx->pix_fmt = PIX_FMT_GBRP; @@ -2802,6 +2797,11 @@ static int decode_slice_header(H264Context *h, H264Context *h0){ hwaccel_pixfmt_list_h264_jpeg_420 : ff_hwaccel_pixfmt_list_420); } + break; + default: + av_log(s->avctx, AV_LOG_ERROR, + "Unsupported bit depth: %d\n", h->sps.bit_depth_luma); + return AVERROR_INVALIDDATA; } s->avctx->hwaccel = ff_find_hwaccel(s->avctx->codec->id, s->avctx->pix_fmt); @@ -2846,6 +2846,11 @@ static int decode_slice_header(H264Context *h, H264Context *h0){ } } + if(h == h0 && h->dequant_coeff_pps != pps_id){ + h->dequant_coeff_pps = pps_id; + init_dequant_tables(h); + } + h->frame_num= get_bits(&s->gb, h->sps.log2_max_frame_num); h->mb_mbaff = 0; From 7f3f85544ca7804fde2210c129a4458536330dc6 Mon Sep 17 00:00:00 2001 From: Justin Ruggles Date: Fri, 24 Feb 2012 23:27:14 -0500 Subject: [PATCH 064/154] avutil: add AVERROR_UNKNOWN Useful to return instead of -1 when the cause of the error is unknown, typically from an external library. (cherry picked from commit c9bca801324f03746757aef8549ebd26599adec2) Conflicts: doc/APIchanges libavutil/avutil.h Signed-off-by: Reinhard Tartler --- doc/APIchanges | 3 +++ libavutil/avutil.h | 2 +- libavutil/error.c | 1 + libavutil/error.h | 1 + 4 files changed, 6 insertions(+), 1 deletion(-) diff --git a/doc/APIchanges b/doc/APIchanges index 58186a082c..78e37f4e95 100644 --- a/doc/APIchanges +++ b/doc/APIchanges @@ -13,6 +13,9 @@ libavutil: 2011-04-18 API changes, most recent first: +2012-03-04 - xxxxxxx - lavu 51.22.1 - error.h + Add AVERROR_UNKNOWN + 2012-02-29 - xxxxxxx - lavf 53.21.0 Add avformat_get_riff_video_tags() and avformat_get_riff_audio_tags(). diff --git a/libavutil/avutil.h b/libavutil/avutil.h index 0e62b4a13f..05e9248375 100644 --- a/libavutil/avutil.h +++ b/libavutil/avutil.h @@ -155,7 +155,7 @@ #define LIBAVUTIL_VERSION_MAJOR 51 #define LIBAVUTIL_VERSION_MINOR 22 -#define LIBAVUTIL_VERSION_MICRO 0 +#define LIBAVUTIL_VERSION_MICRO 1 #define LIBAVUTIL_VERSION_INT AV_VERSION_INT(LIBAVUTIL_VERSION_MAJOR, \ LIBAVUTIL_VERSION_MINOR, \ diff --git a/libavutil/error.c b/libavutil/error.c index a330e9f99c..21b68762d2 100644 --- a/libavutil/error.c +++ b/libavutil/error.c @@ -39,6 +39,7 @@ int av_strerror(int errnum, char *errbuf, size_t errbuf_size) case AVERROR_PROTOCOL_NOT_FOUND:errstr = "Protocol not found" ; break; case AVERROR_STREAM_NOT_FOUND: errstr = "Stream not found" ; break; case AVERROR_BUG: errstr = "Bug detected, please report the issue" ; break; + case AVERROR_UNKNOWN: errstr = "Unknown error occurred" ; break; } if (errstr) { diff --git a/libavutil/error.h b/libavutil/error.h index 2db65cb83f..11bcc5c4c4 100644 --- a/libavutil/error.h +++ b/libavutil/error.h @@ -58,6 +58,7 @@ #define AVERROR_PROTOCOL_NOT_FOUND (-MKTAG(0xF8,'P','R','O')) ///< Protocol not found #define AVERROR_STREAM_NOT_FOUND (-MKTAG(0xF8,'S','T','R')) ///< Stream not found #define AVERROR_BUG (-MKTAG( 'B','U','G',' ')) ///< Bug detected, please report the issue +#define AVERROR_UNKNOWN (-MKTAG( 'U','N','K','N')) ///< Unknown error, typically from an external library /** * Put a description of the AVERROR code errnum in errbuf. From 7e88df99e1d26accc56b0da52d271a57995ecde7 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Wed, 29 Feb 2012 17:50:28 -0800 Subject: [PATCH 065/154] lcl: return negative error codes on decode_init() errors. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit bd17a40a7e0eba21b5d27c67aff795e2910766e4) Signed-off-by: Reinhard Tartler --- libavcodec/lcldec.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/libavcodec/lcldec.c b/libavcodec/lcldec.c index 5b18418169..679824cc1e 100644 --- a/libavcodec/lcldec.c +++ b/libavcodec/lcldec.c @@ -476,7 +476,7 @@ static av_cold int decode_init(AVCodecContext *avctx) if (avctx->extradata_size < 8) { av_log(avctx, AV_LOG_ERROR, "Extradata size too small.\n"); - return 1; + return AVERROR_INVALIDDATA; } /* Check codec type */ @@ -525,7 +525,7 @@ static av_cold int decode_init(AVCodecContext *avctx) break; default: av_log(avctx, AV_LOG_ERROR, "Unsupported image format %d.\n", c->imgtype); - return 1; + return AVERROR_INVALIDDATA; } /* Detect compression method */ @@ -542,7 +542,7 @@ static av_cold int decode_init(AVCodecContext *avctx) break; default: av_log(avctx, AV_LOG_ERROR, "Unsupported compression format for MSZH (%d).\n", c->compression); - return 1; + return AVERROR_INVALIDDATA; } break; #if CONFIG_ZLIB_DECODER @@ -560,7 +560,7 @@ static av_cold int decode_init(AVCodecContext *avctx) default: if (c->compression < Z_NO_COMPRESSION || c->compression > Z_BEST_COMPRESSION) { av_log(avctx, AV_LOG_ERROR, "Unsupported compression level for ZLIB: (%d).\n", c->compression); - return 1; + return AVERROR_INVALIDDATA; } av_log(avctx, AV_LOG_DEBUG, "Compression level for ZLIB: (%d).\n", c->compression); } @@ -568,14 +568,14 @@ static av_cold int decode_init(AVCodecContext *avctx) #endif default: av_log(avctx, AV_LOG_ERROR, "BUG! Unknown codec in compression switch.\n"); - return 1; + return AVERROR_INVALIDDATA; } /* Allocate decompression buffer */ if (c->decomp_size) { if ((c->decomp_buf = av_malloc(max_decomp_size)) == NULL) { av_log(avctx, AV_LOG_ERROR, "Can't allocate decompression buffer.\n"); - return 1; + return AVERROR(ENOMEM); } } @@ -601,7 +601,7 @@ static av_cold int decode_init(AVCodecContext *avctx) if (zret != Z_OK) { av_log(avctx, AV_LOG_ERROR, "Inflate init error: %d\n", zret); av_freep(&c->decomp_buf); - return 1; + return AVERROR_UNKNOWN; } } #endif From 19da1a39e861968c27504b67d481d32339669e2a Mon Sep 17 00:00:00 2001 From: Alex Converse Date: Thu, 1 Mar 2012 14:07:22 -0800 Subject: [PATCH 066/154] rv10/20: Fix a buffer overread caused by losing track of the remaining buffer size. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 2f6528537fdd88820f3a4683d5e595d7b3a62689) Signed-off-by: Reinhard Tartler --- libavcodec/rv10.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/libavcodec/rv10.c b/libavcodec/rv10.c index ccc09443ec..d7d7ed2fb2 100644 --- a/libavcodec/rv10.c +++ b/libavcodec/rv10.c @@ -642,8 +642,12 @@ static int rv10_decode_frame(AVCodecContext *avctx, if(!avctx->slice_count){ slice_count = (*buf++) + 1; + buf_size--; slices_hdr = buf + 4; buf += 8 * slice_count; + buf_size -= 8 * slice_count; + if (buf_size <= 0) + return AVERROR_INVALIDDATA; }else slice_count = avctx->slice_count; @@ -682,7 +686,7 @@ static int rv10_decode_frame(AVCodecContext *avctx, s->current_picture_ptr= NULL; //so we can detect if frame_end wasnt called (find some nicer solution...) } - return buf_size; + return avpkt->size; } AVCodec ff_rv10_decoder = { From fecd7468fcbf9115afdd8bf3dc3d08da0975e4d8 Mon Sep 17 00:00:00 2001 From: Alex Converse Date: Fri, 27 Jan 2012 14:24:07 -0800 Subject: [PATCH 067/154] wmadec: Verify bitstream size makes sense before calling init_get_bits. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind (cherry picked from commit 48f1e5212c90b511c90fa0449655abb06a9edda2) Signed-off-by: Reinhard Tartler --- libavcodec/wmadec.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libavcodec/wmadec.c b/libavcodec/wmadec.c index afc0658eac..b9fc21fd3e 100644 --- a/libavcodec/wmadec.c +++ b/libavcodec/wmadec.c @@ -877,6 +877,8 @@ static int wma_decode_superframe(AVCodecContext *avctx, void *data, /* read each frame starting from bit_offset */ pos = bit_offset + 4 + 4 + s->byte_offset_bits + 3; + if (pos >= MAX_CODED_SUPERFRAME_SIZE * 8) + return AVERROR_INVALIDDATA; init_get_bits(&s->gb, buf + (pos >> 3), (MAX_CODED_SUPERFRAME_SIZE - (pos >> 3))*8); len = pos & 7; if (len > 0) From b863979c0f36b565857c49cf6297810e22a9ba10 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Thu, 1 Mar 2012 16:19:51 -0800 Subject: [PATCH 068/154] wma: fix invalid buffer size assumptions causing random overreads. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 349b7977e408f18cff01ab31dfa66c8249b6584a) Signed-off-by: Reinhard Tartler --- libavcodec/wma.h | 2 +- libavcodec/wmadec.c | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/libavcodec/wma.h b/libavcodec/wma.h index 4acbf04bbf..d6f4880c14 100644 --- a/libavcodec/wma.h +++ b/libavcodec/wma.h @@ -124,7 +124,7 @@ typedef struct WMACodecContext { /* output buffer for one frame and the last for IMDCT windowing */ DECLARE_ALIGNED(32, float, frame_out)[MAX_CHANNELS][BLOCK_MAX_SIZE * 2]; /* last frame info */ - uint8_t last_superframe[MAX_CODED_SUPERFRAME_SIZE + 4]; /* padding added */ + uint8_t last_superframe[MAX_CODED_SUPERFRAME_SIZE + FF_INPUT_BUFFER_PADDING_SIZE]; /* padding added */ int last_bitoffset; int last_superframe_len; float noise_table[NOISE_TAB_SIZE]; diff --git a/libavcodec/wmadec.c b/libavcodec/wmadec.c index b9fc21fd3e..37feca1f7f 100644 --- a/libavcodec/wmadec.c +++ b/libavcodec/wmadec.c @@ -845,6 +845,12 @@ static int wma_decode_superframe(AVCodecContext *avctx, void *data, if (s->use_bit_reservoir) { bit_offset = get_bits(&s->gb, s->byte_offset_bits + 3); + if (bit_offset > get_bits_left(&s->gb)) { + av_log(avctx, AV_LOG_ERROR, + "Invalid last frame bit offset %d > buf size %d (%d)\n", + bit_offset, get_bits_left(&s->gb), buf_size); + goto fail; + } if (s->last_superframe_len > 0) { // printf("skip=%d\n", s->last_bitoffset); @@ -861,9 +867,10 @@ static int wma_decode_superframe(AVCodecContext *avctx, void *data, if (len > 0) { *q++ = (get_bits)(&s->gb, len) << (8 - len); } + memset(q, 0, FF_INPUT_BUFFER_PADDING_SIZE); /* XXX: bit_offset bits into last frame */ - init_get_bits(&s->gb, s->last_superframe, MAX_CODED_SUPERFRAME_SIZE*8); + init_get_bits(&s->gb, s->last_superframe, s->last_superframe_len * 8 + bit_offset); /* skip unused bits */ if (s->last_bitoffset > 0) skip_bits(&s->gb, s->last_bitoffset); @@ -877,9 +884,9 @@ static int wma_decode_superframe(AVCodecContext *avctx, void *data, /* read each frame starting from bit_offset */ pos = bit_offset + 4 + 4 + s->byte_offset_bits + 3; - if (pos >= MAX_CODED_SUPERFRAME_SIZE * 8) + if (pos >= MAX_CODED_SUPERFRAME_SIZE * 8 || pos > buf_size * 8) return AVERROR_INVALIDDATA; - init_get_bits(&s->gb, buf + (pos >> 3), (MAX_CODED_SUPERFRAME_SIZE - (pos >> 3))*8); + init_get_bits(&s->gb, buf + (pos >> 3), (buf_size - (pos >> 3))*8); len = pos & 7; if (len > 0) skip_bits(&s->gb, len); From 9686a2c2cfdb103784bd9153042da4f9656b56c6 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Thu, 1 Mar 2012 17:01:22 -0800 Subject: [PATCH 069/154] matroska: check buffer size for RM-style byte reordering. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 9c239f6026a170866a4a0c96908980ac2cfaa8b3) Signed-off-by: Reinhard Tartler --- libavformat/matroskadec.c | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/libavformat/matroskadec.c b/libavformat/matroskadec.c index 1987b5095f..59e0e1f49d 100644 --- a/libavformat/matroskadec.c +++ b/libavformat/matroskadec.c @@ -1808,15 +1808,31 @@ static int matroska_parse_block(MatroskaDemuxContext *matroska, uint8_t *data, if (!track->audio.pkt_cnt) { if (track->audio.sub_packet_cnt == 0) track->audio.buf_timecode = timecode; - if (st->codec->codec_id == CODEC_ID_RA_288) + if (st->codec->codec_id == CODEC_ID_RA_288) { + if (size < cfs * h / 2) { + av_log(matroska->ctx, AV_LOG_ERROR, + "Corrupt int4 RM-style audio packet size\n"); + return AVERROR_INVALIDDATA; + } for (x=0; xaudio.buf+x*2*w+y*cfs, data+x*cfs, cfs); - else if (st->codec->codec_id == CODEC_ID_SIPR) + } else if (st->codec->codec_id == CODEC_ID_SIPR) { + if (size < w) { + av_log(matroska->ctx, AV_LOG_ERROR, + "Corrupt sipr RM-style audio packet size\n"); + return AVERROR_INVALIDDATA; + } memcpy(track->audio.buf + y*w, data, w); - else + } else { + if (size < sps * w / sps) { + av_log(matroska->ctx, AV_LOG_ERROR, + "Corrupt generic RM-style audio packet size\n"); + return AVERROR_INVALIDDATA; + } for (x=0; xaudio.buf+sps*(h*x+((h+1)/2)*(y&1)+(y>>1)), data+x*sps, sps); + } if (++track->audio.sub_packet_cnt >= h) { if (st->codec->codec_id == CODEC_ID_SIPR) From de2656ec2518cae65a2b2823470a3ebe15934ba9 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Thu, 1 Mar 2012 13:51:21 -0800 Subject: [PATCH 070/154] amrwb: error out early if mode is invalid. Prevents using the invalid mode as an index in a static array, which would generate invalid reads. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 154b8bb80029e71d562e8936164266300dd35a0e) Signed-off-by: Reinhard Tartler --- libavcodec/amrwbdec.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/libavcodec/amrwbdec.c b/libavcodec/amrwbdec.c index 6ea5d228dd..0ebaf47441 100644 --- a/libavcodec/amrwbdec.c +++ b/libavcodec/amrwbdec.c @@ -1095,23 +1095,27 @@ static int amrwb_decode_frame(AVCodecContext *avctx, void *data, buf_out = (float *)ctx->avframe.data[0]; header_size = decode_mime_header(ctx, buf); + if (ctx->fr_cur_mode > MODE_SID) { + av_log(avctx, AV_LOG_ERROR, + "Invalid mode %d\n", ctx->fr_cur_mode); + return AVERROR_INVALIDDATA; + } expected_fr_size = ((cf_sizes_wb[ctx->fr_cur_mode] + 7) >> 3) + 1; if (buf_size < expected_fr_size) { av_log(avctx, AV_LOG_ERROR, "Frame too small (%d bytes). Truncated file?\n", buf_size); *got_frame_ptr = 0; - return buf_size; + return AVERROR_INVALIDDATA; } if (!ctx->fr_quality || ctx->fr_cur_mode > MODE_SID) av_log(avctx, AV_LOG_ERROR, "Encountered a bad or corrupted frame\n"); - if (ctx->fr_cur_mode == MODE_SID) /* Comfort noise frame */ + if (ctx->fr_cur_mode == MODE_SID) { /* Comfort noise frame */ av_log_missing_feature(avctx, "SID mode", 1); - - if (ctx->fr_cur_mode >= MODE_SID) return -1; + } ff_amr_bit_reorder((uint16_t *) &ctx->frame, sizeof(AMRWBFrame), buf + header_size, amr_bit_orderings_by_mode[ctx->fr_cur_mode]); From 78d4f8cc56554e5d19c3f5688902278c3b795a04 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Thu, 1 Mar 2012 15:44:25 -0800 Subject: [PATCH 071/154] amrwb: remove duplicate arguments from extrapolate_isf(). Prevents warnings because the dst and src overlap (are the same) in the memcpy() inside the function. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 9d87374ec0f382c8394ad511243db6980afa42af) Signed-off-by: Reinhard Tartler --- libavcodec/amrwbdec.c | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/libavcodec/amrwbdec.c b/libavcodec/amrwbdec.c index 0ebaf47441..b9ae9ece66 100644 --- a/libavcodec/amrwbdec.c +++ b/libavcodec/amrwbdec.c @@ -898,10 +898,10 @@ static float auto_correlation(float *diff_isf, float mean, int lag) * Extrapolate a ISF vector to the 16kHz range (20th order LP) * used at mode 6k60 LP filter for the high frequency band. * - * @param[out] out Buffer for extrapolated isf - * @param[in] isf Input isf vector + * @param[out] isf Buffer for extrapolated isf; contains LP_ORDER + * values on input */ -static void extrapolate_isf(float out[LP_ORDER_16k], float isf[LP_ORDER]) +static void extrapolate_isf(float isf[LP_ORDER_16k]) { float diff_isf[LP_ORDER - 2], diff_mean; float *diff_hi = diff_isf - LP_ORDER + 1; // diff array for extrapolated indexes @@ -909,8 +909,7 @@ static void extrapolate_isf(float out[LP_ORDER_16k], float isf[LP_ORDER]) float est, scale; int i, i_max_corr; - memcpy(out, isf, (LP_ORDER - 1) * sizeof(float)); - out[LP_ORDER_16k - 1] = isf[LP_ORDER - 1]; + isf[LP_ORDER_16k - 1] = isf[LP_ORDER - 1]; /* Calculate the difference vector */ for (i = 0; i < LP_ORDER - 2; i++) @@ -931,16 +930,16 @@ static void extrapolate_isf(float out[LP_ORDER_16k], float isf[LP_ORDER]) i_max_corr++; for (i = LP_ORDER - 1; i < LP_ORDER_16k - 1; i++) - out[i] = isf[i - 1] + isf[i - 1 - i_max_corr] + isf[i] = isf[i - 1] + isf[i - 1 - i_max_corr] - isf[i - 2 - i_max_corr]; /* Calculate an estimate for ISF(18) and scale ISF based on the error */ - est = 7965 + (out[2] - out[3] - out[4]) / 6.0; - scale = 0.5 * (FFMIN(est, 7600) - out[LP_ORDER - 2]) / - (out[LP_ORDER_16k - 2] - out[LP_ORDER - 2]); + est = 7965 + (isf[2] - isf[3] - isf[4]) / 6.0; + scale = 0.5 * (FFMIN(est, 7600) - isf[LP_ORDER - 2]) / + (isf[LP_ORDER_16k - 2] - isf[LP_ORDER - 2]); for (i = LP_ORDER - 1; i < LP_ORDER_16k - 1; i++) - diff_hi[i] = scale * (out[i] - out[i - 1]); + diff_hi[i] = scale * (isf[i] - isf[i - 1]); /* Stability insurance */ for (i = LP_ORDER; i < LP_ORDER_16k - 1; i++) @@ -952,11 +951,11 @@ static void extrapolate_isf(float out[LP_ORDER_16k], float isf[LP_ORDER]) } for (i = LP_ORDER - 1; i < LP_ORDER_16k - 1; i++) - out[i] = out[i - 1] + diff_hi[i] * (1.0f / (1 << 15)); + isf[i] = isf[i - 1] + diff_hi[i] * (1.0f / (1 << 15)); /* Scale the ISF vector for 16000 Hz */ for (i = 0; i < LP_ORDER_16k - 1; i++) - out[i] *= 0.8; + isf[i] *= 0.8; } /** @@ -1003,7 +1002,7 @@ static void hb_synthesis(AMRWBContext *ctx, int subframe, float *samples, ff_weighted_vector_sumf(e_isf, isf_past, isf, isfp_inter[subframe], 1.0 - isfp_inter[subframe], LP_ORDER); - extrapolate_isf(e_isf, e_isf); + extrapolate_isf(e_isf); e_isf[LP_ORDER_16k - 1] *= 2.0; ff_acelp_lsf2lspd(e_isp, e_isf, LP_ORDER_16k); From 3f7e90cf0c12d739c5b9cd548c1916f23d691185 Mon Sep 17 00:00:00 2001 From: Alex Converse Date: Fri, 2 Mar 2012 10:13:07 -0800 Subject: [PATCH 072/154] mpegts: Pad the packet buffer in handle_packet(). This allows it to be used with get_bits without the thread of overreads. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 1aa708988ac131cf7d5c8bd59aca256a7c974df9) Signed-off-by: Reinhard Tartler --- libavformat/mpegts.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libavformat/mpegts.c b/libavformat/mpegts.c index 15688a9747..85e09527e3 100644 --- a/libavformat/mpegts.c +++ b/libavformat/mpegts.c @@ -1772,7 +1772,7 @@ static int read_packet(AVFormatContext *s, uint8_t *buf, int raw_packet_size) static int handle_packets(MpegTSContext *ts, int nb_packets) { AVFormatContext *s = ts->stream; - uint8_t packet[TS_PACKET_SIZE]; + uint8_t packet[TS_PACKET_SIZE+FF_INPUT_BUFFER_PADDING_SIZE]; int packet_num, ret = 0; if (avio_tell(s->pb) != ts->last_pos) { @@ -1794,6 +1794,7 @@ static int handle_packets(MpegTSContext *ts, int nb_packets) ts->stop_parse = 0; packet_num = 0; + memset(packet + TS_PACKET_SIZE, 0, FF_INPUT_BUFFER_PADDING_SIZE); for(;;) { if (ts->stop_parse>0) break; From b7c8fff80351249d448b93608bfac832c1ee3b4b Mon Sep 17 00:00:00 2001 From: Alex Converse Date: Fri, 2 Mar 2012 10:12:11 -0800 Subject: [PATCH 073/154] mpegts: Do not call read_sl_header() when no bytes remain in the buffer. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 4df369692ea8aee7094ac0f233cef8d1bee139a3) Signed-off-by: Reinhard Tartler --- libavformat/mpegts.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavformat/mpegts.c b/libavformat/mpegts.c index 85e09527e3..cc36e656da 100644 --- a/libavformat/mpegts.c +++ b/libavformat/mpegts.c @@ -889,7 +889,7 @@ static int mpegts_push_data(MpegTSFilter *filter, /* we got the full header. We parse it and get the payload */ pes->state = MPEGTS_PAYLOAD; pes->data_index = 0; - if (pes->stream_type == 0x12) { + if (pes->stream_type == 0x12 && buf_size > 0) { int sl_header_bytes = read_sl_header(pes, &pes->sl, p, buf_size); pes->pes_header_size += sl_header_bytes; p += sl_header_bytes; From 2e341bc99af72f1ae7c9812985635cbfeeb50269 Mon Sep 17 00:00:00 2001 From: Justin Ruggles Date: Fri, 2 Mar 2012 16:33:33 -0500 Subject: [PATCH 074/154] wmaenc: require a large enough output buffer to prevent overwrites The maximum theoretical frame size is around 17000 bytes. Although in practice it will generally be much smaller, we require a larger buffer just to be safe. CC: libav-stable@libav.org (cherry picked from commit dfc4fdedf8cfc56a505579b1f2c1c5efbce4b97e) Signed-off-by: Reinhard Tartler --- libavcodec/wmaenc.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/libavcodec/wmaenc.c b/libavcodec/wmaenc.c index c762a723b9..a9053bbc92 100644 --- a/libavcodec/wmaenc.c +++ b/libavcodec/wmaenc.c @@ -355,6 +355,11 @@ static int encode_superframe(AVCodecContext *avctx, } } + if (buf_size < 2 * MAX_CODED_SUPERFRAME_SIZE) { + av_log(avctx, AV_LOG_ERROR, "output buffer size is too small\n"); + return AVERROR(EINVAL); + } + #if 1 total_gain= 128; for(i=64; i; i>>=1){ From 073891e8758d5b4ed9034b340fa24c687792e8f6 Mon Sep 17 00:00:00 2001 From: Justin Ruggles Date: Fri, 2 Mar 2012 16:10:00 -0500 Subject: [PATCH 075/154] wmaenc: limit block_align to MAX_CODED_SUPERFRAME_SIZE This is near the theoretical limit for wma frame size and is the most that our decoder can handle. Allowing higher bit rates will just end up padding each frame with empty bytes. Fixes invalid writes for avconv when using very high bit rates. CC:libav-stable@libav.org (cherry picked from commit c2b8dea1828f35c808adcf12615893d5c740bc0a) Signed-off-by: Reinhard Tartler --- libavcodec/wmaenc.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/libavcodec/wmaenc.c b/libavcodec/wmaenc.c index a9053bbc92..bc17f5b6ba 100644 --- a/libavcodec/wmaenc.c +++ b/libavcodec/wmaenc.c @@ -71,8 +71,12 @@ static int encode_init(AVCodecContext * avctx){ for(i = 0; i < s->nb_block_sizes; i++) ff_mdct_init(&s->mdct_ctx[i], s->frame_len_bits - i + 1, 0, 1.0); - avctx->block_align= - s->block_align= avctx->bit_rate*(int64_t)s->frame_len / (avctx->sample_rate*8); + s->block_align = avctx->bit_rate * (int64_t)s->frame_len / + (avctx->sample_rate * 8); + s->block_align = FFMIN(s->block_align, MAX_CODED_SUPERFRAME_SIZE); + avctx->block_align = s->block_align; + avctx->bit_rate = avctx->block_align * 8LL * avctx->sample_rate / + s->frame_len; //av_log(NULL, AV_LOG_ERROR, "%d %d %d %d\n", s->block_align, avctx->bit_rate, s->frame_len, avctx->sample_rate); avctx->frame_size= s->frame_len; From 6a073aa7a734d4fbad77071e9f8ee0fe75a17fae Mon Sep 17 00:00:00 2001 From: Justin Ruggles Date: Fri, 2 Mar 2012 16:27:57 -0500 Subject: [PATCH 076/154] wmaenc: limit allowed sample rate to 48kHz ff_wma_init() allows up to 50kHz, but this generates an exponent band size table that requires 65 bands. The code assumes 25 bands in many places, and using sample rates higher than 48kHz will lead to buffer overwrites. CC:libav-stable@libav.org (cherry picked from commit 1ec075cfecac01f9a289965db06f76365b0b1737) Signed-off-by: Reinhard Tartler --- libavcodec/wmaenc.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libavcodec/wmaenc.c b/libavcodec/wmaenc.c index bc17f5b6ba..5135b982aa 100644 --- a/libavcodec/wmaenc.c +++ b/libavcodec/wmaenc.c @@ -39,6 +39,12 @@ static int encode_init(AVCodecContext * avctx){ return AVERROR(EINVAL); } + if (avctx->sample_rate > 48000) { + av_log(avctx, AV_LOG_ERROR, "sample rate is too high: %d > 48kHz", + avctx->sample_rate); + return AVERROR(EINVAL); + } + if(avctx->bit_rate < 24*1000) { av_log(avctx, AV_LOG_ERROR, "bitrate too low: got %i, need 24000 or higher\n", avctx->bit_rate); From 1128b10247739900174991b4e013429a1b8ceaa4 Mon Sep 17 00:00:00 2001 From: Justin Ruggles Date: Fri, 2 Mar 2012 17:11:25 -0500 Subject: [PATCH 077/154] wmaenc: fix m/s stereo encoding for the first frame We need to set ms_stereo in encode_init() in order to avoid incorrectly encoding the first frame as non-m/s while flagging it as m/s. Fixes an uncomfortable pop in the left channel at the start of playback. CC:libav-stable@libav.org (cherry picked from commit 51ddf35c9017018e58c15275ff5b129647a0c94d) Signed-off-by: Reinhard Tartler --- libavcodec/wmaenc.c | 4 +++- tests/ref/acodec/wmav1 | 6 +++--- tests/ref/acodec/wmav2 | 6 +++--- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/libavcodec/wmaenc.c b/libavcodec/wmaenc.c index 5135b982aa..df59cab6aa 100644 --- a/libavcodec/wmaenc.c +++ b/libavcodec/wmaenc.c @@ -70,6 +70,8 @@ static int encode_init(AVCodecContext * avctx){ s->use_exp_vlc = flags2 & 0x0001; s->use_bit_reservoir = flags2 & 0x0002; s->use_variable_block_len = flags2 & 0x0004; + if (avctx->channels == 2) + s->ms_stereo = 1; ff_wma_init(avctx, flags2); @@ -191,7 +193,7 @@ static int encode_block(WMACodecContext *s, float (*src_coefs)[BLOCK_MAX_SIZE], } if (s->nb_channels == 2) { - put_bits(&s->pb, 1, s->ms_stereo= 1); + put_bits(&s->pb, 1, !!s->ms_stereo); } for(ch = 0; ch < s->nb_channels; ch++) { diff --git a/tests/ref/acodec/wmav1 b/tests/ref/acodec/wmav1 index 916e4a8ab6..117aa12a8c 100644 --- a/tests/ref/acodec/wmav1 +++ b/tests/ref/acodec/wmav1 @@ -1,4 +1,4 @@ -26a7f6b0f0b7181df8df3fa589f6bf81 *./tests/data/acodec/wmav1.asf +0260385b8a54df11ad349f9ba8240fd8 *./tests/data/acodec/wmav1.asf 106004 ./tests/data/acodec/wmav1.asf -stddev:12245.52 PSNR: 14.57 MAXDIFF:65521 bytes: 1064960/ 1058400 -stddev: 2095.89 PSNR: 29.90 MAXDIFF:27658 bytes: 1056768/ 1058400 +stddev:12241.90 PSNR: 14.57 MAXDIFF:65521 bytes: 1064960/ 1058400 +stddev: 2074.79 PSNR: 29.99 MAXDIFF:27658 bytes: 1056768/ 1058400 diff --git a/tests/ref/acodec/wmav2 b/tests/ref/acodec/wmav2 index 622b6fcc36..43b19b7530 100644 --- a/tests/ref/acodec/wmav2 +++ b/tests/ref/acodec/wmav2 @@ -1,4 +1,4 @@ -7c6c0cb692af01b312ae345723674b5f *./tests/data/acodec/wmav2.asf +bdb4c312fb109f990be83a70f8ec9bdc *./tests/data/acodec/wmav2.asf 106044 ./tests/data/acodec/wmav2.asf -stddev:12249.93 PSNR: 14.57 MAXDIFF:65521 bytes: 1064960/ 1058400 -stddev: 2089.21 PSNR: 29.93 MAXDIFF:27650 bytes: 1056768/ 1058400 +stddev:12246.35 PSNR: 14.57 MAXDIFF:65521 bytes: 1064960/ 1058400 +stddev: 2068.08 PSNR: 30.02 MAXDIFF:27650 bytes: 1056768/ 1058400 From cd17195d1c0e0f7385946506a5ad2510cf44471b Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Tue, 28 Feb 2012 18:48:27 -0800 Subject: [PATCH 078/154] h264: prevent overreads in intra PCM decoding. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit d1604b3de96575195b219028e2c4f08b2259aa7d) Signed-off-by: Reinhard Tartler --- libavcodec/h264_cabac.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libavcodec/h264_cabac.c b/libavcodec/h264_cabac.c index 75fb02cb63..2ee4bc01a8 100644 --- a/libavcodec/h264_cabac.c +++ b/libavcodec/h264_cabac.c @@ -1996,6 +1996,8 @@ decode_intra_mb: } // The pixels are stored in the same order as levels in h->mb array. + if ((int) (h->cabac.bytestream_end - ptr) < mb_size) + return -1; memcpy(h->mb, ptr, mb_size); ptr+=mb_size; ff_init_cabac_decoder(&h->cabac, ptr, h->cabac.bytestream_end - ptr); From 11f3173e1bae135eb18a10b0060a5dd4b9fdcc74 Mon Sep 17 00:00:00 2001 From: Vitor Sessak Date: Wed, 29 Feb 2012 22:09:10 +0100 Subject: [PATCH 079/154] amrnbdec: check frame size before decoding. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org Signed-off-by: Ronald S. Bultje (cherry picked from commit 882abda5a26ffb8e3d1c5852dfa7cdad0a291d2d) Signed-off-by: Reinhard Tartler --- libavcodec/amrnbdec.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/libavcodec/amrnbdec.c b/libavcodec/amrnbdec.c index fff0e7248a..a7d0b4e337 100644 --- a/libavcodec/amrnbdec.c +++ b/libavcodec/amrnbdec.c @@ -200,6 +200,10 @@ static enum Mode unpack_bitstream(AMRContext *p, const uint8_t *buf, p->bad_frame_indicator = !get_bits1(&gb); // quality bit skip_bits(&gb, 2); // two padding bits + if (mode >= N_MODES || buf_size < frame_sizes_nb[mode] + 1) { + return NO_DATA; + } + if (mode < MODE_DTX) ff_amr_bit_reorder((uint16_t *) &p->frame, sizeof(AMRNBFrame), buf + 1, amr_unpacking_bitmaps_per_mode[mode]); @@ -947,6 +951,10 @@ static int amrnb_decode_frame(AVCodecContext *avctx, void *data, buf_out = (float *)p->avframe.data[0]; p->cur_frame_mode = unpack_bitstream(p, buf, buf_size); + if (p->cur_frame_mode == NO_DATA) { + av_log(avctx, AV_LOG_ERROR, "Corrupt bitstream\n"); + return AVERROR_INVALIDDATA; + } if (p->cur_frame_mode == MODE_DTX) { av_log_missing_feature(avctx, "dtx mode", 1); return -1; From b5331b979bfb31ec1715618b2712429764b6a9b5 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Wed, 29 Feb 2012 13:55:09 -0800 Subject: [PATCH 080/154] cscd: use negative error values to indicate decode_init() failures. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 8a9faf33f2b4f40afbc3393b2be49867cea0c92d) Signed-off-by: Reinhard Tartler --- libavcodec/cscd.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libavcodec/cscd.c b/libavcodec/cscd.c index 00921bc47d..1edab91bcf 100644 --- a/libavcodec/cscd.c +++ b/libavcodec/cscd.c @@ -228,7 +228,7 @@ static av_cold int decode_init(AVCodecContext *avctx) { av_log(avctx, AV_LOG_ERROR, "CamStudio codec error: invalid depth %i bpp\n", avctx->bits_per_coded_sample); - return 1; + return AVERROR_INVALIDDATA; } c->bpp = avctx->bits_per_coded_sample; c->pic.data[0] = NULL; @@ -241,7 +241,7 @@ static av_cold int decode_init(AVCodecContext *avctx) { c->decomp_buf = av_malloc(c->decomp_size + AV_LZO_OUTPUT_PADDING); if (!c->decomp_buf) { av_log(avctx, AV_LOG_ERROR, "Can't allocate decompression buffer.\n"); - return 1; + return AVERROR(ENOMEM); } return 0; } From 5186984ee9cf65946ed8bcf4b480f81c4310a8ce Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Sat, 25 Feb 2012 17:24:56 -0800 Subject: [PATCH 081/154] h264: change underread for 10bit QPEL to overread. This prevents us from reading before the start of the buffer, and thus prevents crashes resulting from this behaviour. Fixes bug 237. (cherry picked from commit 291c9b62855d555ac5385e23219461b6080da7db) Signed-off-by: Reinhard Tartler --- libavcodec/x86/h264_qpel_10bit.asm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavcodec/x86/h264_qpel_10bit.asm b/libavcodec/x86/h264_qpel_10bit.asm index 15dd72ca36..cafd4dabf0 100644 --- a/libavcodec/x86/h264_qpel_10bit.asm +++ b/libavcodec/x86/h264_qpel_10bit.asm @@ -619,7 +619,7 @@ MC MC33 %define PAD 12 %define COUNT 2 %else -%define PAD 0 +%define PAD 4 %define COUNT 3 %endif put_hv%2_10_%1: From 85eb76a23fbba2d26e3742e8163d3994b2972b4b Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Sun, 26 Feb 2012 08:57:14 -0800 Subject: [PATCH 082/154] h264: fix mmxext chroma deblock to use correct TC values. (cherry picked from commit b0c4f04338234ee011d7b704621347ef232294fe) Signed-off-by: Reinhard Tartler --- libavcodec/x86/h264_deblock_10bit.asm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavcodec/x86/h264_deblock_10bit.asm b/libavcodec/x86/h264_deblock_10bit.asm index baac725eec..6744661d65 100644 --- a/libavcodec/x86/h264_deblock_10bit.asm +++ b/libavcodec/x86/h264_deblock_10bit.asm @@ -870,7 +870,7 @@ cglobal deblock_v_chroma_10_%1, 5,7-(mmsize/16),8*(mmsize/16) %if mmsize < 16 add r0, mmsize add r5, mmsize - add r4, mmsize/8 + add r4, mmsize/4 dec r6 jg .loop REP_RET From 003f7e3dd0debfaa28622bd81e77f9217043ee28 Mon Sep 17 00:00:00 2001 From: Fabian Greffrath Date: Mon, 5 Mar 2012 16:06:01 +0100 Subject: [PATCH 083/154] Fix format string vulnerability detected by -Wformat-security. Signed-off-by: Diego Biurrun (cherry picked from commit c9dbac36ad4bac07f6c1d06d465e361ab55bcb95) Signed-off-by: Reinhard Tartler --- libavcodec/srtdec.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libavcodec/srtdec.c b/libavcodec/srtdec.c index 677c5501f8..99cbd9428b 100644 --- a/libavcodec/srtdec.c +++ b/libavcodec/srtdec.c @@ -110,7 +110,7 @@ static const char *srt_to_ass(AVCodecContext *avctx, char *out, char *out_end, for (j=sptr-2; j>=0; j--) if (stack[j].param[i][0]) { out += snprintf(out, out_end-out, - stack[j].param[i]); + "%s", stack[j].param[i]); break; } } else { @@ -146,7 +146,7 @@ static const char *srt_to_ass(AVCodecContext *avctx, char *out, char *out_end, for (i=0; i Date: Thu, 9 Feb 2012 13:00:30 -0500 Subject: [PATCH 084/154] ac3dsp: do not use pshufb in ac3_extract_exponents_ssse3() We need to do unsigned saturation in order to cover the corner case when the absolute coefficient value is 16777215 (the maximum value). Fixes Bug #216 (cherry picked from commit d483bb58c318b0a6152709cf28263d72200b98f9) Signed-off-by: Reinhard Tartler --- libavcodec/x86/ac3dsp.asm | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/libavcodec/x86/ac3dsp.asm b/libavcodec/x86/ac3dsp.asm index c1b0906a85..9312ff6533 100644 --- a/libavcodec/x86/ac3dsp.asm +++ b/libavcodec/x86/ac3dsp.asm @@ -35,7 +35,6 @@ pw_bap_mul2: dw 5, 7, 0, 7, 5, 7, 0, 7 ; used in ff_ac3_extract_exponents() pd_1: times 4 dd 1 pd_151: times 4 dd 151 -pb_shuf_4dwb: db 0, 4, 8, 12 SECTION .text @@ -404,15 +403,12 @@ cglobal ac3_extract_exponents_3dnow, 3,3,0, exp, coef, len %endif %macro AC3_EXTRACT_EXPONENTS 1 -cglobal ac3_extract_exponents_%1, 3,3,5, exp, coef, len +cglobal ac3_extract_exponents_%1, 3,3,4, exp, coef, len add expq, lenq lea coefq, [coefq+4*lenq] neg lenq mova m2, [pd_1] mova m3, [pd_151] -%ifidn %1, ssse3 ; - movd m4, [pb_shuf_4dwb] -%endif .loop: ; move 4 32-bit coefs to xmm0 mova m0, [coefq+4*lenq] @@ -426,12 +422,11 @@ cglobal ac3_extract_exponents_%1, 3,3,5, exp, coef, len mova m0, m3 psubd m0, m1 ; move the lowest byte in each of 4 dwords to the low dword -%ifidn %1, ssse3 - pshufb m0, m4 -%else + ; NOTE: We cannot just extract the low bytes with pshufb because the dword + ; result for 16777215 is -1 due to float inaccuracy. Using packuswb + ; clips this to 0, which is the correct exponent. packssdw m0, m0 packuswb m0, m0 -%endif movd [expq+lenq], m0 add lenq, 4 From ca7e97bdcf0d19c69293de08f5956d1431ee461f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Storsj=C3=B6?= Date: Fri, 2 Mar 2012 17:03:06 +0200 Subject: [PATCH 085/154] g722: Fix the QMF scaling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This fixes clipping if the encoder input used the full 16 bit input range (samples with a magnitude below 16383 worked fine). The filtered subband samples should be 15 bit maximum, while the code earlier produced them scaled to 16 bit. This makes the decoder output have double the magnitude compared to before. The spec reference samples doesn't test the QMF at all, which was why this part slipped past initially. (cherry picked from commit b087ce2bee81db8cc5caffb8f0a4f6c7c92a30fe) Signed-off-by: Martin Storsjö --- libavcodec/g722dec.c | 4 +- libavcodec/g722enc.c | 4 +- tests/ref/acodec/g722 | 8 +- tests/ref/fate/g722dec-1 | 334 +++++++++++++++++++-------------------- tests/ref/fate/g722enc | 2 +- 5 files changed, 176 insertions(+), 176 deletions(-) diff --git a/libavcodec/g722dec.c b/libavcodec/g722dec.c index 50a224ba10..72bb0ef3c7 100644 --- a/libavcodec/g722dec.c +++ b/libavcodec/g722dec.c @@ -126,8 +126,8 @@ static int g722_decode_frame(AVCodecContext *avctx, void *data, c->prev_samples[c->prev_samples_pos++] = rlow - rhigh; ff_g722_apply_qmf(c->prev_samples + c->prev_samples_pos - 24, &xout1, &xout2); - *out_buf++ = av_clip_int16(xout1 >> 12); - *out_buf++ = av_clip_int16(xout2 >> 12); + *out_buf++ = av_clip_int16(xout1 >> 11); + *out_buf++ = av_clip_int16(xout2 >> 11); if (c->prev_samples_pos >= PREV_SAMPLES_BUF_SIZE) { memmove(c->prev_samples, c->prev_samples + c->prev_samples_pos - 22, 22 * sizeof(c->prev_samples[0])); diff --git a/libavcodec/g722enc.c b/libavcodec/g722enc.c index 1cb0070649..b5707e3cd3 100644 --- a/libavcodec/g722enc.c +++ b/libavcodec/g722enc.c @@ -128,8 +128,8 @@ static inline void filter_samples(G722Context *c, const int16_t *samples, c->prev_samples[c->prev_samples_pos++] = samples[0]; c->prev_samples[c->prev_samples_pos++] = samples[1]; ff_g722_apply_qmf(c->prev_samples + c->prev_samples_pos - 24, &xout1, &xout2); - *xlow = xout1 + xout2 >> 13; - *xhigh = xout1 - xout2 >> 13; + *xlow = xout1 + xout2 >> 14; + *xhigh = xout1 - xout2 >> 14; if (c->prev_samples_pos >= PREV_SAMPLES_BUF_SIZE) { memmove(c->prev_samples, c->prev_samples + c->prev_samples_pos - 22, diff --git a/tests/ref/acodec/g722 b/tests/ref/acodec/g722 index 6ea492ae45..0e2f7e75b0 100644 --- a/tests/ref/acodec/g722 +++ b/tests/ref/acodec/g722 @@ -1,4 +1,4 @@ -1975cc4a3521e374b33ae042e182f6b6 *./tests/data/acodec/g722.wav -48053 ./tests/data/acodec/g722.wav -ade04cdcf249e6946395f109b077dd62 *./tests/data/g722.acodec.out.wav -stddev: 8841.24 PSNR: 17.40 MAXDIFF:36225 bytes: 191980/ 1058400 +7b0492eee76b04b710990235f97a0bf2 *./tests/data/acodec/g722.wav + 48053 ./tests/data/acodec/g722.wav +b5568e0e3930ff563824156e8e1015f0 *./tests/data/g722.acodec.out.wav +stddev: 8939.44 PSNR: 17.30 MAXDIFF:40370 bytes: 191980/ 1058400 diff --git a/tests/ref/fate/g722dec-1 b/tests/ref/fate/g722dec-1 index 4c4b2b53e7..cdc54891c8 100644 --- a/tests/ref/fate/g722dec-1 +++ b/tests/ref/fate/g722dec-1 @@ -1,167 +1,167 @@ -0, 0, 4096, 0xde68394d -0, 11520, 4096, 0xa5c28cb7 -0, 23040, 4096, 0x2e3c2f23 -0, 34560, 4096, 0xd7757825 -0, 46080, 4096, 0xafd1fd61 -0, 57600, 4096, 0x686afcbe -0, 69120, 4096, 0x2290e848 -0, 80640, 4096, 0xddd484ad -0, 92160, 4096, 0x148811a6 -0, 103680, 4096, 0x8b965613 -0, 115200, 4096, 0x8b095d51 -0, 126720, 4096, 0xf7625485 -0, 138240, 4096, 0x982a688c -0, 149760, 4096, 0xc290dcfc -0, 161280, 4096, 0x8bdef225 -0, 172800, 4096, 0xfca27fdc -0, 184320, 4096, 0x95eff313 -0, 195840, 4096, 0x691ed4f7 -0, 207360, 4096, 0xd7e7b492 -0, 218880, 4096, 0xb0416bfe -0, 230400, 4096, 0xf94b3ebd -0, 241920, 4096, 0x7f73ca12 -0, 253440, 4096, 0xe91da4a3 -0, 264960, 4096, 0x1f74dc0e -0, 276480, 4096, 0xd95b35e8 -0, 288000, 4096, 0x6dcdde1a -0, 299520, 4096, 0x614fd4e4 -0, 311040, 4096, 0xe38d0fd5 -0, 322560, 4096, 0xfeba2999 -0, 334080, 4096, 0x1bf541e1 -0, 345600, 4096, 0x689f50d8 -0, 357120, 4096, 0x0aa60f5f -0, 368640, 4096, 0x60ac3116 -0, 380160, 4096, 0xfa60e5e6 -0, 391680, 4096, 0xc7207c5b -0, 403200, 4096, 0x01196277 -0, 414720, 4096, 0x609ca46c -0, 426240, 4096, 0xfb799142 -0, 437760, 4096, 0x720910df -0, 449280, 4096, 0xe21a8662 -0, 460800, 4096, 0x07105120 -0, 472320, 4096, 0x593f627e -0, 483840, 4096, 0x28ddc80c -0, 495360, 4096, 0xc69ef356 -0, 506880, 4096, 0x2defc5bd -0, 518400, 4096, 0x82a4f418 -0, 529920, 4096, 0x424cb997 -0, 541440, 4096, 0x167a49b7 -0, 552960, 4096, 0x32a3e0d4 -0, 564480, 4096, 0x08a353ae -0, 576000, 4096, 0x9543577b -0, 587520, 4096, 0x2ed137cf -0, 599040, 4096, 0xd80b0538 -0, 610560, 4096, 0x2ad31bef -0, 622080, 4096, 0x1060cff8 -0, 633600, 4096, 0x76ab5ab8 -0, 645120, 4096, 0x8eedb68d -0, 656640, 4096, 0xf4e2dc46 -0, 668160, 4096, 0xc52d3326 -0, 679680, 4096, 0x25201a26 -0, 691200, 4096, 0x16419378 -0, 702720, 4096, 0x97061f3c -0, 714240, 4096, 0xd54edecd -0, 725760, 4096, 0xc830b07b -0, 737280, 4096, 0x804bae00 -0, 748800, 4096, 0xbb279150 -0, 760320, 4096, 0x95c4d5aa -0, 771840, 4096, 0xc51d5259 -0, 783360, 4096, 0x856e1ab0 -0, 794880, 4096, 0x9e6ccb12 -0, 806400, 4096, 0xa2e5c1bb -0, 817920, 4096, 0xe62fb62f -0, 829440, 4096, 0xf10e3df0 -0, 840960, 4096, 0x76def18b -0, 852480, 4096, 0xc9c3a26d -0, 864000, 4096, 0x8ec0e061 -0, 875520, 4096, 0x3d4e8512 -0, 887040, 4096, 0xec45cd46 -0, 898560, 4096, 0xa34f3ddf -0, 910080, 4096, 0x52b81c53 -0, 921600, 4096, 0xd0f0397a -0, 933120, 4096, 0x7c0de231 -0, 944640, 4096, 0xfe86c032 -0, 956160, 4096, 0x67cdb848 -0, 967680, 4096, 0x90532cc0 -0, 979200, 4096, 0x03bca9e9 -0, 990720, 4096, 0x73169fd1 -0, 1002240, 4096, 0x0b93967d -0, 1013760, 4096, 0x6486d8be -0, 1025280, 4096, 0x555cc2ac -0, 1036800, 4096, 0x07c1912e -0, 1048320, 4096, 0xe0423c66 -0, 1059840, 4096, 0xc12d0fa1 -0, 1071360, 4096, 0xdf497c2f -0, 1082880, 4096, 0x9298d1ba -0, 1094400, 4096, 0x691a4e15 -0, 1105920, 4096, 0x725adc6e -0, 1117440, 4096, 0xf68e88de -0, 1128960, 4096, 0x37a234aa -0, 1140480, 4096, 0x43fb0558 -0, 1152000, 4096, 0x653e4320 -0, 1163520, 4096, 0x651e2f13 -0, 1175040, 4096, 0x179049f9 -0, 1186560, 4096, 0xe02fbb9d -0, 1198080, 4096, 0xb7e9f2a0 -0, 1209600, 4096, 0x94ee81df -0, 1221120, 4096, 0x398a98de -0, 1232640, 4096, 0x1267594a -0, 1244160, 4096, 0x715adbaf -0, 1255680, 4096, 0x28ce1a20 -0, 1267200, 4096, 0x4f8073d0 -0, 1278720, 4096, 0x536846d3 -0, 1290240, 4096, 0x7dc7defe -0, 1301760, 4096, 0x08a28e2a -0, 1313280, 4096, 0xd717c5cd -0, 1324800, 4096, 0x5d6e1efd -0, 1336320, 4096, 0x4d0eea27 -0, 1347840, 4096, 0x70fff90c -0, 1359360, 4096, 0xd5cc8207 -0, 1370880, 4096, 0xf87cae0e -0, 1382400, 4096, 0x26814ab5 -0, 1393920, 4096, 0x9569fb8d -0, 1405440, 4096, 0x7835122e -0, 1416960, 4096, 0xa38840dd -0, 1428480, 4096, 0xfc499ba3 -0, 1440000, 4096, 0x0aa60cb0 -0, 1451520, 4096, 0x530ef56e -0, 1463040, 4096, 0xead968db -0, 1474560, 4096, 0x64484214 -0, 1486080, 4096, 0xfd0cc89e -0, 1497600, 4096, 0x0d452a5d -0, 1509120, 4096, 0x36ef8482 -0, 1520640, 4096, 0x462b641b -0, 1532160, 4096, 0x2a5c1c0c -0, 1543680, 4096, 0x8837ff80 -0, 1555200, 4096, 0x27a3de22 -0, 1566720, 4096, 0xf88d28c1 -0, 1578240, 4096, 0xed85ea97 -0, 1589760, 4096, 0x50c3e7db -0, 1601280, 4096, 0x82bcb480 -0, 1612800, 4096, 0xc50ee536 -0, 1624320, 4096, 0x086280ee -0, 1635840, 4096, 0x6f18f2b2 -0, 1647360, 4096, 0x1c7c0856 -0, 1658880, 4096, 0xc576268a -0, 1670400, 4096, 0x7a9af56d -0, 1681920, 4096, 0x6d058fc5 -0, 1693440, 4096, 0x8fb1107b -0, 1704960, 4096, 0x807588d1 -0, 1716480, 4096, 0x56178443 -0, 1728000, 4096, 0xf2460763 -0, 1739520, 4096, 0x284255f2 -0, 1751040, 4096, 0xb29d17fb -0, 1762560, 4096, 0x5e7e4633 -0, 1774080, 4096, 0x57704db1 -0, 1785600, 4096, 0xd87dcc1d -0, 1797120, 4096, 0x28d4bb93 -0, 1808640, 4096, 0x3a2e5c6c -0, 1820160, 4096, 0xf3581656 -0, 1831680, 4096, 0x42f1942f -0, 1843200, 4096, 0xe75c5092 -0, 1854720, 4096, 0x3fae7f6d -0, 1866240, 4096, 0xf99ad73e -0, 1877760, 4096, 0x80564e3e -0, 1889280, 4096, 0x8ff6ebe5 -0, 1900800, 4096, 0x436d5e69 -0, 1912320, 1368, 0xe0ebeda3 +0, 0, 4096, 0x4f9228b3 +0, 11520, 4096, 0xfab58157 +0, 23040, 4096, 0x0b641c78 +0, 34560, 4096, 0x601c6803 +0, 46080, 4096, 0xb3e2f166 +0, 57600, 4096, 0x5681f206 +0, 69120, 4096, 0x1e69e71f +0, 80640, 4096, 0x05628be3 +0, 92160, 4096, 0x109b1aef +0, 103680, 4096, 0xd5435a9e +0, 115200, 4096, 0xb38b5d28 +0, 126720, 4096, 0x64514c93 +0, 138240, 4096, 0x453350e7 +0, 149760, 4096, 0x6deccce6 +0, 161280, 4096, 0xd427ede1 +0, 172800, 4096, 0xdecb8c42 +0, 184320, 4096, 0x3841e4d2 +0, 195840, 4096, 0x858ac1b1 +0, 207360, 4096, 0x8e9dbfa0 +0, 218880, 4096, 0xcbc0766f +0, 230400, 4096, 0x78d52555 +0, 241920, 4096, 0x600ac7d5 +0, 253440, 4096, 0xafadb7ee +0, 264960, 4096, 0x8009d5a1 +0, 276480, 4096, 0xb07d475e +0, 288000, 4096, 0xfcfecceb +0, 299520, 4096, 0x38b5d85f +0, 311040, 4096, 0xbd48072e +0, 322560, 4096, 0xd04724d8 +0, 334080, 4096, 0x08425144 +0, 345600, 4096, 0x7b14483e +0, 357120, 4096, 0x8858ef4c +0, 368640, 4096, 0x1e3024c2 +0, 380160, 4096, 0xcd6bfe4f +0, 391680, 4096, 0x8cde8d18 +0, 403200, 4096, 0xbbd856b8 +0, 414720, 4096, 0x988c9b7a +0, 426240, 4096, 0x2a858e03 +0, 437760, 4096, 0x6dee1e4a +0, 449280, 4096, 0x8cc38b41 +0, 460800, 4096, 0x48bd5cec +0, 472320, 4096, 0xeb7f606b +0, 483840, 4096, 0x75f5d28c +0, 495360, 4096, 0x5bfeec4b +0, 506880, 4096, 0xfc35c22a +0, 518400, 4096, 0x3a95efba +0, 529920, 4096, 0xefdbce9c +0, 541440, 4096, 0x00594ada +0, 552960, 4096, 0x20ffebfa +0, 564480, 4096, 0x1b31370a +0, 576000, 4096, 0x50766a56 +0, 587520, 4096, 0x0058315a +0, 599040, 4096, 0x98090cbf +0, 610560, 4096, 0x66ed2d40 +0, 622080, 4096, 0xdfd7c0a7 +0, 633600, 4096, 0x2adc57e1 +0, 645120, 4096, 0x838bbc82 +0, 656640, 4096, 0x2c55de1a +0, 668160, 4096, 0xeae027f4 +0, 679680, 4096, 0x09fe00f6 +0, 691200, 4096, 0xa25d9970 +0, 702720, 4096, 0xedb11a20 +0, 714240, 4096, 0x9ce2e63e +0, 725760, 4096, 0xeb699974 +0, 737280, 4096, 0xcc04a296 +0, 748800, 4096, 0xe90e9a12 +0, 760320, 4096, 0xae85c0f7 +0, 771840, 4096, 0x7ee877db +0, 783360, 4096, 0x9ecf14ee +0, 794880, 4096, 0xa821cecd +0, 806400, 4096, 0x2714bb11 +0, 817920, 4096, 0x28f1c1e0 +0, 829440, 4096, 0xf81c4f60 +0, 840960, 4096, 0x1ae0e5a1 +0, 852480, 4096, 0xbdae9d9a +0, 864000, 4096, 0x5202e560 +0, 875520, 4096, 0x82408396 +0, 887040, 4096, 0xc850ce0c +0, 898560, 4096, 0x1d732d88 +0, 910080, 4096, 0xc5c01e33 +0, 921600, 4096, 0x84942d6c +0, 933120, 4096, 0x7c27cd3a +0, 944640, 4096, 0x22adc503 +0, 956160, 4096, 0xfbc3af31 +0, 967680, 4096, 0xe9652b18 +0, 979200, 4096, 0xae75987e +0, 990720, 4096, 0x0f7ea428 +0, 1002240, 4096, 0x92b89582 +0, 1013760, 4096, 0xf393d910 +0, 1025280, 4096, 0x6349b600 +0, 1036800, 4096, 0x16918dbd +0, 1048320, 4096, 0x14ee15ad +0, 1059840, 4096, 0x26b510d3 +0, 1071360, 4096, 0x97007bf8 +0, 1082880, 4096, 0x3718c509 +0, 1094400, 4096, 0x24a54ccd +0, 1105920, 4096, 0xc960df4e +0, 1117440, 4096, 0xc7cb6e6f +0, 1128960, 4096, 0x4c563ae5 +0, 1140480, 4096, 0x0dd51432 +0, 1152000, 4096, 0xdb4243c8 +0, 1163520, 4096, 0x9bb6417f +0, 1175040, 4096, 0xec6a40a1 +0, 1186560, 4096, 0x82d6c3b4 +0, 1198080, 4096, 0xd181e2ec +0, 1209600, 4096, 0xba5d7b55 +0, 1221120, 4096, 0x78fcb938 +0, 1232640, 4096, 0x6691671c +0, 1244160, 4096, 0x44fadee7 +0, 1255680, 4096, 0xa42720d5 +0, 1267200, 4096, 0xc1165a91 +0, 1278720, 4096, 0x86aa3e3f +0, 1290240, 4096, 0xab5ae57d +0, 1301760, 4096, 0x291a91f3 +0, 1313280, 4096, 0xfdf0dcfc +0, 1324800, 4096, 0x1ef91f67 +0, 1336320, 4096, 0xc899efee +0, 1347840, 4096, 0x5ade15ac +0, 1359360, 4096, 0x04516beb +0, 1370880, 4096, 0xbf5ebbb9 +0, 1382400, 4096, 0x4a235122 +0, 1393920, 4096, 0xd7a3f4a6 +0, 1405440, 4096, 0x5f900f20 +0, 1416960, 4096, 0xa90b4365 +0, 1428480, 4096, 0x63149dc4 +0, 1440000, 4096, 0xf12c1ee8 +0, 1451520, 4096, 0x6d0fec8c +0, 1463040, 4096, 0x65e07850 +0, 1474560, 4096, 0x16d951cc +0, 1486080, 4096, 0xd296d0c4 +0, 1497600, 4096, 0x619b2a53 +0, 1509120, 4096, 0x316972d5 +0, 1520640, 4096, 0xcfd64e21 +0, 1532160, 4096, 0xcbcb10c6 +0, 1543680, 4096, 0x20aeff7c +0, 1555200, 4096, 0xd205dabd +0, 1566720, 4096, 0xac9d3001 +0, 1578240, 4096, 0x6d53dfdd +0, 1589760, 4096, 0xbb9fe15c +0, 1601280, 4096, 0x1852b88b +0, 1612800, 4096, 0xb0acec01 +0, 1624320, 4096, 0xb52a9342 +0, 1635840, 4096, 0x7529faee +0, 1647360, 4096, 0x150ff449 +0, 1658880, 4096, 0xa81d31d9 +0, 1670400, 4096, 0xbcb8084a +0, 1681920, 4096, 0x07229514 +0, 1693440, 4096, 0xa85cfd88 +0, 1704960, 4096, 0x0aef9c27 +0, 1716480, 4096, 0x8ec47b39 +0, 1728000, 4096, 0x910b0560 +0, 1739520, 4096, 0x99a8578e +0, 1751040, 4096, 0xb3df1d84 +0, 1762560, 4096, 0x48e52559 +0, 1774080, 4096, 0xb25c4800 +0, 1785600, 4096, 0x913bc8ce +0, 1797120, 4096, 0xb736cc8c +0, 1808640, 4096, 0x13c66646 +0, 1820160, 4096, 0x70a71221 +0, 1831680, 4096, 0x3a50a08e +0, 1843200, 4096, 0xc0a037b0 +0, 1854720, 4096, 0x9a789475 +0, 1866240, 4096, 0xc890ca16 +0, 1877760, 4096, 0xa0d34bed +0, 1889280, 4096, 0x1689fa60 +0, 1900800, 4096, 0x5bac4c83 +0, 1912320, 1368, 0x904be5e5 diff --git a/tests/ref/fate/g722enc b/tests/ref/fate/g722enc index c1094565b5..9b8e469a8b 100644 --- a/tests/ref/fate/g722enc +++ b/tests/ref/fate/g722enc @@ -1 +1 @@ -750269cc236541df28e15da5c7b0df7a +94e2f200d6e05b47cec4aa3e94571cf3 From ffdc41f0395f74cb8844361d2154784ce65e8fdd Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Tue, 24 Jan 2012 22:20:26 +0100 Subject: [PATCH 086/154] nsvdec: Fix use of uninitialized streams. Fixes CVE-2011-3940 (Out of bounds read resulting in out of bounds write) Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind Signed-off-by: Michael Niedermayer (cherry picked from commit 5c011706bc752d34bc6ada31d7df2ca0c9af7c6b) Signed-off-by: Alex Converse (cherry picked from commit 6a89b41d9780325ba6d89a37f2aeb925aa68e6a3) --- libavformat/nsvdec.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libavformat/nsvdec.c b/libavformat/nsvdec.c index 18dfde2867..261cfbb1af 100644 --- a/libavformat/nsvdec.c +++ b/libavformat/nsvdec.c @@ -605,12 +605,12 @@ null_chunk_retry: } /* map back streams to v,a */ - if (s->streams[0]) + if (s->nb_streams > 0) st[s->streams[0]->id] = s->streams[0]; - if (s->streams[1]) + if (s->nb_streams > 1) st[s->streams[1]->id] = s->streams[1]; - if (vsize/* && st[NSV_ST_VIDEO]*/) { + if (vsize && st[NSV_ST_VIDEO]) { nst = st[NSV_ST_VIDEO]->priv_data; pkt = &nsv->ahead[NSV_ST_VIDEO]; av_get_packet(pb, pkt, vsize); @@ -623,7 +623,7 @@ null_chunk_retry: if(st[NSV_ST_VIDEO]) ((NSVStream*)st[NSV_ST_VIDEO]->priv_data)->frame_offset++; - if (asize/*st[NSV_ST_AUDIO]*/) { + if (asize && st[NSV_ST_AUDIO]) { nst = st[NSV_ST_AUDIO]->priv_data; pkt = &nsv->ahead[NSV_ST_AUDIO]; /* read raw audio specific header on the first audio chunk... */ From e410dd17920342b7f08f16675044f077c88c251b Mon Sep 17 00:00:00 2001 From: Alex Converse Date: Thu, 26 Jan 2012 17:21:46 -0800 Subject: [PATCH 087/154] nsvdec: Be more careful with av_malloc(). Check results for av_malloc() and fix an overflow in one call. Related to CVE-2011-3940. Based in part on work from Michael Niedermayer. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind (cherry picked from commit 8fd8a48263ff1437f9d02d7e78dc63efb9b5ed3a) --- libavformat/nsvdec.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/libavformat/nsvdec.c b/libavformat/nsvdec.c index 261cfbb1af..22a1a0938c 100644 --- a/libavformat/nsvdec.c +++ b/libavformat/nsvdec.c @@ -314,7 +314,9 @@ static int nsv_parse_NSVf_header(AVFormatContext *s, AVFormatParameters *ap) char *token, *value; char quote; - p = strings = av_mallocz(strings_size + 1); + p = strings = av_mallocz((size_t)strings_size + 1); + if (!p) + return AVERROR(ENOMEM); endp = strings + strings_size; avio_read(pb, strings, strings_size); while (p < endp) { @@ -349,6 +351,8 @@ static int nsv_parse_NSVf_header(AVFormatContext *s, AVFormatParameters *ap) if((unsigned)table_entries_used >= UINT_MAX / sizeof(uint32_t)) return -1; nsv->nsvs_file_offset = av_malloc((unsigned)table_entries_used * sizeof(uint32_t)); + if (!nsv->nsvs_file_offset) + return AVERROR(ENOMEM); for(i=0;insvs_file_offset[i] = avio_rl32(pb) + size; @@ -356,6 +360,8 @@ static int nsv_parse_NSVf_header(AVFormatContext *s, AVFormatParameters *ap) if(table_entries > table_entries_used && avio_rl32(pb) == MKTAG('T','O','C','2')) { nsv->nsvs_timestamps = av_malloc((unsigned)table_entries_used*sizeof(uint32_t)); + if (!nsv->nsvs_timestamps) + return AVERROR(ENOMEM); for(i=0;insvs_timestamps[i] = avio_rl32(pb); } From dd37038ac7526221a9497b4d07dd808381fc08e4 Mon Sep 17 00:00:00 2001 From: Alex Converse Date: Thu, 26 Jan 2012 17:23:09 -0800 Subject: [PATCH 088/154] nsvdec: Propagate errors Related to CVE-2011-3940. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind (cherry picked from commit c898431ca5ef2a997fe9388b650f658fb60783e5) Conflicts: libavformat/nsvdec.c --- libavformat/nsvdec.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/libavformat/nsvdec.c b/libavformat/nsvdec.c index 22a1a0938c..e5aaf33b97 100644 --- a/libavformat/nsvdec.c +++ b/libavformat/nsvdec.c @@ -532,11 +532,16 @@ static int nsv_read_header(AVFormatContext *s, AVFormatParameters *ap) for (i = 0; i < NSV_MAX_RESYNC_TRIES; i++) { if (nsv_resync(s) < 0) return -1; - if (nsv->state == NSV_FOUND_NSVF) + if (nsv->state == NSV_FOUND_NSVF) { err = nsv_parse_NSVf_header(s, ap); + if (err < 0) + return err; + } /* we need the first NSVs also... */ if (nsv->state == NSV_FOUND_NSVS) { err = nsv_parse_NSVs_header(s, ap); + if (err < 0) + return err; break; /* we just want the first one */ } } From 416849f2e06227b1b4a451c392f100db1d709a0c Mon Sep 17 00:00:00 2001 From: Alex Converse Date: Thu, 26 Jan 2012 17:30:49 +0100 Subject: [PATCH 089/154] kmvc: Check palsize. Fixes: CVE-2011-3952 Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind Based on fix by Michael Niedermayer (cherry picked from commit 386741f887714d3e46c9e8fe577e326a7964037b) --- libavcodec/kmvc.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/libavcodec/kmvc.c b/libavcodec/kmvc.c index 2b54b84e99..a6bb13b95a 100644 --- a/libavcodec/kmvc.c +++ b/libavcodec/kmvc.c @@ -33,6 +33,7 @@ #define KMVC_KEYFRAME 0x80 #define KMVC_PALETTE 0x40 #define KMVC_METHOD 0x0F +#define MAX_PALSIZE 256 /* * Decoder context @@ -43,7 +44,7 @@ typedef struct KmvcContext { int setpal; int palsize; - uint32_t pal[256]; + uint32_t pal[MAX_PALSIZE]; uint8_t *cur, *prev; uint8_t *frm0, *frm1; GetByteContext g; @@ -380,6 +381,10 @@ static av_cold int decode_init(AVCodecContext * avctx) c->palsize = 127; } else { c->palsize = AV_RL16(avctx->extradata + 10); + if (c->palsize >= MAX_PALSIZE) { + av_log(avctx, AV_LOG_ERROR, "KMVC palette too large\n"); + return AVERROR_INVALIDDATA; + } } if (avctx->extradata_size == 1036) { // palette in extradata From d5f2382d0389ed47a566ea536887af908bf9b14f Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Wed, 25 Jan 2012 23:23:35 +0100 Subject: [PATCH 090/154] kgv1dec: Increase offsets array size so it is large enough. Fixes CVE-2011-3945 Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind Signed-off-by: Michael Niedermayer (cherry picked from commit 807a045ab7f51993a2c1b3116016cbbd4f3d20d6) Signed-off-by: Alex Converse (cherry picked from commit a02e8df973f5478ec82f4c507f5b5b191a5ecb6b) --- libavcodec/kgv1dec.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libavcodec/kgv1dec.c b/libavcodec/kgv1dec.c index c4c3dac016..42bbcae530 100644 --- a/libavcodec/kgv1dec.c +++ b/libavcodec/kgv1dec.c @@ -46,7 +46,7 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, AVPac const uint8_t *buf = avpkt->data; const uint8_t *buf_end = buf + avpkt->size; KgvContext * const c = avctx->priv_data; - int offsets[7]; + int offsets[8]; uint16_t *out, *prev; int outcnt = 0, maxcnt; int w, h, i, res; @@ -79,7 +79,7 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *data_size, AVPac prev = NULL; } - for (i = 0; i < 7; i++) + for (i = 0; i < 8; i++) offsets[i] = -1; while (outcnt < maxcnt && buf_end - 2 > buf) { From 1ca84aa162a811def05bcd31394b1cea7ee19093 Mon Sep 17 00:00:00 2001 From: Alex Converse Date: Fri, 27 Jan 2012 15:50:24 -0800 Subject: [PATCH 091/154] mpeg12: Pad framerate tab to 16 entries. There are many places where we read an unchecked 4-bit index into it. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind (cherry picked from commit dfa37fe8a3d9243dd339d94befa065e2c90b29e6) --- libavcodec/mpeg12data.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavcodec/mpeg12data.c b/libavcodec/mpeg12data.c index 5ac8c243a5..a0dd6e5784 100644 --- a/libavcodec/mpeg12data.c +++ b/libavcodec/mpeg12data.c @@ -305,7 +305,7 @@ const uint8_t ff_mpeg12_mbMotionVectorTable[17][2] = { { 0xc, 10 }, }; -const AVRational avpriv_frame_rate_tab[] = { +const AVRational avpriv_frame_rate_tab[16] = { { 0, 0}, {24000, 1001}, { 24, 1}, From d0e53ecff736fd23c985c184051a7ae44529e448 Mon Sep 17 00:00:00 2001 From: Alex Converse Date: Wed, 25 Jan 2012 15:46:14 -0800 Subject: [PATCH 092/154] mp3dec: Fix a heap-buffer-overflow In some cases, what is left to read from ptr is smaller than EXTRABYTES. Based on a patch by Thierry Foucu . Signed-off-by: Alex Converse (cherry picked from commit f372ce119bd2458fa0b4ddfb2af3a36621df99f7) --- libavcodec/mpegaudiodec.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libavcodec/mpegaudiodec.c b/libavcodec/mpegaudiodec.c index 860c0c3d73..04c1d9842c 100644 --- a/libavcodec/mpegaudiodec.c +++ b/libavcodec/mpegaudiodec.c @@ -1378,16 +1378,17 @@ static int mp_decode_layer3(MPADecodeContext *s) if (!s->adu_mode) { int skip; const uint8_t *ptr = s->gb.buffer + (get_bits_count(&s->gb)>>3); + int extrasize = av_clip(get_bits_left(&s->gb) >> 3, 0, EXTRABYTES); assert((get_bits_count(&s->gb) & 7) == 0); /* now we get bits from the main_data_begin offset */ av_dlog(s->avctx, "seekback: %d\n", main_data_begin); //av_log(NULL, AV_LOG_ERROR, "backstep:%d, lastbuf:%d\n", main_data_begin, s->last_buf_size); - memcpy(s->last_buf + s->last_buf_size, ptr, EXTRABYTES); + memcpy(s->last_buf + s->last_buf_size, ptr, extrasize); s->in_gb = s->gb; init_get_bits(&s->gb, s->last_buf, s->last_buf_size*8); #if !UNCHECKED_BITSTREAM_READER - s->gb.size_in_bits_plus8 += EXTRABYTES * 8; + s->gb.size_in_bits_plus8 += extrasize * 8; #endif s->last_buf_size <<= 3; for (gr = 0; gr < nb_granules && (s->last_buf_size >> 3) < main_data_begin; gr++) { From feed0c6b6ae31cb3d5af144c74dd2040051780b7 Mon Sep 17 00:00:00 2001 From: Dale Curtis Date: Fri, 24 Feb 2012 13:17:39 -0500 Subject: [PATCH 093/154] mpegaudiodec: Prevent premature clipping of mp3 input buffer. Instead of clipping extrasize based on EXTRABYTES, clip based on the amount of buffer actually left. Without this fix, there are warbles and other distortions in the test case below. http://kevincennis.com/mix/assets/sounds/1901_voxfx.mp3 (cherry picked from commit b7165426917f91ebcad84bdff366824f03b32bfe) Signed-off-by: Alex Converse --- libavcodec/mpegaudiodec.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/libavcodec/mpegaudiodec.c b/libavcodec/mpegaudiodec.c index 04c1d9842c..d90257303c 100644 --- a/libavcodec/mpegaudiodec.c +++ b/libavcodec/mpegaudiodec.c @@ -40,6 +40,7 @@ #define BACKSTEP_SIZE 512 #define EXTRABYTES 24 +#define LAST_BUF_SIZE 2 * BACKSTEP_SIZE + EXTRABYTES /* layer 3 "granule" */ typedef struct GranuleDef { @@ -63,7 +64,7 @@ typedef struct GranuleDef { typedef struct MPADecodeContext { MPA_DECODE_HEADER - uint8_t last_buf[2 * BACKSTEP_SIZE + EXTRABYTES]; + uint8_t last_buf[LAST_BUF_SIZE]; int last_buf_size; /* next header (used in free format parsing) */ uint32_t free_format_next_header; @@ -1378,7 +1379,8 @@ static int mp_decode_layer3(MPADecodeContext *s) if (!s->adu_mode) { int skip; const uint8_t *ptr = s->gb.buffer + (get_bits_count(&s->gb)>>3); - int extrasize = av_clip(get_bits_left(&s->gb) >> 3, 0, EXTRABYTES); + int extrasize = av_clip(get_bits_left(&s->gb) >> 3, 0, + FFMAX(0, LAST_BUF_SIZE - s->last_buf_size)); assert((get_bits_count(&s->gb) & 7) == 0); /* now we get bits from the main_data_begin offset */ av_dlog(s->avctx, "seekback: %d\n", main_data_begin); From d7fddc97d40025876e1342109a49f07ba8fa6878 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Tue, 24 Jan 2012 17:48:23 +0100 Subject: [PATCH 094/154] dv: check stype dv: check stype Fixes part1 of CVE-2011-3929 Possibly fixes part of CVE-2011-3936 Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind Reviewed-by: Roman Shaposhnik Signed-off-by: Michael Niedermayer Signed-off-by: Alex Converse (cherry picked from commit 635bcfccd439480003b74a665b5aa7c872c1ad6b) --- libavformat/dv.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libavformat/dv.c b/libavformat/dv.c index 805f25271c..4896b0a3fe 100644 --- a/libavformat/dv.c +++ b/libavformat/dv.c @@ -204,6 +204,12 @@ static int dv_extract_audio_info(DVDemuxContext* c, uint8_t* frame) stype = (as_pack[3] & 0x1f); /* 0 - 2CH, 2 - 4CH, 3 - 8CH */ quant = as_pack[4] & 0x07; /* 0 - 16bit linear, 1 - 12bit nonlinear */ + if (stype > 3) { + av_log(c->fctx, AV_LOG_ERROR, "stype %d is invalid\n", stype); + c->ach = 0; + return 0; + } + /* note: ach counts PAIRS of channels (i.e. stereo channels) */ ach = ((int[4]){ 1, 0, 2, 4})[stype]; if (ach == 1 && quant && freq == 2) From efd30c4d95c56680f011c36a7f75c5c7389e34f2 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Tue, 24 Jan 2012 17:51:40 +0100 Subject: [PATCH 095/154] dv: Fix null pointer dereference due to ach=0 dv: Fix null pointer dereference due to ach=0 Fixes part2 of CVE-2011-3929 Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind Reviewed-by: Roman Shaposhnik Signed-off-by: Michael Niedermayer Signed-off-by: Alex Converse (cherry picked from commit 5a396bb3a66a61a68b80f2369d0249729bf85e04) --- libavformat/dv.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libavformat/dv.c b/libavformat/dv.c index 4896b0a3fe..bde8ca340a 100644 --- a/libavformat/dv.c +++ b/libavformat/dv.c @@ -343,7 +343,8 @@ int avpriv_dv_produce_packet(DVDemuxContext *c, AVPacket *pkt, c->audio_pkt[i].pts = c->abytes * 30000*8 / c->ast[i]->codec->bit_rate; ppcm[i] = c->audio_buf[i]; } - dv_extract_audio(buf, ppcm, c->sys); + if (c->ach) + dv_extract_audio(buf, ppcm, c->sys); /* We work with 720p frames split in half, thus even frames have * channels 0,1 and odd 2,3. */ From 3e8434bceafa11ede27657b0efec899d7178c06d Mon Sep 17 00:00:00 2001 From: Alex Converse Date: Thu, 26 Jan 2012 15:08:26 -0800 Subject: [PATCH 096/154] dv: Fix small stack overread related to CVE-2011-3929 and CVE-2011-3936. Found with asan. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind Signed-off-by: Alex Converse (cherry picked from commit 2d1c0dea5f6b91bec7f5fa53ec050913d851e366) --- libavformat/dv.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/libavformat/dv.c b/libavformat/dv.c index bde8ca340a..e517855b34 100644 --- a/libavformat/dv.c +++ b/libavformat/dv.c @@ -127,10 +127,14 @@ static int dv_extract_audio(uint8_t* frame, uint8_t* ppcm[4], /* We work with 720p frames split in half, thus even frames have * channels 0,1 and odd 2,3. */ ipcm = (sys->height == 720 && !(frame[1] & 0x0C)) ? 2 : 0; - pcm = ppcm[ipcm++]; /* for each DIF channel */ for (chan = 0; chan < sys->n_difchan; chan++) { + /* next stereo channel (50Mbps and 100Mbps only) */ + pcm = ppcm[ipcm++]; + if (!pcm) + break; + /* for each DIF segment */ for (i = 0; i < sys->difseg_size; i++) { frame += 6 * 80; /* skip DIF segment header */ @@ -178,11 +182,6 @@ static int dv_extract_audio(uint8_t* frame, uint8_t* ppcm[4], frame += 16 * 80; /* 15 Video DIFs + 1 Audio DIF */ } } - - /* next stereo channel (50Mbps and 100Mbps only) */ - pcm = ppcm[ipcm++]; - if (!pcm) - break; } return size; From 627f4621f5cb1a808e29026480570bd173c28d9b Mon Sep 17 00:00:00 2001 From: Mans Rullgard Date: Tue, 31 Jan 2012 10:20:33 -0800 Subject: [PATCH 097/154] ac3: Do not read past the end of ff_ac3_band_start_tab. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind Signed-off-by: Alex Converse (cherry picked from commit 034b03e7a0e8e4f8f66c82b736f2c0aa7c063ec0) --- libavcodec/ac3dsp.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/libavcodec/ac3dsp.c b/libavcodec/ac3dsp.c index 98c73573cb..b751aec902 100644 --- a/libavcodec/ac3dsp.c +++ b/libavcodec/ac3dsp.c @@ -108,7 +108,7 @@ static void ac3_bit_alloc_calc_bap_c(int16_t *mask, int16_t *psd, int snr_offset, int floor, const uint8_t *bap_tab, uint8_t *bap) { - int bin, band; + int bin, band, band_end; /* special case, if snr offset is -960, set all bap's to zero */ if (snr_offset == -960) { @@ -120,12 +120,14 @@ static void ac3_bit_alloc_calc_bap_c(int16_t *mask, int16_t *psd, band = ff_ac3_bin_to_band_tab[start]; do { int m = (FFMAX(mask[band] - snr_offset - floor, 0) & 0x1FE0) + floor; - int band_end = FFMIN(ff_ac3_band_start_tab[band+1], end); + band_end = ff_ac3_band_start_tab[++band]; + band_end = FFMIN(band_end, end); + for (; bin < band_end; bin++) { int address = av_clip((psd[bin] - m) >> 5, 0, 63); bap[bin] = bap_tab[address]; } - } while (end > ff_ac3_band_start_tab[band++]); + } while (end > band_end); } static void ac3_update_bap_counts_c(uint16_t mant_cnt[16], uint8_t *bap, From ce14f00dea933c930f46d1fb820dd02824a89fb4 Mon Sep 17 00:00:00 2001 From: Alex Converse Date: Fri, 3 Feb 2012 10:43:21 -0800 Subject: [PATCH 098/154] movdec: Avoid av_malloc(0) in stss Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind (cherry picked from commit 29a20ac4a19df5acc0eef306ca5a737778a31358) --- libavformat/mov.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libavformat/mov.c b/libavformat/mov.c index c6022d5d3a..d14ae7ee0d 100644 --- a/libavformat/mov.c +++ b/libavformat/mov.c @@ -1503,6 +1503,8 @@ static int mov_read_stss(MOVContext *c, AVIOContext *pb, MOVAtom atom) av_dlog(c->fc, "keyframe_count = %d\n", entries); + if (!entries) + return 0; if (entries >= UINT_MAX / sizeof(int)) return AVERROR_INVALIDDATA; sc->keyframes = av_malloc(entries * sizeof(int)); From e3743869e97568b75c100b643bf8df4c70f7d93e Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Thu, 2 Feb 2012 22:27:27 -0500 Subject: [PATCH 099/154] ac3dec: Move center and surround mix level tables to the parser. That way all mix levels as exported by avpriv_ac3_parse_header() will have the same meaning. Previously the 3-bit center mix level for E-AC-3 was used to index in a 4-entry table, leading to out-of-array reads. Signed-off-by: Michael Niedermayer Signed-off-by: Justin Ruggles Signed-off-by: Alex Converse (cherry picked from commit e6d9fa66f12cf5a3024c9bc7c4c608f7fc59207e) --- libavcodec/ac3_parser.c | 20 ++++++++++++++++---- libavcodec/ac3dec.c | 16 ++-------------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/libavcodec/ac3_parser.c b/libavcodec/ac3_parser.c index e3c46fd332..067b4f9879 100644 --- a/libavcodec/ac3_parser.c +++ b/libavcodec/ac3_parser.c @@ -34,6 +34,18 @@ static const uint8_t eac3_blocks[4] = { 1, 2, 3, 6 }; +/** + * Table for center mix levels + * reference: Section 5.4.2.4 cmixlev + */ +static const uint8_t center_levels[4] = { 4, 5, 6, 5 }; + +/** + * Table for surround mix levels + * reference: Section 5.4.2.5 surmixlev + */ +static const uint8_t surround_levels[4] = { 4, 6, 7, 6 }; + int avpriv_ac3_parse_header(GetBitContext *gbc, AC3HeaderInfo *hdr) { @@ -53,8 +65,8 @@ int avpriv_ac3_parse_header(GetBitContext *gbc, AC3HeaderInfo *hdr) hdr->num_blocks = 6; /* set default mix levels */ - hdr->center_mix_level = 1; // -4.5dB - hdr->surround_mix_level = 1; // -6.0dB + hdr->center_mix_level = 5; // -4.5dB + hdr->surround_mix_level = 6; // -6.0dB if(hdr->bitstream_id <= 10) { /* Normal AC-3 */ @@ -76,9 +88,9 @@ int avpriv_ac3_parse_header(GetBitContext *gbc, AC3HeaderInfo *hdr) skip_bits(gbc, 2); // skip dsurmod } else { if((hdr->channel_mode & 1) && hdr->channel_mode != AC3_CHMODE_MONO) - hdr->center_mix_level = get_bits(gbc, 2); + hdr-> center_mix_level = center_levels[get_bits(gbc, 2)]; if(hdr->channel_mode & 4) - hdr->surround_mix_level = get_bits(gbc, 2); + hdr->surround_mix_level = surround_levels[get_bits(gbc, 2)]; } hdr->lfe_on = get_bits1(gbc); diff --git a/libavcodec/ac3dec.c b/libavcodec/ac3dec.c index 662ea91d1f..1020eea0d6 100644 --- a/libavcodec/ac3dec.c +++ b/libavcodec/ac3dec.c @@ -76,18 +76,6 @@ static const float gain_levels[9] = { LEVEL_MINUS_9DB }; -/** - * Table for center mix levels - * reference: Section 5.4.2.4 cmixlev - */ -static const uint8_t center_levels[4] = { 4, 5, 6, 5 }; - -/** - * Table for surround mix levels - * reference: Section 5.4.2.5 surmixlev - */ -static const uint8_t surround_levels[4] = { 4, 6, 7, 6 }; - /** * Table for default stereo downmixing coefficients * reference: Section 7.8.2 Downmixing Into Two Channels @@ -320,8 +308,8 @@ static int parse_frame_header(AC3DecodeContext *s) static void set_downmix_coeffs(AC3DecodeContext *s) { int i; - float cmix = gain_levels[center_levels[s->center_mix_level]]; - float smix = gain_levels[surround_levels[s->surround_mix_level]]; + float cmix = gain_levels[s-> center_mix_level]; + float smix = gain_levels[s->surround_mix_level]; float norm0, norm1; for (i = 0; i < s->fbw_channels; i++) { From 035dd77cbb01215daeef7e4e9cf1218b7fee354c Mon Sep 17 00:00:00 2001 From: Alex Converse Date: Thu, 9 Feb 2012 17:11:55 -0800 Subject: [PATCH 100/154] dv: Fix small overread in audio frequency table. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind (cherry picked from commit 0ab3687924457cb4fd81897bd39ab3cc5b699588) --- libavformat/dv.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/libavformat/dv.c b/libavformat/dv.c index e517855b34..65d0f873dc 100644 --- a/libavformat/dv.c +++ b/libavformat/dv.c @@ -121,6 +121,9 @@ static int dv_extract_audio(uint8_t* frame, uint8_t* ppcm[4], if (quant > 1) return -1; /* unsupported quantization */ + if (freq >= FF_ARRAY_ELEMS(dv_audio_frequency)) + return AVERROR_INVALIDDATA; + size = (sys->audio_min_samples[freq] + smpls) * 4; /* 2ch, 2bytes */ half_ch = sys->difseg_size / 2; @@ -203,6 +206,12 @@ static int dv_extract_audio_info(DVDemuxContext* c, uint8_t* frame) stype = (as_pack[3] & 0x1f); /* 0 - 2CH, 2 - 4CH, 3 - 8CH */ quant = as_pack[4] & 0x07; /* 0 - 16bit linear, 1 - 12bit nonlinear */ + if (freq >= FF_ARRAY_ELEMS(dv_audio_frequency)) { + av_log(c->fctx, AV_LOG_ERROR, + "Unrecognized audio sample rate index (%d)\n", freq); + return 0; + } + if (stype > 3) { av_log(c->fctx, AV_LOG_ERROR, "stype %d is invalid\n", stype); c->ach = 0; From db315c796d7f07f0dcd7d3be1e9cb77ae6afee6e Mon Sep 17 00:00:00 2001 From: Alex Converse Date: Thu, 9 Feb 2012 20:21:47 -0800 Subject: [PATCH 101/154] svq3: Prevent illegal reads while parsing extradata. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind (cherry picked from commit 9e1db721c4329f4ac166a0bcc002c8d75f831aba) --- libavcodec/svq3.c | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/libavcodec/svq3.c b/libavcodec/svq3.c index 5cc57a745d..eeb8ed7051 100644 --- a/libavcodec/svq3.c +++ b/libavcodec/svq3.c @@ -811,7 +811,9 @@ static av_cold int svq3_decode_init(AVCodecContext *avctx) MpegEncContext *s = &h->s; int m; unsigned char *extradata; + unsigned char *extradata_end; unsigned int size; + int marker_found = 0; if (ff_h264_decode_init(avctx) < 0) return -1; @@ -831,19 +833,26 @@ static av_cold int svq3_decode_init(AVCodecContext *avctx) /* prowl for the "SEQH" marker in the extradata */ extradata = (unsigned char *)avctx->extradata; - for (m = 0; m < avctx->extradata_size; m++) { - if (!memcmp(extradata, "SEQH", 4)) - break; - extradata++; + extradata_end = avctx->extradata + avctx->extradata_size; + if (extradata) { + for (m = 0; m + 8 < avctx->extradata_size; m++) { + if (!memcmp(extradata, "SEQH", 4)) { + marker_found = 1; + break; + } + extradata++; + } } /* if a match was found, parse the extra data */ - if (extradata && !memcmp(extradata, "SEQH", 4)) { + if (marker_found) { GetBitContext gb; int frame_size_code; size = AV_RB32(&extradata[4]); + if (size > extradata_end - extradata - 8) + return AVERROR_INVALIDDATA; init_get_bits(&gb, extradata + 8, size*8); /* 'frame size code' and optional 'width, height' */ From eaeaeb265fe46e1d81452960de918227541873b4 Mon Sep 17 00:00:00 2001 From: Alex Converse Date: Fri, 17 Feb 2012 14:13:40 -0800 Subject: [PATCH 102/154] dpcm: ignore extra unpaired bytes in stereo streams. Fixes: CVE-2011-3951 Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind (cherry picked from commit ce7aee9b733134649a6ce2fa743e51733f33e67e) --- libavcodec/dpcm.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/libavcodec/dpcm.c b/libavcodec/dpcm.c index 1b0f6b005b..7f5dbfe3b9 100644 --- a/libavcodec/dpcm.c +++ b/libavcodec/dpcm.c @@ -183,6 +183,11 @@ static int dpcm_decode_frame(AVCodecContext *avctx, void *data, int stereo = s->channels - 1; int16_t *output_samples; + if (stereo && (buf_size & 1)) { + buf_size--; + buf_end--; + } + /* calculate output size */ switch(avctx->codec->id) { case CODEC_ID_ROQ_DPCM: @@ -317,7 +322,7 @@ static int dpcm_decode_frame(AVCodecContext *avctx, void *data, *got_frame_ptr = 1; *(AVFrame *)data = s->frame; - return buf_size; + return avpkt->size; } #define DPCM_DECODER(id_, name_, long_name_) \ From ef673211e7052d6db4dbec4b58db0f514b292288 Mon Sep 17 00:00:00 2001 From: Alex Converse Date: Thu, 23 Feb 2012 10:22:51 -0800 Subject: [PATCH 103/154] tiff: Make the TIFF_LONG and TIFF_SHORT types unsigned. TIFF v6.0 (unimplemented) adds signed equivalents. (cherry picked from commit e32548d1331ce05a054f1028fcdda8823a4f215a) --- libavcodec/tiff.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/libavcodec/tiff.c b/libavcodec/tiff.c index 6810f81b35..d807149922 100644 --- a/libavcodec/tiff.c +++ b/libavcodec/tiff.c @@ -59,24 +59,24 @@ typedef struct TiffContext { LZWState *lzw; } TiffContext; -static int tget_short(const uint8_t **p, int le){ - int v = le ? AV_RL16(*p) : AV_RB16(*p); +static unsigned tget_short(const uint8_t **p, int le) { + unsigned v = le ? AV_RL16(*p) : AV_RB16(*p); *p += 2; return v; } -static int tget_long(const uint8_t **p, int le){ - int v = le ? AV_RL32(*p) : AV_RB32(*p); +static unsigned tget_long(const uint8_t **p, int le) { + unsigned v = le ? AV_RL32(*p) : AV_RB32(*p); *p += 4; return v; } -static int tget(const uint8_t **p, int type, int le){ +static unsigned tget(const uint8_t **p, int type, int le) { switch(type){ case TIFF_BYTE : return *(*p)++; case TIFF_SHORT: return tget_short(p, le); case TIFF_LONG : return tget_long (p, le); - default : return -1; + default : return UINT_MAX; } } @@ -277,7 +277,7 @@ static int init_image(TiffContext *s) static int tiff_decode_tag(TiffContext *s, const uint8_t *start, const uint8_t *buf, const uint8_t *end_buf) { - int tag, type, count, off, value = 0; + unsigned tag, type, count, off, value = 0; int i, j; uint32_t *pal; const uint8_t *rp, *gp, *bp; @@ -312,7 +312,7 @@ static int tiff_decode_tag(TiffContext *s, const uint8_t *start, const uint8_t * break; } default: - value = -1; + value = UINT_MAX; buf = start + off; } } else { @@ -398,7 +398,7 @@ static int tiff_decode_tag(TiffContext *s, const uint8_t *start, const uint8_t * } break; case TIFF_ROWSPERSTRIP: - if(type == TIFF_LONG && value == -1) + if (type == TIFF_LONG && value == UINT_MAX) value = s->avctx->height; if(value < 1){ av_log(s->avctx, AV_LOG_ERROR, "Incorrect value of rows per strip\n"); From e891ee4bf639099c21bb146a734d31ad7f910acf Mon Sep 17 00:00:00 2001 From: Alex Converse Date: Tue, 28 Feb 2012 11:50:22 -0800 Subject: [PATCH 104/154] adpcm: Clip step_index values read from the bitstream at the beginning of each frame. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind (cherry picked from commit bbeb29133b55b7256d18f5aaab8b5c8e919a173a) --- libavcodec/adpcm.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/libavcodec/adpcm.c b/libavcodec/adpcm.c index b319635ed4..a540a9b12f 100644 --- a/libavcodec/adpcm.c +++ b/libavcodec/adpcm.c @@ -690,7 +690,7 @@ static int adpcm_decode_frame(AVCodecContext *avctx, void *data, for (channel = 0; channel < avctx->channels; channel++) { cs = &c->status[channel]; cs->predictor = (int16_t)bytestream_get_le16(&src); - cs->step_index = *src++; + cs->step_index = av_clip(*src++, 0, 88); src++; *samples++ = cs->predictor; } @@ -713,8 +713,8 @@ static int adpcm_decode_frame(AVCodecContext *avctx, void *data, c->status[0].predictor = (int16_t)AV_RL16(src + 10); c->status[1].predictor = (int16_t)AV_RL16(src + 12); - c->status[0].step_index = src[14]; - c->status[1].step_index = src[15]; + c->status[0].step_index = av_clip(src[14], 0, 88); + c->status[1].step_index = av_clip(src[15], 0, 88); /* sign extend the predictors */ src += 16; diff_channel = c->status[1].predictor; @@ -754,7 +754,7 @@ static int adpcm_decode_frame(AVCodecContext *avctx, void *data, for (channel = 0; channel < avctx->channels; channel++) { cs = &c->status[channel]; cs->predictor = (int16_t)bytestream_get_le16(&src); - cs->step_index = *src++; + cs->step_index = av_clip(*src++, 0, 88); src++; } @@ -793,7 +793,7 @@ static int adpcm_decode_frame(AVCodecContext *avctx, void *data, src += 4; // skip sample count (already read) for (i=0; i<=st; i++) - c->status[i].step_index = bytestream_get_le32(&src); + c->status[i].step_index = av_clip(bytestream_get_le32(&src), 0, 88); for (i=0; i<=st; i++) c->status[i].predictor = bytestream_get_le32(&src); @@ -1007,11 +1007,11 @@ static int adpcm_decode_frame(AVCodecContext *avctx, void *data, case CODEC_ID_ADPCM_IMA_SMJPEG: if (avctx->codec->id == CODEC_ID_ADPCM_IMA_AMV) { c->status[0].predictor = sign_extend(bytestream_get_le16(&src), 16); - c->status[0].step_index = bytestream_get_le16(&src); + c->status[0].step_index = av_clip(bytestream_get_le16(&src), 0, 88); src += 4; } else { c->status[0].predictor = sign_extend(bytestream_get_be16(&src), 16); - c->status[0].step_index = bytestream_get_byte(&src); + c->status[0].step_index = av_clip(bytestream_get_byte(&src), 0, 88); src += 1; } From 522645e38f6d0aa78ebf3afb356e7427bf4eb248 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Fri, 17 Feb 2012 13:35:10 -0800 Subject: [PATCH 105/154] h263dec: Disallow width/height changing with frame threads. Fixes CVE-2011-3937 Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind Signed-off-by: Michael Niedermayer (cherry picked from commit 71db86d53b5c6872cea31bf714a1a38ec78feaba) Conflicts: libavcodec/h263dec.c Signed-off-by: Alex Converse --- libavcodec/h263dec.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/libavcodec/h263dec.c b/libavcodec/h263dec.c index f056d1fbe2..ba0ea4f9f9 100644 --- a/libavcodec/h263dec.c +++ b/libavcodec/h263dec.c @@ -556,8 +556,7 @@ retry: #if HAVE_MMX if (s->codec_id == CODEC_ID_MPEG4 && s->xvid_build>=0 && avctx->idct_algo == FF_IDCT_AUTO && (av_get_cpu_flags() & AV_CPU_FLAG_MMX)) { avctx->idct_algo= FF_IDCT_XVIDMMX; - avctx->coded_width= 0; // force reinit -// dsputil_init(&s->dsp, avctx); + ff_dct_common_init(s); s->picture_number=0; } #endif @@ -571,6 +570,12 @@ retry: || s->height != avctx->coded_height) { /* H.263 could change picture size any time */ ParseContext pc= s->parse_context; //FIXME move these demuxng hack to avformat + + if (HAVE_THREADS && (s->avctx->active_thread_type&FF_THREAD_FRAME)) { + av_log_missing_feature(s->avctx, "Width/height/bit depth/chroma idc changing with threads is", 0); + return -1; // width / height changed during parallelized decoding + } + s->parse_context.buffer=0; MPV_common_end(s); s->parse_context= pc; From 48ac765efe05826184bc129678e0fdf3474b99dd Mon Sep 17 00:00:00 2001 From: Alex Converse Date: Thu, 1 Mar 2012 13:24:55 -0800 Subject: [PATCH 106/154] rv10/20: Fix slice overflow with checked bitstream reader. (cherry picked from commit 9243ec4a508c81a621e941bb7e012e2d45d93659) --- libavcodec/rv10.c | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/libavcodec/rv10.c b/libavcodec/rv10.c index d7d7ed2fb2..ff6c9c3078 100644 --- a/libavcodec/rv10.c +++ b/libavcodec/rv10.c @@ -499,9 +499,10 @@ static int rv10_decode_packet(AVCodecContext *avctx, const uint8_t *buf, int buf_size, int buf_size2) { MpegEncContext *s = avctx->priv_data; - int mb_count, mb_pos, left, start_mb_x; + int mb_count, mb_pos, left, start_mb_x, active_bits_size; - init_get_bits(&s->gb, buf, buf_size*8); + active_bits_size = buf_size * 8; + init_get_bits(&s->gb, buf, FFMAX(buf_size, buf_size2) * 8); if(s->codec_id ==CODEC_ID_RV10) mb_count = rv10_decode_picture_header(s); else @@ -584,13 +585,26 @@ static int rv10_decode_packet(AVCodecContext *avctx, s->mv_type = MV_TYPE_16X16; ret=ff_h263_decode_mb(s, s->block); - if (ret != SLICE_ERROR && s->gb.size_in_bits < get_bits_count(&s->gb) && 8*buf_size2 >= get_bits_count(&s->gb)){ - av_log(avctx, AV_LOG_DEBUG, "update size from %d to %d\n", s->gb.size_in_bits, 8*buf_size2); - s->gb.size_in_bits= 8*buf_size2; + // Repeat the slice end check from ff_h263_decode_mb with our active + // bitstream size + if (ret != SLICE_ERROR) { + int v = show_bits(&s->gb, 16); + + if (get_bits_count(&s->gb) + 16 > active_bits_size) + v >>= get_bits_count(&s->gb) + 16 - active_bits_size; + + if (!v) + ret = SLICE_END; + } + if (ret != SLICE_ERROR && active_bits_size < get_bits_count(&s->gb) && + 8 * buf_size2 >= get_bits_count(&s->gb)) { + active_bits_size = buf_size2 * 8; + av_log(avctx, AV_LOG_DEBUG, "update size from %d to %d\n", + 8 * buf_size, active_bits_size); ret= SLICE_OK; } - if (ret == SLICE_ERROR || s->gb.size_in_bits < get_bits_count(&s->gb)) { + if (ret == SLICE_ERROR || active_bits_size < get_bits_count(&s->gb)) { av_log(s->avctx, AV_LOG_ERROR, "ERROR at MB %d %d\n", s->mb_x, s->mb_y); return -1; } @@ -612,7 +626,7 @@ static int rv10_decode_packet(AVCodecContext *avctx, ff_er_add_slice(s, start_mb_x, s->resync_mb_y, s->mb_x-1, s->mb_y, ER_MB_END); - return s->gb.size_in_bits; + return active_bits_size; } static int get_slice_offset(AVCodecContext *avctx, const uint8_t *buf, int n) From 4a325ddeae486c0bb2f73b886e16e30e305f9d20 Mon Sep 17 00:00:00 2001 From: Alex Converse Date: Tue, 21 Feb 2012 14:08:02 -0800 Subject: [PATCH 107/154] mov: Add support for MPEG2 HDV 720p24 (hdv4) (cherry picked from commit 0ad522afb3a3b3d22402ecb82dd4609f7655031b) --- libavformat/isom.c | 1 + 1 file changed, 1 insertion(+) diff --git a/libavformat/isom.c b/libavformat/isom.c index eab304c006..edd64f4627 100644 --- a/libavformat/isom.c +++ b/libavformat/isom.c @@ -159,6 +159,7 @@ const AVCodecTag codec_movvideo_tags[] = { { CODEC_ID_MPEG2VIDEO, MKTAG('h', 'd', 'v', '1') }, /* MPEG2 HDV 720p30 */ { CODEC_ID_MPEG2VIDEO, MKTAG('h', 'd', 'v', '2') }, /* MPEG2 HDV 1080i60 */ { CODEC_ID_MPEG2VIDEO, MKTAG('h', 'd', 'v', '3') }, /* MPEG2 HDV 1080i50 */ + { CODEC_ID_MPEG2VIDEO, MKTAG('h', 'd', 'v', '4') }, /* MPEG2 HDV 720p24 */ { CODEC_ID_MPEG2VIDEO, MKTAG('h', 'd', 'v', '5') }, /* MPEG2 HDV 720p25 */ { CODEC_ID_MPEG2VIDEO, MKTAG('h', 'd', 'v', '6') }, /* MPEG2 HDV 1080p24 */ { CODEC_ID_MPEG2VIDEO, MKTAG('h', 'd', 'v', '7') }, /* MPEG2 HDV 1080p25 */ From fb049da952668a54c3a82f3fee93d8384b254738 Mon Sep 17 00:00:00 2001 From: Alex Converse Date: Tue, 21 Feb 2012 15:37:35 -0800 Subject: [PATCH 108/154] mov: Add more HDV and XDCAM FourCCs. Reference: VLC (cherry picked from commit b142496c5630b9bc88fb9eaccae7f6bd62fb23e7) --- libavformat/isom.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libavformat/isom.c b/libavformat/isom.c index edd64f4627..07f22ca123 100644 --- a/libavformat/isom.c +++ b/libavformat/isom.c @@ -164,6 +164,8 @@ const AVCodecTag codec_movvideo_tags[] = { { CODEC_ID_MPEG2VIDEO, MKTAG('h', 'd', 'v', '6') }, /* MPEG2 HDV 1080p24 */ { CODEC_ID_MPEG2VIDEO, MKTAG('h', 'd', 'v', '7') }, /* MPEG2 HDV 1080p25 */ { CODEC_ID_MPEG2VIDEO, MKTAG('h', 'd', 'v', '8') }, /* MPEG2 HDV 1080p30 */ + { CODEC_ID_MPEG2VIDEO, MKTAG('h', 'd', 'v', '9') }, /* MPEG2 HDV 720p60 JVC */ + { CODEC_ID_MPEG2VIDEO, MKTAG('h', 'd', 'v', 'a') }, /* MPEG2 HDV 720p50 */ { CODEC_ID_MPEG2VIDEO, MKTAG('m', 'x', '5', 'n') }, /* MPEG2 IMX NTSC 525/60 50mb/s produced by FCP */ { CODEC_ID_MPEG2VIDEO, MKTAG('m', 'x', '5', 'p') }, /* MPEG2 IMX PAL 625/50 50mb/s produced by FCP */ { CODEC_ID_MPEG2VIDEO, MKTAG('m', 'x', '4', 'n') }, /* MPEG2 IMX NTSC 525/60 40mb/s produced by FCP */ @@ -194,6 +196,8 @@ const AVCodecTag codec_movvideo_tags[] = { { CODEC_ID_MPEG2VIDEO, MKTAG('x', 'd', 'v', 'd') }, /* XDCAM EX 1080p24 VBR */ { CODEC_ID_MPEG2VIDEO, MKTAG('x', 'd', 'v', 'e') }, /* XDCAM EX 1080p25 VBR */ { CODEC_ID_MPEG2VIDEO, MKTAG('x', 'd', 'v', 'f') }, /* XDCAM EX 1080p30 VBR */ + { CODEC_ID_MPEG2VIDEO, MKTAG('x', 'd', 'h', 'd') }, /* XDCAM HD 540p */ + { CODEC_ID_MPEG2VIDEO, MKTAG('x', 'd', 'h', '2') }, /* XDCAM HD422 540p */ { CODEC_ID_MPEG2VIDEO, MKTAG('A', 'V', 'm', 'p') }, /* AVID IMX PAL */ { CODEC_ID_JPEG2000, MKTAG('m', 'j', 'p', '2') }, /* JPEG 2000 produced by FCP */ From a47b96bdd31e00dfa03429ee3b04b84d035bf7f8 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Mon, 13 Feb 2012 21:10:48 +0100 Subject: [PATCH 109/154] rv34: handle size changes during frame multithreading Factors all context dynamic memory handling to its own functions. Fixes bug 220. (cherry picked from commit 2bd730010da24d035639586bb13862abe36cc1b8) Signed-off-by: Reinhard Tartler --- libavcodec/rv34.c | 146 ++++++++++++++++++++++++++-------------------- 1 file changed, 82 insertions(+), 64 deletions(-) diff --git a/libavcodec/rv34.c b/libavcodec/rv34.c index e6af0793d3..0aecc2379e 100644 --- a/libavcodec/rv34.c +++ b/libavcodec/rv34.c @@ -711,8 +711,7 @@ static inline void rv34_mc(RV34DecContext *r, const int block_type, if (HAVE_THREADS && (s->avctx->active_thread_type & FF_THREAD_FRAME)) { /* wait for the referenced mb row to be finished */ - int mb_row = FFMIN(s->mb_height - 1, - s->mb_y + ((yoff + my + 5 + 8 * height) >> 4)); + int mb_row = s->mb_y + ((yoff + my + 5 + 8 * height) >> 4); AVFrame *f = dir ? &s->next_picture_ptr->f : &s->last_picture_ptr->f; ff_thread_await_progress(f, mb_row, 0); } @@ -1361,6 +1360,53 @@ static int check_slice_end(RV34DecContext *r, MpegEncContext *s) return 0; } + +static void rv34_decoder_free(RV34DecContext *r) +{ + av_freep(&r->intra_types_hist); + r->intra_types = NULL; + av_freep(&r->tmp_b_block_base); + av_freep(&r->mb_type); + av_freep(&r->cbp_luma); + av_freep(&r->cbp_chroma); + av_freep(&r->deblock_coefs); +} + + +static int rv34_decoder_alloc(RV34DecContext *r) +{ + r->intra_types_stride = r->s.mb_width * 4 + 4; + + r->cbp_chroma = av_malloc(r->s.mb_stride * r->s.mb_height * + sizeof(*r->cbp_chroma)); + r->cbp_luma = av_malloc(r->s.mb_stride * r->s.mb_height * + sizeof(*r->cbp_luma)); + r->deblock_coefs = av_malloc(r->s.mb_stride * r->s.mb_height * + sizeof(*r->deblock_coefs)); + r->intra_types_hist = av_malloc(r->intra_types_stride * 4 * 2 * + sizeof(*r->intra_types_hist)); + r->mb_type = av_mallocz(r->s.mb_stride * r->s.mb_height * + sizeof(*r->mb_type)); + + if (!(r->cbp_chroma && r->cbp_luma && r->deblock_coefs && + r->intra_types_hist && r->mb_type)) { + rv34_decoder_free(r); + return AVERROR(ENOMEM); + } + + r->intra_types = r->intra_types_hist + r->intra_types_stride * 4; + + return 0; +} + + +static int rv34_decoder_realloc(RV34DecContext *r) +{ + rv34_decoder_free(r); + return rv34_decoder_alloc(r); +} + + static int rv34_decode_slice(RV34DecContext *r, int end, const uint8_t* buf, int buf_size) { MpegEncContext *s = &r->s; @@ -1376,22 +1422,19 @@ static int rv34_decode_slice(RV34DecContext *r, int end, const uint8_t* buf, int } if ((s->mb_x == 0 && s->mb_y == 0) || s->current_picture_ptr==NULL) { - if(s->width != r->si.width || s->height != r->si.height){ - av_log(s->avctx, AV_LOG_DEBUG, "Changing dimensions to %dx%d\n", r->si.width,r->si.height); + if (s->width != r->si.width || s->height != r->si.height) { + int err; + + av_log(s->avctx, AV_LOG_WARNING, "Changing dimensions to %dx%d\n", + r->si.width, r->si.height); MPV_common_end(s); s->width = r->si.width; s->height = r->si.height; avcodec_set_dimensions(s->avctx, s->width, s->height); - if(MPV_common_init(s) < 0) - return -1; - r->intra_types_stride = s->mb_width*4 + 4; - r->intra_types_hist = av_realloc(r->intra_types_hist, r->intra_types_stride * 4 * 2 * sizeof(*r->intra_types_hist)); - r->intra_types = r->intra_types_hist + r->intra_types_stride * 4; - r->mb_type = av_realloc(r->mb_type, r->s.mb_stride * r->s.mb_height * sizeof(*r->mb_type)); - r->cbp_luma = av_realloc(r->cbp_luma, r->s.mb_stride * r->s.mb_height * sizeof(*r->cbp_luma)); - r->cbp_chroma = av_realloc(r->cbp_chroma, r->s.mb_stride * r->s.mb_height * sizeof(*r->cbp_chroma)); - r->deblock_coefs = av_realloc(r->deblock_coefs, r->s.mb_stride * r->s.mb_height * sizeof(*r->deblock_coefs)); - av_freep(&r->tmp_b_block_base); + if ((err = MPV_common_init(s)) < 0) + return err; + if ((err = rv34_decoder_realloc(r)) < 0) + return err; } s->pict_type = r->si.type ? r->si.type : AV_PICTURE_TYPE_I; if(MPV_frame_start(s, s->avctx) < 0) @@ -1496,6 +1539,7 @@ av_cold int ff_rv34_decode_init(AVCodecContext *avctx) { RV34DecContext *r = avctx->priv_data; MpegEncContext *s = &r->s; + int ret; MPV_decode_defaults(s); s->avctx = avctx; @@ -1512,8 +1556,8 @@ av_cold int ff_rv34_decode_init(AVCodecContext *avctx) avctx->has_b_frames = 1; s->low_delay = 0; - if (MPV_common_init(s) < 0) - return -1; + if ((ret = MPV_common_init(s)) < 0) + return ret; ff_h264_pred_init(&r->h, CODEC_ID_RV40, 8, 1); @@ -1526,15 +1570,8 @@ av_cold int ff_rv34_decode_init(AVCodecContext *avctx) ff_rv40dsp_init(&r->rdsp, &r->s.dsp); #endif - r->intra_types_stride = 4*s->mb_stride + 4; - r->intra_types_hist = av_malloc(r->intra_types_stride * 4 * 2 * sizeof(*r->intra_types_hist)); - r->intra_types = r->intra_types_hist + r->intra_types_stride * 4; - - r->mb_type = av_mallocz(r->s.mb_stride * r->s.mb_height * sizeof(*r->mb_type)); - - r->cbp_luma = av_malloc(r->s.mb_stride * r->s.mb_height * sizeof(*r->cbp_luma)); - r->cbp_chroma = av_malloc(r->s.mb_stride * r->s.mb_height * sizeof(*r->cbp_chroma)); - r->deblock_coefs = av_malloc(r->s.mb_stride * r->s.mb_height * sizeof(*r->deblock_coefs)); + if ((ret = rv34_decoder_alloc(r)) < 0) + return ret; if(!intra_vlcs[0].cbppattern[0].bits) rv34_init_tables(); @@ -1544,40 +1581,17 @@ av_cold int ff_rv34_decode_init(AVCodecContext *avctx) int ff_rv34_decode_init_thread_copy(AVCodecContext *avctx) { + int err; RV34DecContext *r = avctx->priv_data; r->s.avctx = avctx; if (avctx->internal->is_copy) { - r->cbp_chroma = av_malloc(r->s.mb_stride * r->s.mb_height * - sizeof(*r->cbp_chroma)); - r->cbp_luma = av_malloc(r->s.mb_stride * r->s.mb_height * - sizeof(*r->cbp_luma)); - r->deblock_coefs = av_malloc(r->s.mb_stride * r->s.mb_height * - sizeof(*r->deblock_coefs)); - r->intra_types_hist = av_malloc(r->intra_types_stride * 4 * 2 * - sizeof(*r->intra_types_hist)); - r->mb_type = av_malloc(r->s.mb_stride * r->s.mb_height * - sizeof(*r->mb_type)); - - if (!(r->cbp_chroma && r->cbp_luma && r->deblock_coefs && - r->intra_types_hist && r->mb_type)) { - av_freep(&r->cbp_chroma); - av_freep(&r->cbp_luma); - av_freep(&r->deblock_coefs); - av_freep(&r->intra_types_hist); - av_freep(&r->mb_type); - r->intra_types = NULL; - return AVERROR(ENOMEM); - } - - r->intra_types = r->intra_types_hist + r->intra_types_stride * 4; r->tmp_b_block_base = NULL; - - memset(r->mb_type, 0, r->s.mb_stride * r->s.mb_height * - sizeof(*r->mb_type)); - - MPV_common_init(&r->s); + if ((err = MPV_common_init(&r->s)) < 0) + return err; + if ((err = rv34_decoder_alloc(r)) < 0) + return err; } return 0; } @@ -1591,6 +1605,16 @@ int ff_rv34_decode_update_thread_context(AVCodecContext *dst, const AVCodecConte if (dst == src || !s1->context_initialized) return 0; + if (s->height != s1->height || s->width != s1->width) { + MPV_common_end(s); + s->height = s1->height; + s->width = s1->width; + if ((err = MPV_common_init(s)) < 0) + return err; + if ((err = rv34_decoder_realloc(r)) < 0) + return err; + } + if ((err = ff_mpeg_update_thread_context(dst, src))) return err; @@ -1708,11 +1732,12 @@ int ff_rv34_decode_frame(AVCodecContext *avctx, if(last && s->current_picture_ptr){ if(r->loop_filter) r->loop_filter(r, s->mb_height - 1); - if (HAVE_THREADS && (s->avctx->active_thread_type & FF_THREAD_FRAME)) - ff_thread_report_progress(&s->current_picture_ptr->f, - s->mb_height - 1, 0); ff_er_frame_end(s); MPV_frame_end(s); + + if (HAVE_THREADS && (s->avctx->active_thread_type & FF_THREAD_FRAME)) + ff_thread_report_progress(&s->current_picture_ptr->f, INT_MAX, 0); + if (s->pict_type == AV_PICTURE_TYPE_B || s->low_delay) { *pict = *(AVFrame*)s->current_picture_ptr; } else if (s->last_picture_ptr != NULL) { @@ -1733,14 +1758,7 @@ av_cold int ff_rv34_decode_end(AVCodecContext *avctx) RV34DecContext *r = avctx->priv_data; MPV_common_end(&r->s); - - av_freep(&r->intra_types_hist); - r->intra_types = NULL; - av_freep(&r->tmp_b_block_base); - av_freep(&r->mb_type); - av_freep(&r->cbp_luma); - av_freep(&r->cbp_chroma); - av_freep(&r->deblock_coefs); + rv34_decoder_free(r); return 0; } From 4a15240a274c1eada288d27c889443ebd6aa62f8 Mon Sep 17 00:00:00 2001 From: Justin Ruggles Date: Sun, 12 Feb 2012 15:06:58 -0500 Subject: [PATCH 110/154] mov: set channel layout for AC-3 streams based on the 'dac3' atom info fixes Bug 225 (cherry picked from commit 3798205a77ce275613098ecb48645e6029811f14) Signed-off-by: Reinhard Tartler --- libavcodec/Makefile | 2 +- libavcodec/ac3_parser.c | 2 +- libavcodec/ac3dec.c | 2 +- libavcodec/ac3tab.c | 2 +- libavcodec/ac3tab.h | 2 +- libavformat/mov.c | 5 +++++ 6 files changed, 10 insertions(+), 5 deletions(-) diff --git a/libavcodec/Makefile b/libavcodec/Makefile index 1e8d09b956..5a4fa4cbe9 100644 --- a/libavcodec/Makefile +++ b/libavcodec/Makefile @@ -549,7 +549,7 @@ OBJS-$(CONFIG_MATROSKA_MUXER) += xiph.o mpeg4audio.o \ flacdec.o flacdata.o flac.o \ mpegaudiodata.o OBJS-$(CONFIG_MP3_MUXER) += mpegaudiodata.o mpegaudiodecheader.o -OBJS-$(CONFIG_MOV_DEMUXER) += mpeg4audio.o mpegaudiodata.o +OBJS-$(CONFIG_MOV_DEMUXER) += mpeg4audio.o mpegaudiodata.o ac3tab.o OBJS-$(CONFIG_MOV_MUXER) += mpeg4audio.o mpegaudiodata.o OBJS-$(CONFIG_MPEGTS_MUXER) += mpegvideo.o mpeg4audio.o OBJS-$(CONFIG_MPEGTS_DEMUXER) += mpeg4audio.o mpegaudiodata.o diff --git a/libavcodec/ac3_parser.c b/libavcodec/ac3_parser.c index 067b4f9879..d9ba1fd70b 100644 --- a/libavcodec/ac3_parser.c +++ b/libavcodec/ac3_parser.c @@ -134,7 +134,7 @@ int avpriv_ac3_parse_header(GetBitContext *gbc, AC3HeaderInfo *hdr) (hdr->num_blocks * 256.0)); hdr->channels = ff_ac3_channels_tab[hdr->channel_mode] + hdr->lfe_on; } - hdr->channel_layout = ff_ac3_channel_layout_tab[hdr->channel_mode]; + hdr->channel_layout = avpriv_ac3_channel_layout_tab[hdr->channel_mode]; if (hdr->lfe_on) hdr->channel_layout |= AV_CH_LOW_FREQUENCY; diff --git a/libavcodec/ac3dec.c b/libavcodec/ac3dec.c index 1020eea0d6..fdc1d6830e 100644 --- a/libavcodec/ac3dec.c +++ b/libavcodec/ac3dec.c @@ -1383,7 +1383,7 @@ static int ac3_decode_frame(AVCodecContext * avctx, void *data, avctx->request_channels < s->channels) { s->out_channels = avctx->request_channels; s->output_mode = avctx->request_channels == 1 ? AC3_CHMODE_MONO : AC3_CHMODE_STEREO; - s->channel_layout = ff_ac3_channel_layout_tab[s->output_mode]; + s->channel_layout = avpriv_ac3_channel_layout_tab[s->output_mode]; } avctx->channels = s->out_channels; avctx->channel_layout = s->channel_layout; diff --git a/libavcodec/ac3tab.c b/libavcodec/ac3tab.c index 7df3d828fb..951a1014ce 100644 --- a/libavcodec/ac3tab.c +++ b/libavcodec/ac3tab.c @@ -84,7 +84,7 @@ const uint8_t ff_ac3_channels_tab[8] = { /** * Map audio coding mode (acmod) to channel layout mask. */ -const uint16_t ff_ac3_channel_layout_tab[8] = { +const uint16_t avpriv_ac3_channel_layout_tab[8] = { AV_CH_LAYOUT_STEREO, AV_CH_LAYOUT_MONO, AV_CH_LAYOUT_STEREO, diff --git a/libavcodec/ac3tab.h b/libavcodec/ac3tab.h index e5cd368bb7..8ed50520e6 100644 --- a/libavcodec/ac3tab.h +++ b/libavcodec/ac3tab.h @@ -33,7 +33,7 @@ extern const uint16_t ff_ac3_frame_size_tab[38][3]; extern const uint8_t ff_ac3_channels_tab[8]; -extern const uint16_t ff_ac3_channel_layout_tab[8]; +extern const uint16_t avpriv_ac3_channel_layout_tab[8]; extern const uint8_t ff_ac3_enc_channel_map[8][2][6]; extern const uint8_t ff_ac3_dec_channel_map[8][2][6]; extern const uint16_t ff_ac3_sample_rate_tab[3]; diff --git a/libavformat/mov.c b/libavformat/mov.c index d14ae7ee0d..089cdea558 100644 --- a/libavformat/mov.c +++ b/libavformat/mov.c @@ -25,11 +25,13 @@ //#define DEBUG //#define MOV_EXPORT_ALL_METADATA +#include "libavutil/audioconvert.h" #include "libavutil/intreadwrite.h" #include "libavutil/intfloat.h" #include "libavutil/mathematics.h" #include "libavutil/avstring.h" #include "libavutil/dict.h" +#include "libavcodec/ac3tab.h" #include "avformat.h" #include "internal.h" #include "avio_internal.h" @@ -548,6 +550,9 @@ static int mov_read_dac3(MOVContext *c, AVIOContext *pb, MOVAtom atom) acmod = (ac3info >> 11) & 0x7; lfeon = (ac3info >> 10) & 0x1; st->codec->channels = ((int[]){2,1,2,3,3,4,4,5})[acmod] + lfeon; + st->codec->channel_layout = avpriv_ac3_channel_layout_tab[acmod]; + if (lfeon) + st->codec->channel_layout |= AV_CH_LOW_FREQUENCY; st->codec->audio_service_type = bsmod; if (st->codec->channels > 1 && bsmod == 0x7) st->codec->audio_service_type = AV_AUDIO_SERVICE_TYPE_KARAOKE; From 9550c631963bbae78e9d33fa7f05f7138518dc8e Mon Sep 17 00:00:00 2001 From: Reinhard Tartler Date: Mon, 5 Mar 2012 20:40:37 +0100 Subject: [PATCH 111/154] Prepare for 0.8.1 Release --- RELEASE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE b/RELEASE index aec258df73..6f4eebdf6f 100644 --- a/RELEASE +++ b/RELEASE @@ -1 +1 @@ -0.8 +0.8.1 From 7b676935ee885d66d106436ec1acabdb8e335eca Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Mon, 5 Mar 2012 17:03:32 -0800 Subject: [PATCH 112/154] svq3: protect against negative quantizers. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 11b940a1a8e7e5d5b212935a3ce78aeda577f5f2) Signed-off-by: Reinhard Tartler --- libavcodec/svq3.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavcodec/svq3.c b/libavcodec/svq3.c index eeb8ed7051..49ca456bef 100644 --- a/libavcodec/svq3.c +++ b/libavcodec/svq3.c @@ -651,7 +651,7 @@ static int svq3_decode_mb(SVQ3Context *svq3, unsigned int mb_type) if (IS_INTRA16x16(mb_type) || (s->pict_type != AV_PICTURE_TYPE_I && s->adaptive_quant && cbp)) { s->qscale += svq3_get_se_golomb(&s->gb); - if (s->qscale > 31){ + if (s->qscale > 31u){ av_log(h->s.avctx, AV_LOG_ERROR, "qscale:%d\n", s->qscale); return -1; } From 9def2f200e55f625161b4040aa5ce2d86ae69ed3 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Tue, 6 Mar 2012 10:27:05 -0800 Subject: [PATCH 113/154] error_resilience: initialize s->block_index[]. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 6193ff68549ecbaf1a4d63a0e06964ec580ac620) Signed-off-by: Reinhard Tartler --- libavcodec/error_resilience.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/libavcodec/error_resilience.c b/libavcodec/error_resilience.c index bf59efad1a..96f49c8adb 100644 --- a/libavcodec/error_resilience.c +++ b/libavcodec/error_resilience.c @@ -419,9 +419,14 @@ static void guess_mv(MpegEncContext *s) if ((!(s->avctx->error_concealment&FF_EC_GUESS_MVS)) || num_avail <= mb_width / 2) { for (mb_y = 0; mb_y < s->mb_height; mb_y++) { + s->mb_x = 0; + s->mb_y = mb_y; + ff_init_block_index(s); for (mb_x = 0; mb_x < s->mb_width; mb_x++) { const int mb_xy = mb_x + mb_y * s->mb_stride; + ff_update_block_index(s); + if (IS_INTRA(s->current_picture.f.mb_type[mb_xy])) continue; if (!(s->error_status_table[mb_xy] & ER_MV_ERROR)) @@ -456,6 +461,9 @@ static void guess_mv(MpegEncContext *s) changed = 0; for (mb_y = 0; mb_y < s->mb_height; mb_y++) { + s->mb_x = 0; + s->mb_y = mb_y; + ff_init_block_index(s); for (mb_x = 0; mb_x < s->mb_width; mb_x++) { const int mb_xy = mb_x + mb_y * s->mb_stride; int mv_predictor[8][2] = { { 0 } }; @@ -467,6 +475,8 @@ static void guess_mv(MpegEncContext *s) const int mot_index = (mb_x + mb_y * mot_stride) * mot_step; int prev_x, prev_y, prev_ref; + ff_update_block_index(s); + if ((mb_x ^ mb_y ^ pass) & 1) continue; @@ -1072,11 +1082,16 @@ void ff_er_frame_end(MpegEncContext *s) /* handle inter blocks with damaged AC */ for (mb_y = 0; mb_y < s->mb_height; mb_y++) { + s->mb_x = 0; + s->mb_y = mb_y; + ff_init_block_index(s); for (mb_x = 0; mb_x < s->mb_width; mb_x++) { const int mb_xy = mb_x + mb_y * s->mb_stride; const int mb_type = s->current_picture.f.mb_type[mb_xy]; int dir = !s->last_picture.f.data[0]; + ff_update_block_index(s); + error = s->error_status_table[mb_xy]; if (IS_INTRA(mb_type)) @@ -1114,11 +1129,16 @@ void ff_er_frame_end(MpegEncContext *s) /* guess MVs */ if (s->pict_type == AV_PICTURE_TYPE_B) { for (mb_y = 0; mb_y < s->mb_height; mb_y++) { + s->mb_x = 0; + s->mb_y = mb_y; + ff_init_block_index(s); for (mb_x = 0; mb_x < s->mb_width; mb_x++) { int xy = mb_x * 2 + mb_y * 2 * s->b8_stride; const int mb_xy = mb_x + mb_y * s->mb_stride; const int mb_type = s->current_picture.f.mb_type[mb_xy]; + ff_update_block_index(s); + error = s->error_status_table[mb_xy]; if (IS_INTRA(mb_type)) From 7503861b424f7a1151bf4c4714bd46b4bdc5b496 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Mon, 5 Mar 2012 12:26:42 -0800 Subject: [PATCH 114/154] swscale: make filterPos 32bit. Fixes overflows for large image sizes. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 2254b559cbcfc0418135f09add37c0a5866b1981) Signed-off-by: Reinhard Tartler --- libswscale/ppc/swscale_altivec.c | 2 +- libswscale/swscale.c | 20 ++++++++++---------- libswscale/swscale_internal.h | 12 ++++++------ libswscale/utils.c | 4 ++-- libswscale/x86/scale.asm | 31 +++++++++++++++++-------------- libswscale/x86/swscale_mmx.c | 6 +++--- libswscale/x86/swscale_template.c | 4 ++-- 7 files changed, 41 insertions(+), 38 deletions(-) diff --git a/libswscale/ppc/swscale_altivec.c b/libswscale/ppc/swscale_altivec.c index 87059d9430..d7b58eea59 100644 --- a/libswscale/ppc/swscale_altivec.c +++ b/libswscale/ppc/swscale_altivec.c @@ -147,7 +147,7 @@ yuv2planeX_altivec(const int16_t *filter, int filterSize, static void hScale_altivec_real(SwsContext *c, int16_t *dst, int dstW, const uint8_t *src, const int16_t *filter, - const int16_t *filterPos, int filterSize) + const int32_t *filterPos, int filterSize) { register int i; DECLARE_ALIGNED(16, int, tempo)[4]; diff --git a/libswscale/swscale.c b/libswscale/swscale.c index 05ee8a4d91..1d0ea1b730 100644 --- a/libswscale/swscale.c +++ b/libswscale/swscale.c @@ -1874,7 +1874,7 @@ static void planar_rgb16be_to_uv(uint8_t *_dstU, uint8_t *_dstV, const uint8_t * static void hScale16To19_c(SwsContext *c, int16_t *_dst, int dstW, const uint8_t *_src, const int16_t *filter, - const int16_t *filterPos, int filterSize) + const int32_t *filterPos, int filterSize) { int i; int32_t *dst = (int32_t *) _dst; @@ -1897,7 +1897,7 @@ static void hScale16To19_c(SwsContext *c, int16_t *_dst, int dstW, const uint8_t static void hScale16To15_c(SwsContext *c, int16_t *dst, int dstW, const uint8_t *_src, const int16_t *filter, - const int16_t *filterPos, int filterSize) + const int32_t *filterPos, int filterSize) { int i; const uint16_t *src = (const uint16_t *) _src; @@ -1918,7 +1918,7 @@ static void hScale16To15_c(SwsContext *c, int16_t *dst, int dstW, const uint8_t // bilinear / bicubic scaling static void hScale8To15_c(SwsContext *c, int16_t *dst, int dstW, const uint8_t *src, - const int16_t *filter, const int16_t *filterPos, + const int16_t *filter, const int32_t *filterPos, int filterSize) { int i; @@ -1936,7 +1936,7 @@ static void hScale8To15_c(SwsContext *c, int16_t *dst, int dstW, const uint8_t * } static void hScale8To19_c(SwsContext *c, int16_t *_dst, int dstW, const uint8_t *src, - const int16_t *filter, const int16_t *filterPos, + const int16_t *filter, const int32_t *filterPos, int filterSize) { int i; @@ -2037,7 +2037,7 @@ static void hyscale_fast_c(SwsContext *c, int16_t *dst, int dstWidth, static av_always_inline void hyscale(SwsContext *c, int16_t *dst, int dstWidth, const uint8_t *src_in[4], int srcW, int xInc, const int16_t *hLumFilter, - const int16_t *hLumFilterPos, int hLumFilterSize, + const int32_t *hLumFilterPos, int hLumFilterSize, uint8_t *formatConvBuffer, uint32_t *pal, int isAlpha) { @@ -2081,7 +2081,7 @@ static void hcscale_fast_c(SwsContext *c, int16_t *dst1, int16_t *dst2, static av_always_inline void hcscale(SwsContext *c, int16_t *dst1, int16_t *dst2, int dstWidth, const uint8_t *src_in[4], int srcW, int xInc, const int16_t *hChrFilter, - const int16_t *hChrFilterPos, int hChrFilterSize, + const int32_t *hChrFilterPos, int hChrFilterSize, uint8_t *formatConvBuffer, uint32_t *pal) { const uint8_t *src1 = src_in[1], *src2 = src_in[2]; @@ -2369,10 +2369,10 @@ static int swScale(SwsContext *c, const uint8_t* src[], const int chrXInc= c->chrXInc; const enum PixelFormat dstFormat= c->dstFormat; const int flags= c->flags; - int16_t *vLumFilterPos= c->vLumFilterPos; - int16_t *vChrFilterPos= c->vChrFilterPos; - int16_t *hLumFilterPos= c->hLumFilterPos; - int16_t *hChrFilterPos= c->hChrFilterPos; + int32_t *vLumFilterPos= c->vLumFilterPos; + int32_t *vChrFilterPos= c->vChrFilterPos; + int32_t *hLumFilterPos= c->hLumFilterPos; + int32_t *hChrFilterPos= c->hChrFilterPos; int16_t *vLumFilter= c->vLumFilter; int16_t *vChrFilter= c->vChrFilter; int16_t *hLumFilter= c->hLumFilter; diff --git a/libswscale/swscale_internal.h b/libswscale/swscale_internal.h index 3436b92788..a71699538d 100644 --- a/libswscale/swscale_internal.h +++ b/libswscale/swscale_internal.h @@ -295,10 +295,10 @@ typedef struct SwsContext { int16_t *hChrFilter; ///< Array of horizontal filter coefficients for chroma planes. int16_t *vLumFilter; ///< Array of vertical filter coefficients for luma/alpha planes. int16_t *vChrFilter; ///< Array of vertical filter coefficients for chroma planes. - int16_t *hLumFilterPos; ///< Array of horizontal filter starting positions for each dst[i] for luma/alpha planes. - int16_t *hChrFilterPos; ///< Array of horizontal filter starting positions for each dst[i] for chroma planes. - int16_t *vLumFilterPos; ///< Array of vertical filter starting positions for each dst[i] for luma/alpha planes. - int16_t *vChrFilterPos; ///< Array of vertical filter starting positions for each dst[i] for chroma planes. + int32_t *hLumFilterPos; ///< Array of horizontal filter starting positions for each dst[i] for luma/alpha planes. + int32_t *hChrFilterPos; ///< Array of horizontal filter starting positions for each dst[i] for chroma planes. + int32_t *vLumFilterPos; ///< Array of vertical filter starting positions for each dst[i] for luma/alpha planes. + int32_t *vChrFilterPos; ///< Array of vertical filter starting positions for each dst[i] for chroma planes. int hLumFilterSize; ///< Horizontal filter size for luma/alpha pixels. int hChrFilterSize; ///< Horizontal filter size for chroma pixels. int vLumFilterSize; ///< Vertical filter size for luma/alpha pixels. @@ -508,10 +508,10 @@ typedef struct SwsContext { /** @{ */ void (*hyScale)(struct SwsContext *c, int16_t *dst, int dstW, const uint8_t *src, const int16_t *filter, - const int16_t *filterPos, int filterSize); + const int32_t *filterPos, int filterSize); void (*hcScale)(struct SwsContext *c, int16_t *dst, int dstW, const uint8_t *src, const int16_t *filter, - const int16_t *filterPos, int filterSize); + const int32_t *filterPos, int filterSize); /** @} */ /// Color range conversion function for luma plane if needed. diff --git a/libswscale/utils.c b/libswscale/utils.c index 51bc3842dc..f3a501230f 100644 --- a/libswscale/utils.c +++ b/libswscale/utils.c @@ -180,7 +180,7 @@ static double getSplineCoeff(double a, double b, double c, double d, double dist dist-1.0); } -static int initFilter(int16_t **outFilter, int16_t **filterPos, int *outFilterSize, int xInc, +static int initFilter(int16_t **outFilter, int32_t **filterPos, int *outFilterSize, int xInc, int srcW, int dstW, int filterAlign, int one, int flags, int cpu_flags, SwsVector *srcFilter, SwsVector *dstFilter, double param[2], int is_horizontal) { @@ -196,7 +196,7 @@ static int initFilter(int16_t **outFilter, int16_t **filterPos, int *outFilterSi emms_c(); //FIXME this should not be required but it IS (even for non-MMX versions) // NOTE: the +3 is for the MMX(+1)/SSE(+3) scaler which reads over the end - FF_ALLOC_OR_GOTO(NULL, *filterPos, (dstW+3)*sizeof(int16_t), fail); + FF_ALLOC_OR_GOTO(NULL, *filterPos, (dstW+3)*sizeof(**filterPos), fail); if (FFABS(xInc - 0x10000) <10) { // unscaled int i; diff --git a/libswscale/x86/scale.asm b/libswscale/x86/scale.asm index d35589419c..2b0b6dd230 100644 --- a/libswscale/x86/scale.asm +++ b/libswscale/x86/scale.asm @@ -38,7 +38,7 @@ SECTION .text ; (SwsContext *c, int{16,32}_t *dst, ; int dstW, const uint{8,16}_t *src, ; const int16_t *filter, -; const int16_t *filterPos, int filterSize); +; const int32_t *filterPos, int filterSize); ; ; Scale one horizontal line. Input is either 8-bits width or 16-bits width ; ($source_width can be either 8, 9, 10 or 16, difference is whether we have to @@ -53,6 +53,9 @@ SECTION .text cglobal hscale%1to%2_%4_%5, %6, 7, %7 %ifdef ARCH_X86_64 movsxd r2, r2d +%define mov32 movsxd +%else ; x86-32 +%define mov32 mov %endif ; x86-64 %if %2 == 19 %if mmsize == 8 ; mmx @@ -95,14 +98,14 @@ cglobal hscale%1to%2_%4_%5, %6, 7, %7 %else ; %2 == 19 lea r1, [r1+r2*(4>>r2shr)] %endif ; %2 == 15/19 - lea r5, [r5+r2*(2>>r2shr)] + lea r5, [r5+r2*(4>>r2shr)] neg r2 .loop: %if %3 == 4 ; filterSize == 4 scaling ; load 2x4 or 4x4 source pixels into m0/m1 - movsx r0, word [r5+r2*2+0] ; filterPos[0] - movsx r6, word [r5+r2*2+2] ; filterPos[1] + mov32 r0, dword [r5+r2*4+0] ; filterPos[0] + mov32 r6, dword [r5+r2*4+4] ; filterPos[1] movlh m0, [r3+r0*srcmul] ; src[filterPos[0] + {0,1,2,3}] %if mmsize == 8 movlh m1, [r3+r6*srcmul] ; src[filterPos[1] + {0,1,2,3}] @@ -112,8 +115,8 @@ cglobal hscale%1to%2_%4_%5, %6, 7, %7 %else ; %1 == 8 movd m4, [r3+r6*srcmul] ; src[filterPos[1] + {0,1,2,3}] %endif - movsx r0, word [r5+r2*2+4] ; filterPos[2] - movsx r6, word [r5+r2*2+6] ; filterPos[3] + mov32 r0, dword [r5+r2*4+8] ; filterPos[2] + mov32 r6, dword [r5+r2*4+12] ; filterPos[3] movlh m1, [r3+r0*srcmul] ; src[filterPos[2] + {0,1,2,3}] %if %1 > 8 movhps m1, [r3+r6*srcmul] ; src[filterPos[3] + {0,1,2,3}] @@ -156,8 +159,8 @@ cglobal hscale%1to%2_%4_%5, %6, 7, %7 %endif ; mmx/sse2/ssse3/sse4 %else ; %3 == 8, i.e. filterSize == 8 scaling ; load 2x8 or 4x8 source pixels into m0, m1, m4 and m5 - movsx r0, word [r5+r2*1+0] ; filterPos[0] - movsx r6, word [r5+r2*1+2] ; filterPos[1] + mov32 r0, dword [r5+r2*2+0] ; filterPos[0] + mov32 r6, dword [r5+r2*2+4] ; filterPos[1] movbh m0, [r3+ r0 *srcmul] ; src[filterPos[0] + {0,1,2,3,4,5,6,7}] %if mmsize == 8 movbh m1, [r3+(r0+4)*srcmul] ; src[filterPos[0] + {4,5,6,7}] @@ -165,8 +168,8 @@ cglobal hscale%1to%2_%4_%5, %6, 7, %7 movbh m5, [r3+(r6+4)*srcmul] ; src[filterPos[1] + {4,5,6,7}] %else ; mmsize == 16 movbh m1, [r3+ r6 *srcmul] ; src[filterPos[1] + {0,1,2,3,4,5,6,7}] - movsx r0, word [r5+r2*1+4] ; filterPos[2] - movsx r6, word [r5+r2*1+6] ; filterPos[3] + mov32 r0, dword [r5+r2*2+8] ; filterPos[2] + mov32 r6, dword [r5+r2*2+12] ; filterPos[3] movbh m4, [r3+ r0 *srcmul] ; src[filterPos[2] + {0,1,2,3,4,5,6,7}] movbh m5, [r3+ r6 *srcmul] ; src[filterPos[3] + {0,1,2,3,4,5,6,7}] %endif ; mmsize == 8/16 @@ -251,7 +254,7 @@ cglobal hscale%1to%2_%4_%5, %6, 7, %7 %define r1x r1 %define filter2 r6m %endif ; x86-32/64 - lea r5, [r5+r2*2] + lea r5, [r5+r2*4] %if %2 == 15 lea r1, [r1+r2*2] %else ; %2 == 19 @@ -261,8 +264,8 @@ cglobal hscale%1to%2_%4_%5, %6, 7, %7 neg r2 .loop: - movsx r0, word [r5+r2*2+0] ; filterPos[0] - movsx r1x, word [r5+r2*2+2] ; filterPos[1] + mov32 r0, dword [r5+r2*4+0] ; filterPos[0] + mov32 r1x, dword [r5+r2*4+4] ; filterPos[1] ; FIXME maybe do 4px/iteration on x86-64 (x86-32 wouldn't have enough regs)? pxor m4, m4 pxor m5, m5 @@ -293,7 +296,7 @@ cglobal hscale%1to%2_%4_%5, %6, 7, %7 jl .innerloop %ifidn %4, X4 - movsx r1x, word [r5+r2*2+2] ; filterPos[1] + mov32 r1x, dword [r5+r2*4+4] ; filterPos[1] movlh m0, [src_reg+r0 *srcmul] ; split last 4 srcpx of dstpx[0] sub r1x, r6 ; and first 4 srcpx of dstpx[1] %if %1 > 8 diff --git a/libswscale/x86/swscale_mmx.c b/libswscale/x86/swscale_mmx.c index 0853e12c41..f70d719f16 100644 --- a/libswscale/x86/swscale_mmx.c +++ b/libswscale/x86/swscale_mmx.c @@ -108,8 +108,8 @@ void updateMMXDitherTables(SwsContext *c, int dstY, int lumBufIndex, int chrBufI int16_t **alpPixBuf= c->alpPixBuf; const int vLumBufSize= c->vLumBufSize; const int vChrBufSize= c->vChrBufSize; - int16_t *vLumFilterPos= c->vLumFilterPos; - int16_t *vChrFilterPos= c->vChrFilterPos; + int32_t *vLumFilterPos= c->vLumFilterPos; + int32_t *vChrFilterPos= c->vChrFilterPos; int16_t *vLumFilter= c->vLumFilter; int16_t *vChrFilter= c->vChrFilter; int32_t *lumMmxFilter= c->lumMmxFilter; @@ -219,7 +219,7 @@ extern void ff_hscale ## from_bpc ## to ## to_bpc ## _ ## filter_n ## _ ## opt( SwsContext *c, int16_t *data, \ int dstW, const uint8_t *src, \ const int16_t *filter, \ - const int16_t *filterPos, int filterSize) + const int32_t *filterPos, int filterSize) #define SCALE_FUNCS(filter_n, opt) \ SCALE_FUNC(filter_n, 8, 15, opt); \ diff --git a/libswscale/x86/swscale_template.c b/libswscale/x86/swscale_template.c index 5db166b00a..40188d8019 100644 --- a/libswscale/x86/swscale_template.c +++ b/libswscale/x86/swscale_template.c @@ -1508,7 +1508,7 @@ static void RENAME(hyscale_fast)(SwsContext *c, int16_t *dst, int dstWidth, const uint8_t *src, int srcW, int xInc) { - int16_t *filterPos = c->hLumFilterPos; + int32_t *filterPos = c->hLumFilterPos; int16_t *filter = c->hLumFilter; void *mmx2FilterCode= c->lumMmx2FilterCode; int i; @@ -1604,7 +1604,7 @@ static void RENAME(hcscale_fast)(SwsContext *c, int16_t *dst1, int16_t *dst2, int dstWidth, const uint8_t *src1, const uint8_t *src2, int srcW, int xInc) { - int16_t *filterPos = c->hChrFilterPos; + int32_t *filterPos = c->hChrFilterPos; int16_t *filter = c->hChrFilter; void *mmx2FilterCode= c->chrMmx2FilterCode; int i; From 12247a13e018d64ba59012283d9b16374358985b Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Mon, 5 Mar 2012 16:01:19 -0800 Subject: [PATCH 115/154] Don't use ff_cropTbl[] for IDCT. Results of IDCT can by far outreach the range of ff_cropTbl[], leading to overreads and potentially crashes. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit c23acbaed40101c677dfcfbbfe0d2c230a8e8f44) Signed-off-by: Reinhard Tartler --- libavcodec/dsputil.c | 70 ++++++++++------------ libavcodec/h264idct_template.c | 32 +++++----- libavcodec/rv34dsp.c | 15 ++--- libavcodec/simple_idct.c | 19 +++--- libavcodec/simple_idct_template.c | 34 +++++------ libavcodec/svq3.c | 9 ++- libavcodec/vc1dsp.c | 97 +++++++++++++------------------ libavcodec/vp3dsp.c | 68 +++++++++++----------- libavcodec/vp8dsp.c | 18 +++--- 9 files changed, 161 insertions(+), 201 deletions(-) diff --git a/libavcodec/dsputil.c b/libavcodec/dsputil.c index 5c1039b028..66f1f933d0 100644 --- a/libavcodec/dsputil.c +++ b/libavcodec/dsputil.c @@ -367,18 +367,17 @@ void ff_put_pixels_clamped_c(const DCTELEM *block, uint8_t *restrict pixels, int line_size) { int i; - uint8_t *cm = ff_cropTbl + MAX_NEG_CROP; /* read the pixels */ for(i=0;i<8;i++) { - pixels[0] = cm[block[0]]; - pixels[1] = cm[block[1]]; - pixels[2] = cm[block[2]]; - pixels[3] = cm[block[3]]; - pixels[4] = cm[block[4]]; - pixels[5] = cm[block[5]]; - pixels[6] = cm[block[6]]; - pixels[7] = cm[block[7]]; + pixels[0] = av_clip_uint8(block[0]); + pixels[1] = av_clip_uint8(block[1]); + pixels[2] = av_clip_uint8(block[2]); + pixels[3] = av_clip_uint8(block[3]); + pixels[4] = av_clip_uint8(block[4]); + pixels[5] = av_clip_uint8(block[5]); + pixels[6] = av_clip_uint8(block[6]); + pixels[7] = av_clip_uint8(block[7]); pixels += line_size; block += 8; @@ -389,14 +388,13 @@ static void put_pixels_clamped4_c(const DCTELEM *block, uint8_t *restrict pixels int line_size) { int i; - uint8_t *cm = ff_cropTbl + MAX_NEG_CROP; /* read the pixels */ for(i=0;i<4;i++) { - pixels[0] = cm[block[0]]; - pixels[1] = cm[block[1]]; - pixels[2] = cm[block[2]]; - pixels[3] = cm[block[3]]; + pixels[0] = av_clip_uint8(block[0]); + pixels[1] = av_clip_uint8(block[1]); + pixels[2] = av_clip_uint8(block[2]); + pixels[3] = av_clip_uint8(block[3]); pixels += line_size; block += 8; @@ -407,12 +405,11 @@ static void put_pixels_clamped2_c(const DCTELEM *block, uint8_t *restrict pixels int line_size) { int i; - uint8_t *cm = ff_cropTbl + MAX_NEG_CROP; /* read the pixels */ for(i=0;i<2;i++) { - pixels[0] = cm[block[0]]; - pixels[1] = cm[block[1]]; + pixels[0] = av_clip_uint8(block[0]); + pixels[1] = av_clip_uint8(block[1]); pixels += line_size; block += 8; @@ -444,18 +441,17 @@ void ff_add_pixels_clamped_c(const DCTELEM *block, uint8_t *restrict pixels, int line_size) { int i; - uint8_t *cm = ff_cropTbl + MAX_NEG_CROP; /* read the pixels */ for(i=0;i<8;i++) { - pixels[0] = cm[pixels[0] + block[0]]; - pixels[1] = cm[pixels[1] + block[1]]; - pixels[2] = cm[pixels[2] + block[2]]; - pixels[3] = cm[pixels[3] + block[3]]; - pixels[4] = cm[pixels[4] + block[4]]; - pixels[5] = cm[pixels[5] + block[5]]; - pixels[6] = cm[pixels[6] + block[6]]; - pixels[7] = cm[pixels[7] + block[7]]; + pixels[0] = av_clip_uint8(pixels[0] + block[0]); + pixels[1] = av_clip_uint8(pixels[1] + block[1]); + pixels[2] = av_clip_uint8(pixels[2] + block[2]); + pixels[3] = av_clip_uint8(pixels[3] + block[3]); + pixels[4] = av_clip_uint8(pixels[4] + block[4]); + pixels[5] = av_clip_uint8(pixels[5] + block[5]); + pixels[6] = av_clip_uint8(pixels[6] + block[6]); + pixels[7] = av_clip_uint8(pixels[7] + block[7]); pixels += line_size; block += 8; } @@ -465,14 +461,13 @@ static void add_pixels_clamped4_c(const DCTELEM *block, uint8_t *restrict pixels int line_size) { int i; - uint8_t *cm = ff_cropTbl + MAX_NEG_CROP; /* read the pixels */ for(i=0;i<4;i++) { - pixels[0] = cm[pixels[0] + block[0]]; - pixels[1] = cm[pixels[1] + block[1]]; - pixels[2] = cm[pixels[2] + block[2]]; - pixels[3] = cm[pixels[3] + block[3]]; + pixels[0] = av_clip_uint8(pixels[0] + block[0]); + pixels[1] = av_clip_uint8(pixels[1] + block[1]); + pixels[2] = av_clip_uint8(pixels[2] + block[2]); + pixels[3] = av_clip_uint8(pixels[3] + block[3]); pixels += line_size; block += 8; } @@ -482,12 +477,11 @@ static void add_pixels_clamped2_c(const DCTELEM *block, uint8_t *restrict pixels int line_size) { int i; - uint8_t *cm = ff_cropTbl + MAX_NEG_CROP; /* read the pixels */ for(i=0;i<2;i++) { - pixels[0] = cm[pixels[0] + block[0]]; - pixels[1] = cm[pixels[1] + block[1]]; + pixels[0] = av_clip_uint8(pixels[0] + block[0]); + pixels[1] = av_clip_uint8(pixels[1] + block[1]); pixels += line_size; block += 8; } @@ -2745,15 +2739,11 @@ static void ff_jref_idct2_add(uint8_t *dest, int line_size, DCTELEM *block) static void ff_jref_idct1_put(uint8_t *dest, int line_size, DCTELEM *block) { - uint8_t *cm = ff_cropTbl + MAX_NEG_CROP; - - dest[0] = cm[(block[0] + 4)>>3]; + dest[0] = av_clip_uint8((block[0] + 4)>>3); } static void ff_jref_idct1_add(uint8_t *dest, int line_size, DCTELEM *block) { - uint8_t *cm = ff_cropTbl + MAX_NEG_CROP; - - dest[0] = cm[dest[0] + ((block[0] + 4)>>3)]; + dest[0] = av_clip_uint8(dest[0] + ((block[0] + 4)>>3)); } static void just_return(void *mem av_unused, int stride av_unused, int h av_unused) { return; } diff --git a/libavcodec/h264idct_template.c b/libavcodec/h264idct_template.c index eba850ac6f..e476f89a6f 100644 --- a/libavcodec/h264idct_template.c +++ b/libavcodec/h264idct_template.c @@ -49,7 +49,6 @@ static const uint8_t scan8[16*3]={ void FUNCC(ff_h264_idct_add)(uint8_t *_dst, DCTELEM *_block, int stride) { int i; - INIT_CLIP pixel *dst = (pixel*)_dst; dctcoef *block = (dctcoef*)_block; stride /= sizeof(pixel); @@ -74,16 +73,15 @@ void FUNCC(ff_h264_idct_add)(uint8_t *_dst, DCTELEM *_block, int stride) const int z2= (block[1 + 4*i]>>1) - block[3 + 4*i]; const int z3= block[1 + 4*i] + (block[3 + 4*i]>>1); - dst[i + 0*stride]= CLIP(dst[i + 0*stride] + ((z0 + z3) >> 6)); - dst[i + 1*stride]= CLIP(dst[i + 1*stride] + ((z1 + z2) >> 6)); - dst[i + 2*stride]= CLIP(dst[i + 2*stride] + ((z1 - z2) >> 6)); - dst[i + 3*stride]= CLIP(dst[i + 3*stride] + ((z0 - z3) >> 6)); + dst[i + 0*stride]= av_clip_pixel(dst[i + 0*stride] + ((z0 + z3) >> 6)); + dst[i + 1*stride]= av_clip_pixel(dst[i + 1*stride] + ((z1 + z2) >> 6)); + dst[i + 2*stride]= av_clip_pixel(dst[i + 2*stride] + ((z1 - z2) >> 6)); + dst[i + 3*stride]= av_clip_pixel(dst[i + 3*stride] + ((z0 - z3) >> 6)); } } void FUNCC(ff_h264_idct8_add)(uint8_t *_dst, DCTELEM *_block, int stride){ int i; - INIT_CLIP pixel *dst = (pixel*)_dst; dctcoef *block = (dctcoef*)_block; stride /= sizeof(pixel); @@ -143,14 +141,14 @@ void FUNCC(ff_h264_idct8_add)(uint8_t *_dst, DCTELEM *_block, int stride){ const int b5 = (a3>>2) - a5; const int b7 = a7 - (a1>>2); - dst[i + 0*stride] = CLIP( dst[i + 0*stride] + ((b0 + b7) >> 6) ); - dst[i + 1*stride] = CLIP( dst[i + 1*stride] + ((b2 + b5) >> 6) ); - dst[i + 2*stride] = CLIP( dst[i + 2*stride] + ((b4 + b3) >> 6) ); - dst[i + 3*stride] = CLIP( dst[i + 3*stride] + ((b6 + b1) >> 6) ); - dst[i + 4*stride] = CLIP( dst[i + 4*stride] + ((b6 - b1) >> 6) ); - dst[i + 5*stride] = CLIP( dst[i + 5*stride] + ((b4 - b3) >> 6) ); - dst[i + 6*stride] = CLIP( dst[i + 6*stride] + ((b2 - b5) >> 6) ); - dst[i + 7*stride] = CLIP( dst[i + 7*stride] + ((b0 - b7) >> 6) ); + dst[i + 0*stride] = av_clip_pixel( dst[i + 0*stride] + ((b0 + b7) >> 6) ); + dst[i + 1*stride] = av_clip_pixel( dst[i + 1*stride] + ((b2 + b5) >> 6) ); + dst[i + 2*stride] = av_clip_pixel( dst[i + 2*stride] + ((b4 + b3) >> 6) ); + dst[i + 3*stride] = av_clip_pixel( dst[i + 3*stride] + ((b6 + b1) >> 6) ); + dst[i + 4*stride] = av_clip_pixel( dst[i + 4*stride] + ((b6 - b1) >> 6) ); + dst[i + 5*stride] = av_clip_pixel( dst[i + 5*stride] + ((b4 - b3) >> 6) ); + dst[i + 6*stride] = av_clip_pixel( dst[i + 6*stride] + ((b2 - b5) >> 6) ); + dst[i + 7*stride] = av_clip_pixel( dst[i + 7*stride] + ((b0 - b7) >> 6) ); } } @@ -158,13 +156,12 @@ void FUNCC(ff_h264_idct8_add)(uint8_t *_dst, DCTELEM *_block, int stride){ void FUNCC(ff_h264_idct_dc_add)(uint8_t *_dst, DCTELEM *block, int stride){ int i, j; int dc = (((dctcoef*)block)[0] + 32) >> 6; - INIT_CLIP pixel *dst = (pixel*)_dst; stride /= sizeof(pixel); for( j = 0; j < 4; j++ ) { for( i = 0; i < 4; i++ ) - dst[i] = CLIP( dst[i] + dc ); + dst[i] = av_clip_pixel( dst[i] + dc ); dst += stride; } } @@ -172,13 +169,12 @@ void FUNCC(ff_h264_idct_dc_add)(uint8_t *_dst, DCTELEM *block, int stride){ void FUNCC(ff_h264_idct8_dc_add)(uint8_t *_dst, DCTELEM *block, int stride){ int i, j; int dc = (((dctcoef*)block)[0] + 32) >> 6; - INIT_CLIP pixel *dst = (pixel*)_dst; stride /= sizeof(pixel); for( j = 0; j < 8; j++ ) { for( i = 0; i < 8; i++ ) - dst[i] = CLIP( dst[i] + dc ); + dst[i] = av_clip_pixel( dst[i] + dc ); dst += stride; } } diff --git a/libavcodec/rv34dsp.c b/libavcodec/rv34dsp.c index e2251773af..919703d1e3 100644 --- a/libavcodec/rv34dsp.c +++ b/libavcodec/rv34dsp.c @@ -55,7 +55,6 @@ static av_always_inline void rv34_row_transform(int temp[16], DCTELEM *block) */ static void rv34_idct_add_c(uint8_t *dst, int stride, DCTELEM *block){ int temp[16]; - uint8_t *cm = ff_cropTbl + MAX_NEG_CROP; int i; rv34_row_transform(temp, block); @@ -67,10 +66,10 @@ static void rv34_idct_add_c(uint8_t *dst, int stride, DCTELEM *block){ const int z2 = 7* temp[4*1+i] - 17*temp[4*3+i]; const int z3 = 17* temp[4*1+i] + 7*temp[4*3+i]; - dst[0] = cm[ dst[0] + ( (z0 + z3) >> 10 ) ]; - dst[1] = cm[ dst[1] + ( (z1 + z2) >> 10 ) ]; - dst[2] = cm[ dst[2] + ( (z1 - z2) >> 10 ) ]; - dst[3] = cm[ dst[3] + ( (z0 - z3) >> 10 ) ]; + dst[0] = av_clip_uint8( dst[0] + ( (z0 + z3) >> 10 ) ); + dst[1] = av_clip_uint8( dst[1] + ( (z1 + z2) >> 10 ) ); + dst[2] = av_clip_uint8( dst[2] + ( (z1 - z2) >> 10 ) ); + dst[3] = av_clip_uint8( dst[3] + ( (z0 - z3) >> 10 ) ); dst += stride; } @@ -103,15 +102,13 @@ static void rv34_inv_transform_noround_c(DCTELEM *block){ static void rv34_idct_dc_add_c(uint8_t *dst, int stride, int dc) { - const uint8_t *cm = ff_cropTbl + MAX_NEG_CROP; int i, j; - cm += (13*13*dc + 0x200) >> 10; - + dc = (13*13*dc + 0x200) >> 10; for (i = 0; i < 4; i++) { for (j = 0; j < 4; j++) - dst[j] = cm[ dst[j] ]; + dst[j] = av_clip_uint8( dst[j] + dc ); dst += stride; } diff --git a/libavcodec/simple_idct.c b/libavcodec/simple_idct.c index 0c75261079..5812a87705 100644 --- a/libavcodec/simple_idct.c +++ b/libavcodec/simple_idct.c @@ -53,7 +53,6 @@ static inline void idct4col_put(uint8_t *dest, int line_size, const DCTELEM *col) { int c0, c1, c2, c3, a0, a1, a2, a3; - const uint8_t *cm = ff_cropTbl + MAX_NEG_CROP; a0 = col[8*0]; a1 = col[8*2]; @@ -63,13 +62,13 @@ static inline void idct4col_put(uint8_t *dest, int line_size, const DCTELEM *col c2 = ((a0 - a2) << (CN_SHIFT - 1)) + (1 << (C_SHIFT - 1)); c1 = a1 * C1 + a3 * C2; c3 = a1 * C2 - a3 * C1; - dest[0] = cm[(c0 + c1) >> C_SHIFT]; + dest[0] = av_clip_uint8((c0 + c1) >> C_SHIFT); dest += line_size; - dest[0] = cm[(c2 + c3) >> C_SHIFT]; + dest[0] = av_clip_uint8((c2 + c3) >> C_SHIFT); dest += line_size; - dest[0] = cm[(c2 - c3) >> C_SHIFT]; + dest[0] = av_clip_uint8((c2 - c3) >> C_SHIFT); dest += line_size; - dest[0] = cm[(c0 - c1) >> C_SHIFT]; + dest[0] = av_clip_uint8((c0 - c1) >> C_SHIFT); } #define BF(k) \ @@ -133,7 +132,6 @@ void ff_simple_idct248_put(uint8_t *dest, int line_size, DCTELEM *block) static inline void idct4col_add(uint8_t *dest, int line_size, const DCTELEM *col) { int c0, c1, c2, c3, a0, a1, a2, a3; - const uint8_t *cm = ff_cropTbl + MAX_NEG_CROP; a0 = col[8*0]; a1 = col[8*1]; @@ -143,13 +141,13 @@ static inline void idct4col_add(uint8_t *dest, int line_size, const DCTELEM *col c2 = (a0 - a2)*C3 + (1 << (C_SHIFT - 1)); c1 = a1 * C1 + a3 * C2; c3 = a1 * C2 - a3 * C1; - dest[0] = cm[dest[0] + ((c0 + c1) >> C_SHIFT)]; + dest[0] = av_clip_uint8(dest[0] + ((c0 + c1) >> C_SHIFT)); dest += line_size; - dest[0] = cm[dest[0] + ((c2 + c3) >> C_SHIFT)]; + dest[0] = av_clip_uint8(dest[0] + ((c2 + c3) >> C_SHIFT)); dest += line_size; - dest[0] = cm[dest[0] + ((c2 - c3) >> C_SHIFT)]; + dest[0] = av_clip_uint8(dest[0] + ((c2 - c3) >> C_SHIFT)); dest += line_size; - dest[0] = cm[dest[0] + ((c0 - c1) >> C_SHIFT)]; + dest[0] = av_clip_uint8(dest[0] + ((c0 - c1) >> C_SHIFT)); } #define RN_SHIFT 15 @@ -161,7 +159,6 @@ static inline void idct4col_add(uint8_t *dest, int line_size, const DCTELEM *col static inline void idct4row(DCTELEM *row) { int c0, c1, c2, c3, a0, a1, a2, a3; - //const uint8_t *cm = ff_cropTbl + MAX_NEG_CROP; a0 = row[0]; a1 = row[1]; diff --git a/libavcodec/simple_idct_template.c b/libavcodec/simple_idct_template.c index fdec3aab2b..3c855e3825 100644 --- a/libavcodec/simple_idct_template.c +++ b/libavcodec/simple_idct_template.c @@ -224,50 +224,48 @@ static inline void FUNC(idctSparseColPut)(pixel *dest, int line_size, DCTELEM *col) { int a0, a1, a2, a3, b0, b1, b2, b3; - INIT_CLIP; IDCT_COLS; - dest[0] = CLIP((a0 + b0) >> COL_SHIFT); + dest[0] = av_clip_pixel((a0 + b0) >> COL_SHIFT); dest += line_size; - dest[0] = CLIP((a1 + b1) >> COL_SHIFT); + dest[0] = av_clip_pixel((a1 + b1) >> COL_SHIFT); dest += line_size; - dest[0] = CLIP((a2 + b2) >> COL_SHIFT); + dest[0] = av_clip_pixel((a2 + b2) >> COL_SHIFT); dest += line_size; - dest[0] = CLIP((a3 + b3) >> COL_SHIFT); + dest[0] = av_clip_pixel((a3 + b3) >> COL_SHIFT); dest += line_size; - dest[0] = CLIP((a3 - b3) >> COL_SHIFT); + dest[0] = av_clip_pixel((a3 - b3) >> COL_SHIFT); dest += line_size; - dest[0] = CLIP((a2 - b2) >> COL_SHIFT); + dest[0] = av_clip_pixel((a2 - b2) >> COL_SHIFT); dest += line_size; - dest[0] = CLIP((a1 - b1) >> COL_SHIFT); + dest[0] = av_clip_pixel((a1 - b1) >> COL_SHIFT); dest += line_size; - dest[0] = CLIP((a0 - b0) >> COL_SHIFT); + dest[0] = av_clip_pixel((a0 - b0) >> COL_SHIFT); } static inline void FUNC(idctSparseColAdd)(pixel *dest, int line_size, DCTELEM *col) { int a0, a1, a2, a3, b0, b1, b2, b3; - INIT_CLIP; IDCT_COLS; - dest[0] = CLIP(dest[0] + ((a0 + b0) >> COL_SHIFT)); + dest[0] = av_clip_pixel(dest[0] + ((a0 + b0) >> COL_SHIFT)); dest += line_size; - dest[0] = CLIP(dest[0] + ((a1 + b1) >> COL_SHIFT)); + dest[0] = av_clip_pixel(dest[0] + ((a1 + b1) >> COL_SHIFT)); dest += line_size; - dest[0] = CLIP(dest[0] + ((a2 + b2) >> COL_SHIFT)); + dest[0] = av_clip_pixel(dest[0] + ((a2 + b2) >> COL_SHIFT)); dest += line_size; - dest[0] = CLIP(dest[0] + ((a3 + b3) >> COL_SHIFT)); + dest[0] = av_clip_pixel(dest[0] + ((a3 + b3) >> COL_SHIFT)); dest += line_size; - dest[0] = CLIP(dest[0] + ((a3 - b3) >> COL_SHIFT)); + dest[0] = av_clip_pixel(dest[0] + ((a3 - b3) >> COL_SHIFT)); dest += line_size; - dest[0] = CLIP(dest[0] + ((a2 - b2) >> COL_SHIFT)); + dest[0] = av_clip_pixel(dest[0] + ((a2 - b2) >> COL_SHIFT)); dest += line_size; - dest[0] = CLIP(dest[0] + ((a1 - b1) >> COL_SHIFT)); + dest[0] = av_clip_pixel(dest[0] + ((a1 - b1) >> COL_SHIFT)); dest += line_size; - dest[0] = CLIP(dest[0] + ((a0 - b0) >> COL_SHIFT)); + dest[0] = av_clip_pixel(dest[0] + ((a0 - b0) >> COL_SHIFT)); } static inline void FUNC(idctSparseCol)(DCTELEM *col) diff --git a/libavcodec/svq3.c b/libavcodec/svq3.c index 49ca456bef..3be71a0812 100644 --- a/libavcodec/svq3.c +++ b/libavcodec/svq3.c @@ -173,7 +173,6 @@ void ff_svq3_add_idct_c(uint8_t *dst, DCTELEM *block, int stride, int qp, { const int qmul = svq3_dequant_coeff[qp]; int i; - uint8_t *cm = ff_cropTbl + MAX_NEG_CROP; if (dc) { dc = 13*13*((dc == 1) ? 1538*block[0] : ((qmul*(block[0] >> 3)) / 2)); @@ -199,10 +198,10 @@ void ff_svq3_add_idct_c(uint8_t *dst, DCTELEM *block, int stride, int qp, const int z3 = 17* block[i + 4*1] + 7*block[i + 4*3]; const int rr = (dc + 0x80000); - dst[i + stride*0] = cm[ dst[i + stride*0] + (((z0 + z3)*qmul + rr) >> 20) ]; - dst[i + stride*1] = cm[ dst[i + stride*1] + (((z1 + z2)*qmul + rr) >> 20) ]; - dst[i + stride*2] = cm[ dst[i + stride*2] + (((z1 - z2)*qmul + rr) >> 20) ]; - dst[i + stride*3] = cm[ dst[i + stride*3] + (((z0 - z3)*qmul + rr) >> 20) ]; + dst[i + stride*0] = av_clip_uint8( dst[i + stride*0] + (((z0 + z3)*qmul + rr) >> 20) ); + dst[i + stride*1] = av_clip_uint8( dst[i + stride*1] + (((z1 + z2)*qmul + rr) >> 20) ); + dst[i + stride*2] = av_clip_uint8( dst[i + stride*2] + (((z1 - z2)*qmul + rr) >> 20) ); + dst[i + stride*3] = av_clip_uint8( dst[i + stride*3] + (((z0 - z3)*qmul + rr) >> 20) ); } } diff --git a/libavcodec/vc1dsp.c b/libavcodec/vc1dsp.c index 9bd107cdd9..b40824b86a 100644 --- a/libavcodec/vc1dsp.c +++ b/libavcodec/vc1dsp.c @@ -139,8 +139,6 @@ static void vc1_h_s_overlap_c(DCTELEM *left, DCTELEM *right) * @see 8.6 */ static av_always_inline int vc1_filter_line(uint8_t* src, int stride, int pq){ - uint8_t *cm = ff_cropTbl + MAX_NEG_CROP; - int a0 = (2*(src[-2*stride] - src[ 1*stride]) - 5*(src[-1*stride] - src[ 0*stride]) + 4) >> 3; int a0_sign = a0 >> 31; /* Store sign */ a0 = (a0 ^ a0_sign) - a0_sign; /* a0 = FFABS(a0); */ @@ -163,8 +161,8 @@ static av_always_inline int vc1_filter_line(uint8_t* src, int stride, int pq){ else{ d = FFMIN(d, clip); d = (d ^ d_sign) - d_sign; /* Restore sign */ - src[-1*stride] = cm[src[-1*stride] - d]; - src[ 0*stride] = cm[src[ 0*stride] + d]; + src[-1*stride] = av_clip_uint8(src[-1*stride] - d); + src[ 0*stride] = av_clip_uint8(src[ 0*stride] + d); } return 1; } @@ -234,19 +232,17 @@ static void vc1_inv_trans_8x8_dc_c(uint8_t *dest, int linesize, DCTELEM *block) { int i; int dc = block[0]; - const uint8_t *cm; dc = (3 * dc + 1) >> 1; dc = (3 * dc + 16) >> 5; - cm = ff_cropTbl + MAX_NEG_CROP + dc; for(i = 0; i < 8; i++){ - dest[0] = cm[dest[0]]; - dest[1] = cm[dest[1]]; - dest[2] = cm[dest[2]]; - dest[3] = cm[dest[3]]; - dest[4] = cm[dest[4]]; - dest[5] = cm[dest[5]]; - dest[6] = cm[dest[6]]; - dest[7] = cm[dest[7]]; + dest[0] = av_clip_uint8(dest[0] + dc); + dest[1] = av_clip_uint8(dest[1] + dc); + dest[2] = av_clip_uint8(dest[2] + dc); + dest[3] = av_clip_uint8(dest[3] + dc); + dest[4] = av_clip_uint8(dest[4] + dc); + dest[5] = av_clip_uint8(dest[5] + dc); + dest[6] = av_clip_uint8(dest[6] + dc); + dest[7] = av_clip_uint8(dest[7] + dc); dest += linesize; } } @@ -326,19 +322,17 @@ static void vc1_inv_trans_8x4_dc_c(uint8_t *dest, int linesize, DCTELEM *block) { int i; int dc = block[0]; - const uint8_t *cm; dc = ( 3 * dc + 1) >> 1; dc = (17 * dc + 64) >> 7; - cm = ff_cropTbl + MAX_NEG_CROP + dc; for(i = 0; i < 4; i++){ - dest[0] = cm[dest[0]]; - dest[1] = cm[dest[1]]; - dest[2] = cm[dest[2]]; - dest[3] = cm[dest[3]]; - dest[4] = cm[dest[4]]; - dest[5] = cm[dest[5]]; - dest[6] = cm[dest[6]]; - dest[7] = cm[dest[7]]; + dest[0] = av_clip_uint8(dest[0] + dc); + dest[1] = av_clip_uint8(dest[1] + dc); + dest[2] = av_clip_uint8(dest[2] + dc); + dest[3] = av_clip_uint8(dest[3] + dc); + dest[4] = av_clip_uint8(dest[4] + dc); + dest[5] = av_clip_uint8(dest[5] + dc); + dest[6] = av_clip_uint8(dest[6] + dc); + dest[7] = av_clip_uint8(dest[7] + dc); dest += linesize; } } @@ -348,7 +342,6 @@ static void vc1_inv_trans_8x4_c(uint8_t *dest, int linesize, DCTELEM *block) int i; register int t1,t2,t3,t4,t5,t6,t7,t8; DCTELEM *src, *dst; - const uint8_t *cm = ff_cropTbl + MAX_NEG_CROP; src = block; dst = block; @@ -388,10 +381,10 @@ static void vc1_inv_trans_8x4_c(uint8_t *dest, int linesize, DCTELEM *block) t3 = 22 * src[ 8] + 10 * src[24]; t4 = 22 * src[24] - 10 * src[ 8]; - dest[0*linesize] = cm[dest[0*linesize] + ((t1 + t3) >> 7)]; - dest[1*linesize] = cm[dest[1*linesize] + ((t2 - t4) >> 7)]; - dest[2*linesize] = cm[dest[2*linesize] + ((t2 + t4) >> 7)]; - dest[3*linesize] = cm[dest[3*linesize] + ((t1 - t3) >> 7)]; + dest[0*linesize] = av_clip_uint8(dest[0*linesize] + ((t1 + t3) >> 7)); + dest[1*linesize] = av_clip_uint8(dest[1*linesize] + ((t2 - t4) >> 7)); + dest[2*linesize] = av_clip_uint8(dest[2*linesize] + ((t2 + t4) >> 7)); + dest[3*linesize] = av_clip_uint8(dest[3*linesize] + ((t1 - t3) >> 7)); src ++; dest++; @@ -404,15 +397,13 @@ static void vc1_inv_trans_4x8_dc_c(uint8_t *dest, int linesize, DCTELEM *block) { int i; int dc = block[0]; - const uint8_t *cm; dc = (17 * dc + 4) >> 3; dc = (12 * dc + 64) >> 7; - cm = ff_cropTbl + MAX_NEG_CROP + dc; for(i = 0; i < 8; i++){ - dest[0] = cm[dest[0]]; - dest[1] = cm[dest[1]]; - dest[2] = cm[dest[2]]; - dest[3] = cm[dest[3]]; + dest[0] = av_clip_uint8(dest[0] + dc); + dest[1] = av_clip_uint8(dest[1] + dc); + dest[2] = av_clip_uint8(dest[2] + dc); + dest[3] = av_clip_uint8(dest[3] + dc); dest += linesize; } } @@ -422,7 +413,6 @@ static void vc1_inv_trans_4x8_c(uint8_t *dest, int linesize, DCTELEM *block) int i; register int t1,t2,t3,t4,t5,t6,t7,t8; DCTELEM *src, *dst; - const uint8_t *cm = ff_cropTbl + MAX_NEG_CROP; src = block; dst = block; @@ -458,14 +448,14 @@ static void vc1_inv_trans_4x8_c(uint8_t *dest, int linesize, DCTELEM *block) t3 = 9 * src[ 8] - 16 * src[24] + 4 * src[40] + 15 * src[56]; t4 = 4 * src[ 8] - 9 * src[24] + 15 * src[40] - 16 * src[56]; - dest[0*linesize] = cm[dest[0*linesize] + ((t5 + t1) >> 7)]; - dest[1*linesize] = cm[dest[1*linesize] + ((t6 + t2) >> 7)]; - dest[2*linesize] = cm[dest[2*linesize] + ((t7 + t3) >> 7)]; - dest[3*linesize] = cm[dest[3*linesize] + ((t8 + t4) >> 7)]; - dest[4*linesize] = cm[dest[4*linesize] + ((t8 - t4 + 1) >> 7)]; - dest[5*linesize] = cm[dest[5*linesize] + ((t7 - t3 + 1) >> 7)]; - dest[6*linesize] = cm[dest[6*linesize] + ((t6 - t2 + 1) >> 7)]; - dest[7*linesize] = cm[dest[7*linesize] + ((t5 - t1 + 1) >> 7)]; + dest[0*linesize] = av_clip_uint8(dest[0*linesize] + ((t5 + t1) >> 7)); + dest[1*linesize] = av_clip_uint8(dest[1*linesize] + ((t6 + t2) >> 7)); + dest[2*linesize] = av_clip_uint8(dest[2*linesize] + ((t7 + t3) >> 7)); + dest[3*linesize] = av_clip_uint8(dest[3*linesize] + ((t8 + t4) >> 7)); + dest[4*linesize] = av_clip_uint8(dest[4*linesize] + ((t8 - t4 + 1) >> 7)); + dest[5*linesize] = av_clip_uint8(dest[5*linesize] + ((t7 - t3 + 1) >> 7)); + dest[6*linesize] = av_clip_uint8(dest[6*linesize] + ((t6 - t2 + 1) >> 7)); + dest[7*linesize] = av_clip_uint8(dest[7*linesize] + ((t5 - t1 + 1) >> 7)); src ++; dest++; @@ -478,15 +468,13 @@ static void vc1_inv_trans_4x4_dc_c(uint8_t *dest, int linesize, DCTELEM *block) { int i; int dc = block[0]; - const uint8_t *cm; dc = (17 * dc + 4) >> 3; dc = (17 * dc + 64) >> 7; - cm = ff_cropTbl + MAX_NEG_CROP + dc; for(i = 0; i < 4; i++){ - dest[0] = cm[dest[0]]; - dest[1] = cm[dest[1]]; - dest[2] = cm[dest[2]]; - dest[3] = cm[dest[3]]; + dest[0] = av_clip_uint8(dest[0] + dc); + dest[1] = av_clip_uint8(dest[1] + dc); + dest[2] = av_clip_uint8(dest[2] + dc); + dest[3] = av_clip_uint8(dest[3] + dc); dest += linesize; } } @@ -496,7 +484,6 @@ static void vc1_inv_trans_4x4_c(uint8_t *dest, int linesize, DCTELEM *block) int i; register int t1,t2,t3,t4; DCTELEM *src, *dst; - const uint8_t *cm = ff_cropTbl + MAX_NEG_CROP; src = block; dst = block; @@ -522,10 +509,10 @@ static void vc1_inv_trans_4x4_c(uint8_t *dest, int linesize, DCTELEM *block) t3 = 22 * src[ 8] + 10 * src[24]; t4 = 22 * src[24] - 10 * src[ 8]; - dest[0*linesize] = cm[dest[0*linesize] + ((t1 + t3) >> 7)]; - dest[1*linesize] = cm[dest[1*linesize] + ((t2 - t4) >> 7)]; - dest[2*linesize] = cm[dest[2*linesize] + ((t2 + t4) >> 7)]; - dest[3*linesize] = cm[dest[3*linesize] + ((t1 - t3) >> 7)]; + dest[0*linesize] = av_clip_uint8(dest[0*linesize] + ((t1 + t3) >> 7)); + dest[1*linesize] = av_clip_uint8(dest[1*linesize] + ((t2 - t4) >> 7)); + dest[2*linesize] = av_clip_uint8(dest[2*linesize] + ((t2 + t4) >> 7)); + dest[3*linesize] = av_clip_uint8(dest[3*linesize] + ((t1 - t3) >> 7)); src ++; dest++; diff --git a/libavcodec/vp3dsp.c b/libavcodec/vp3dsp.c index baa22a5519..438ae76b57 100644 --- a/libavcodec/vp3dsp.c +++ b/libavcodec/vp3dsp.c @@ -41,7 +41,6 @@ static av_always_inline void idct(uint8_t *dst, int stride, int16_t *input, int type) { int16_t *ip = input; - uint8_t *cm = ff_cropTbl + MAX_NEG_CROP; int A, B, C, D, Ad, Bd, Cd, Dd, E, F, G, H; int Ed, Gd, Add, Bdd, Fd, Hd; @@ -147,29 +146,29 @@ static av_always_inline void idct(uint8_t *dst, int stride, int16_t *input, int ip[5*8] = (Fd + Bdd ) >> 4; ip[6*8] = (Fd - Bdd ) >> 4; }else if(type==1){ - dst[0*stride] = cm[(Gd + Cd ) >> 4]; - dst[7*stride] = cm[(Gd - Cd ) >> 4]; + dst[0*stride] = av_clip_uint8((Gd + Cd ) >> 4); + dst[7*stride] = av_clip_uint8((Gd - Cd ) >> 4); - dst[1*stride] = cm[(Add + Hd ) >> 4]; - dst[2*stride] = cm[(Add - Hd ) >> 4]; + dst[1*stride] = av_clip_uint8((Add + Hd ) >> 4); + dst[2*stride] = av_clip_uint8((Add - Hd ) >> 4); - dst[3*stride] = cm[(Ed + Dd ) >> 4]; - dst[4*stride] = cm[(Ed - Dd ) >> 4]; + dst[3*stride] = av_clip_uint8((Ed + Dd ) >> 4); + dst[4*stride] = av_clip_uint8((Ed - Dd ) >> 4); - dst[5*stride] = cm[(Fd + Bdd ) >> 4]; - dst[6*stride] = cm[(Fd - Bdd ) >> 4]; + dst[5*stride] = av_clip_uint8((Fd + Bdd ) >> 4); + dst[6*stride] = av_clip_uint8((Fd - Bdd ) >> 4); }else{ - dst[0*stride] = cm[dst[0*stride] + ((Gd + Cd ) >> 4)]; - dst[7*stride] = cm[dst[7*stride] + ((Gd - Cd ) >> 4)]; + dst[0*stride] = av_clip_uint8(dst[0*stride] + ((Gd + Cd ) >> 4)); + dst[7*stride] = av_clip_uint8(dst[7*stride] + ((Gd - Cd ) >> 4)); - dst[1*stride] = cm[dst[1*stride] + ((Add + Hd ) >> 4)]; - dst[2*stride] = cm[dst[2*stride] + ((Add - Hd ) >> 4)]; + dst[1*stride] = av_clip_uint8(dst[1*stride] + ((Add + Hd ) >> 4)); + dst[2*stride] = av_clip_uint8(dst[2*stride] + ((Add - Hd ) >> 4)); - dst[3*stride] = cm[dst[3*stride] + ((Ed + Dd ) >> 4)]; - dst[4*stride] = cm[dst[4*stride] + ((Ed - Dd ) >> 4)]; + dst[3*stride] = av_clip_uint8(dst[3*stride] + ((Ed + Dd ) >> 4)); + dst[4*stride] = av_clip_uint8(dst[4*stride] + ((Ed - Dd ) >> 4)); - dst[5*stride] = cm[dst[5*stride] + ((Fd + Bdd ) >> 4)]; - dst[6*stride] = cm[dst[6*stride] + ((Fd - Bdd ) >> 4)]; + dst[5*stride] = av_clip_uint8(dst[5*stride] + ((Fd + Bdd ) >> 4)); + dst[6*stride] = av_clip_uint8(dst[6*stride] + ((Fd - Bdd ) >> 4)); } } else { @@ -190,18 +189,18 @@ static av_always_inline void idct(uint8_t *dst, int stride, int16_t *input, int dst[4*stride]= dst[5*stride]= dst[6*stride]= - dst[7*stride]= cm[128 + ((xC4S4 * ip[0*8] + (IdctAdjustBeforeShift<<16))>>20)]; + dst[7*stride]= av_clip_uint8(128 + ((xC4S4 * ip[0*8] + (IdctAdjustBeforeShift<<16))>>20)); }else{ if(ip[0*8]){ int v= ((xC4S4 * ip[0*8] + (IdctAdjustBeforeShift<<16))>>20); - dst[0*stride] = cm[dst[0*stride] + v]; - dst[1*stride] = cm[dst[1*stride] + v]; - dst[2*stride] = cm[dst[2*stride] + v]; - dst[3*stride] = cm[dst[3*stride] + v]; - dst[4*stride] = cm[dst[4*stride] + v]; - dst[5*stride] = cm[dst[5*stride] + v]; - dst[6*stride] = cm[dst[6*stride] + v]; - dst[7*stride] = cm[dst[7*stride] + v]; + dst[0*stride] = av_clip_uint8(dst[0*stride] + v); + dst[1*stride] = av_clip_uint8(dst[1*stride] + v); + dst[2*stride] = av_clip_uint8(dst[2*stride] + v); + dst[3*stride] = av_clip_uint8(dst[3*stride] + v); + dst[4*stride] = av_clip_uint8(dst[4*stride] + v); + dst[5*stride] = av_clip_uint8(dst[5*stride] + v); + dst[6*stride] = av_clip_uint8(dst[6*stride] + v); + dst[7*stride] = av_clip_uint8(dst[7*stride] + v); } } } @@ -225,17 +224,16 @@ void ff_vp3_idct_add_c(uint8_t *dest/*align 8*/, int line_size, DCTELEM *block/* void ff_vp3_idct_dc_add_c(uint8_t *dest/*align 8*/, int line_size, const DCTELEM *block/*align 16*/){ int i, dc = (block[0] + 15) >> 5; - const uint8_t *cm = ff_cropTbl + MAX_NEG_CROP + dc; for(i = 0; i < 8; i++){ - dest[0] = cm[dest[0]]; - dest[1] = cm[dest[1]]; - dest[2] = cm[dest[2]]; - dest[3] = cm[dest[3]]; - dest[4] = cm[dest[4]]; - dest[5] = cm[dest[5]]; - dest[6] = cm[dest[6]]; - dest[7] = cm[dest[7]]; + dest[0] = av_clip_uint8(dest[0] + dc); + dest[1] = av_clip_uint8(dest[1] + dc); + dest[2] = av_clip_uint8(dest[2] + dc); + dest[3] = av_clip_uint8(dest[3] + dc); + dest[4] = av_clip_uint8(dest[4] + dc); + dest[5] = av_clip_uint8(dest[5] + dc); + dest[6] = av_clip_uint8(dest[6] + dc); + dest[7] = av_clip_uint8(dest[7] + dc); dest += line_size; } } diff --git a/libavcodec/vp8dsp.c b/libavcodec/vp8dsp.c index 89c3453efc..20bf66f206 100644 --- a/libavcodec/vp8dsp.c +++ b/libavcodec/vp8dsp.c @@ -80,7 +80,6 @@ static void vp8_luma_dc_wht_dc_c(DCTELEM block[4][4][16], DCTELEM dc[16]) static void vp8_idct_add_c(uint8_t *dst, DCTELEM block[16], int stride) { int i, t0, t1, t2, t3; - uint8_t *cm = ff_cropTbl + MAX_NEG_CROP; DCTELEM tmp[16]; for (i = 0; i < 4; i++) { @@ -105,10 +104,10 @@ static void vp8_idct_add_c(uint8_t *dst, DCTELEM block[16], int stride) t2 = MUL_35468(tmp[1*4+i]) - MUL_20091(tmp[3*4+i]); t3 = MUL_20091(tmp[1*4+i]) + MUL_35468(tmp[3*4+i]); - dst[0] = cm[dst[0] + ((t0 + t3 + 4) >> 3)]; - dst[1] = cm[dst[1] + ((t1 + t2 + 4) >> 3)]; - dst[2] = cm[dst[2] + ((t1 - t2 + 4) >> 3)]; - dst[3] = cm[dst[3] + ((t0 - t3 + 4) >> 3)]; + dst[0] = av_clip_uint8(dst[0] + ((t0 + t3 + 4) >> 3)); + dst[1] = av_clip_uint8(dst[1] + ((t1 + t2 + 4) >> 3)); + dst[2] = av_clip_uint8(dst[2] + ((t1 - t2 + 4) >> 3)); + dst[3] = av_clip_uint8(dst[3] + ((t0 - t3 + 4) >> 3)); dst += stride; } } @@ -116,14 +115,13 @@ static void vp8_idct_add_c(uint8_t *dst, DCTELEM block[16], int stride) static void vp8_idct_dc_add_c(uint8_t *dst, DCTELEM block[16], int stride) { int i, dc = (block[0] + 4) >> 3; - uint8_t *cm = ff_cropTbl + MAX_NEG_CROP + dc; block[0] = 0; for (i = 0; i < 4; i++) { - dst[0] = cm[dst[0]]; - dst[1] = cm[dst[1]]; - dst[2] = cm[dst[2]]; - dst[3] = cm[dst[3]]; + dst[0] = av_clip_uint8(dst[0] + dc); + dst[1] = av_clip_uint8(dst[1] + dc); + dst[2] = av_clip_uint8(dst[2] + dc); + dst[3] = av_clip_uint8(dst[3] + dc); dst += stride; } } From c3bf08d04cdec3d4fd5c4ea70e14b5edca2c45a7 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Tue, 6 Mar 2012 17:24:20 -0800 Subject: [PATCH 116/154] smacker: error out if palette copy-with-offset overruns palette size. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit a93b572ae4f517ce0c35cf085167c318e9215908) Signed-off-by: Reinhard Tartler --- libavformat/smacker.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/libavformat/smacker.c b/libavformat/smacker.c index 770f5364d3..6df8b8b619 100644 --- a/libavformat/smacker.c +++ b/libavformat/smacker.c @@ -265,8 +265,15 @@ static int smacker_read_packet(AVFormatContext *s, AVPacket *pkt) sz += (t & 0x7F) + 1; pal += ((t & 0x7F) + 1) * 3; } else if(t & 0x40){ /* copy with offset */ - off = avio_r8(s->pb) * 3; + off = avio_r8(s->pb); j = (t & 0x3F) + 1; + if (off + j > 0xff) { + av_log(s, AV_LOG_ERROR, + "Invalid palette update, offset=%d length=%d extends beyond palette size\n", + off, j); + return AVERROR_INVALIDDATA; + } + off *= 3; while(j-- && sz < 256) { *pal++ = oldpal[off + 0]; *pal++ = oldpal[off + 1]; From e1b4614ab463f8ef4e350e0750fdefddea392135 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Tue, 6 Mar 2012 20:08:17 -0800 Subject: [PATCH 117/154] lpcm: fix sample size calculation for 20bit LCPM. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit f1320dc3bed281bb2f3c5531c52b6a6246e2394a) Signed-off-by: Reinhard Tartler --- libavcodec/pcm-mpeg.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavcodec/pcm-mpeg.c b/libavcodec/pcm-mpeg.c index 9ab6fc3ff0..f010b970bf 100644 --- a/libavcodec/pcm-mpeg.c +++ b/libavcodec/pcm-mpeg.c @@ -156,7 +156,7 @@ static int pcm_bluray_decode_frame(AVCodecContext *avctx, void *data, /* There's always an even number of channels in the source */ num_source_channels = FFALIGN(avctx->channels, 2); - sample_size = (num_source_channels * avctx->bits_per_coded_sample) >> 3; + sample_size = (num_source_channels * (avctx->sample_fmt == AV_SAMPLE_FMT_S16 ? 16 : 24)) >> 3; samples = buf_size / sample_size; /* get output buffer */ From ed6aaf579db01d114d6198257fb734e20bc09f42 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Tue, 28 Feb 2012 18:11:59 -0800 Subject: [PATCH 118/154] dca: prevent accessing static arrays with invalid indexes. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit e6ffd997cbc06426e75d3fa291b991866c84a79b) Signed-off-by: Reinhard Tartler --- libavcodec/dca.c | 37 ++++++++++++++++++++++++++----------- libavcodec/dcadata.h | 2 +- 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/libavcodec/dca.c b/libavcodec/dca.c index 3735b5a7fd..1bd31c9ac4 100644 --- a/libavcodec/dca.c +++ b/libavcodec/dca.c @@ -639,13 +639,20 @@ static int dca_parse_frame_header(DCAContext *s) } -static inline int get_scale(GetBitContext *gb, int level, int value) +static inline int get_scale(GetBitContext *gb, int level, int value, int log2range) { if (level < 5) { /* huffman encoded */ value += get_bitalloc(gb, &dca_scalefactor, level); - } else if (level < 8) - value = get_bits(gb, level + 1); + value = av_clip_uintp2(value, log2range); + } else if (level < 8) { + if (level + 1 > log2range) { + skip_bits(gb, level + 1 - log2range); + value = get_bits(gb, log2range); + } else { + value = get_bits(gb, level + 1); + } + } return value; } @@ -718,28 +725,31 @@ static int dca_subframe_header(DCAContext *s, int base_channel, int block_index) for (j = base_channel; j < s->prim_channels; j++) { const uint32_t *scale_table; - int scale_sum; + int scale_sum, log_size; memset(s->scale_factor[j], 0, s->subband_activity[j] * sizeof(s->scale_factor[0][0][0]) * 2); - if (s->scalefactor_huffman[j] == 6) + if (s->scalefactor_huffman[j] == 6) { scale_table = scale_factor_quant7; - else + log_size = 7; + } else { scale_table = scale_factor_quant6; + log_size = 6; + } /* When huffman coded, only the difference is encoded */ scale_sum = 0; for (k = 0; k < s->subband_activity[j]; k++) { if (k >= s->vq_start_subband[j] || s->bitalloc[j][k] > 0) { - scale_sum = get_scale(&s->gb, s->scalefactor_huffman[j], scale_sum); + scale_sum = get_scale(&s->gb, s->scalefactor_huffman[j], scale_sum, log_size); s->scale_factor[j][k][0] = scale_table[scale_sum]; } if (k < s->vq_start_subband[j] && s->transition_mode[j][k]) { /* Get second scale factor */ - scale_sum = get_scale(&s->gb, s->scalefactor_huffman[j], scale_sum); + scale_sum = get_scale(&s->gb, s->scalefactor_huffman[j], scale_sum, log_size); s->scale_factor[j][k][1] = scale_table[scale_sum]; } } @@ -768,8 +778,7 @@ static int dca_subframe_header(DCAContext *s, int base_channel, int block_index) * (is this valid as well for joint scales ???) */ for (k = s->subband_activity[j]; k < s->subband_activity[source_channel]; k++) { - scale = get_scale(&s->gb, s->joint_huff[j], 0); - scale += 64; /* bias */ + scale = get_scale(&s->gb, s->joint_huff[j], 64 /* bias */, 7); s->joint_scale_factor[j][k] = scale; /*joint_scale_table[scale]; */ } @@ -790,6 +799,11 @@ static int dca_subframe_header(DCAContext *s, int base_channel, int block_index) } } else { int am = s->amode & DCA_CHANNEL_MASK; + if (am >= FF_ARRAY_ELEMS(dca_default_coeffs)) { + av_log(s->avctx, AV_LOG_ERROR, + "Invalid channel mode %d\n", am); + return AVERROR_INVALIDDATA; + } for (j = base_channel; j < s->prim_channels; j++) { s->downmix_coef[j][0] = dca_default_coeffs[am][j][0]; s->downmix_coef[j][1] = dca_default_coeffs[am][j][1]; @@ -829,7 +843,8 @@ static int dca_subframe_header(DCAContext *s, int base_channel, int block_index) } /* Scale factor index */ - s->lfe_scale_factor = scale_factor_quant7[get_bits(&s->gb, 8)]; + skip_bits(&s->gb, 1); + s->lfe_scale_factor = scale_factor_quant7[get_bits(&s->gb, 7)]; /* Quantization step size * scale factor */ lfe_scale = 0.035 * s->lfe_scale_factor; diff --git a/libavcodec/dcadata.h b/libavcodec/dcadata.h index 0a83cdfae7..4b58ef7c38 100644 --- a/libavcodec/dcadata.h +++ b/libavcodec/dcadata.h @@ -7528,7 +7528,7 @@ static const float dca_downmix_coeffs[65] = { 0.001412537544623, 0.001000000000000, 0.000501187233627, 0.000251188643151, 0.000000000000000, }; -static const uint8_t dca_default_coeffs[16][5][2] = { +static const uint8_t dca_default_coeffs[10][5][2] = { { { 13, 13 }, }, { { 0, 64 }, { 64, 0 }, }, { { 0, 64 }, { 64, 0 }, }, From 9cb7f6e54a426e132396548a745cb32ff825b1fa Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Tue, 6 Mar 2012 16:08:10 -0800 Subject: [PATCH 119/154] raw: move buffer size check up. This way, it protects against overreads for 4bpp/2bpp content also. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit cc5dd632cecc5114717d0b90f8c2be162b1c6ee8) Signed-off-by: Reinhard Tartler --- libavcodec/rawdec.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libavcodec/rawdec.c b/libavcodec/rawdec.c index 427d109a2b..83b2a216b5 100644 --- a/libavcodec/rawdec.c +++ b/libavcodec/rawdec.c @@ -129,6 +129,9 @@ static int raw_decode(AVCodecContext *avctx, frame->reordered_opaque = avctx->reordered_opaque; frame->pkt_pts = avctx->pkt->pts; + if(buf_size < context->length - (avctx->pix_fmt==PIX_FMT_PAL8 ? 256*4 : 0)) + return -1; + //2bpp and 4bpp raw in avi and mov (yes this is ugly ...) if (context->buffer) { int i; @@ -153,9 +156,6 @@ static int raw_decode(AVCodecContext *avctx, avctx->codec_tag == MKTAG('A', 'V', 'u', 'p')) buf += buf_size - context->length; - if(buf_size < context->length - (avctx->pix_fmt==PIX_FMT_PAL8 ? 256*4 : 0)) - return -1; - avpicture_fill(picture, buf, avctx->pix_fmt, avctx->width, avctx->height); if((avctx->pix_fmt==PIX_FMT_PAL8 && buf_size < context->length) || (avctx->pix_fmt!=PIX_FMT_PAL8 && From 74871ac70ae387470a5da469157050cb2d3ed36f Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Wed, 7 Mar 2012 13:48:41 -0800 Subject: [PATCH 120/154] dv: check buffer size before reading profile. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit e97efecec82ca8458a9bbd75a91ebf556abde362) Signed-off-by: Reinhard Tartler --- libavcodec/dvdata.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/libavcodec/dvdata.c b/libavcodec/dvdata.c index 3a135a9ac7..62e569c576 100644 --- a/libavcodec/dvdata.c +++ b/libavcodec/dvdata.c @@ -248,11 +248,13 @@ static const DVprofile dv_profiles[] = { const DVprofile* avpriv_dv_frame_profile(const DVprofile *sys, const uint8_t* frame, unsigned buf_size) { - int i; + int i, dsf, stype; - int dsf = (frame[3] & 0x80) >> 7; + if (buf_size < 80*5 + 48 + 4) + return NULL; - int stype = frame[80*5 + 48 + 3] & 0x1f; + dsf = (frame[3] & 0x80) >> 7; + stype = frame[80*5 + 48 + 3] & 0x1f; /* 576i50 25Mbps 4:1:1 is a special case */ if (dsf == 1 && stype == 0 && frame[4] & 0x07 /* the APT field */) { From 1fcc2c60914c1fd9c516203f675676e1586b0376 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Wed, 7 Mar 2012 14:18:14 -0800 Subject: [PATCH 121/154] wma: fix off-by-one in array bounds check. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit b4bccf3e4e58f6fe58043791ca09db01a4343fac) Signed-off-by: Reinhard Tartler --- libavcodec/wmadec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavcodec/wmadec.c b/libavcodec/wmadec.c index 37feca1f7f..a7300594ca 100644 --- a/libavcodec/wmadec.c +++ b/libavcodec/wmadec.c @@ -356,7 +356,7 @@ static int decode_exp_vlc(WMACodecContext *s, int ch) } /* NOTE: this offset is the same as MPEG4 AAC ! */ last_exp += code - 60; - if ((unsigned)last_exp + 60 > FF_ARRAY_ELEMS(pow_tab)) { + if ((unsigned)last_exp + 60 >= FF_ARRAY_ELEMS(pow_tab)) { av_log(s->avctx, AV_LOG_ERROR, "Exponent out of range: %d\n", last_exp); return -1; From 2744fdbd9e1ee6a10f7627147be6556d04c1a88a Mon Sep 17 00:00:00 2001 From: Alex Converse Date: Tue, 6 Mar 2012 17:00:29 -0800 Subject: [PATCH 122/154] tiffdec: Prevent illegal memory access caused by recycled pointers. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit fd0be63049ed46660993d0550a4f0847a0b942ea) Signed-off-by: Reinhard Tartler --- libavcodec/tiff.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libavcodec/tiff.c b/libavcodec/tiff.c index d807149922..a0db1f1d28 100644 --- a/libavcodec/tiff.c +++ b/libavcodec/tiff.c @@ -535,6 +535,8 @@ static int decode_frame(AVCodecContext *avctx, av_log(avctx, AV_LOG_ERROR, "The answer to life, universe and everything is not correct!\n"); return -1; } + // Reset these pointers so we can tell if they were set this frame + s->stripsizes = s->stripdata = NULL; /* parse image file directory */ off = tget_long(&buf, le); if (off >= UINT_MAX - 14 || end_buf - orig_buf < off + 14) { From d4f2786cda271ed408e59f68e4a656f610a39808 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Wed, 15 Feb 2012 16:21:34 -0800 Subject: [PATCH 123/154] avs: fix infinite loop on end-of-stream. The codec would keep returning the last decoded frame if the stream contains B-frames, since it wouldn't clear that frame from the list of frames to be returned to the user. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 83f15a1228895434a982c840b09edccd1c64e800) Conflicts: libavcodec/cavsdec.c Signed-off-by: Reinhard Tartler --- libavcodec/cavsdec.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libavcodec/cavsdec.c b/libavcodec/cavsdec.c index 2f4b6e3b14..b0e517bbc5 100644 --- a/libavcodec/cavsdec.c +++ b/libavcodec/cavsdec.c @@ -655,7 +655,8 @@ static int cavs_decode_frame(AVCodecContext * avctx,void *data, int *data_size, if (buf_size == 0) { if (!s->low_delay && h->DPB[0].f.data[0]) { *data_size = sizeof(AVPicture); - *picture = *(AVFrame *) &h->DPB[0]; + *picture = h->DPB[0].f; + memset(&h->DPB[0], 0, sizeof(h->DPB[0])); } return 0; } From 9980e4df3bfcf49da2d3b22ed808b3dca0e7bbf2 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Wed, 7 Mar 2012 16:29:23 -0800 Subject: [PATCH 124/154] huffyuv: add padding to classic (v1) huffman tables. We slightly overread the input buffer, so we require padding at the end of the buffer, as is documented in the get_bits API. Without padding, we'll read uninitialized data or beyond the end of the .rodata, which may crash. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 4ffe5e2aa5241f8da9afd2c8fbc854dcc916c5f9) Signed-off-by: Reinhard Tartler --- libavcodec/huffyuv.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/libavcodec/huffyuv.c b/libavcodec/huffyuv.c index 412fe4b68d..2e1db043ea 100644 --- a/libavcodec/huffyuv.c +++ b/libavcodec/huffyuv.c @@ -82,13 +82,15 @@ typedef struct HYuvContext{ DSPContext dsp; }HYuvContext; -static const unsigned char classic_shift_luma[] = { +#define classic_shift_luma_table_size 42 +static const unsigned char classic_shift_luma[classic_shift_luma_table_size + FF_INPUT_BUFFER_PADDING_SIZE] = { 34,36,35,69,135,232,9,16,10,24,11,23,12,16,13,10,14,8,15,8, 16,8,17,20,16,10,207,206,205,236,11,8,10,21,9,23,8,8,199,70, 69,68, 0 }; -static const unsigned char classic_shift_chroma[] = { +#define classic_shift_chroma_table_size 59 +static const unsigned char classic_shift_chroma[classic_shift_chroma_table_size + FF_INPUT_BUFFER_PADDING_SIZE] = { 66,36,37,38,39,40,41,75,76,77,110,239,144,81,82,83,84,85,118,183, 56,57,88,89,56,89,154,57,58,57,26,141,57,56,58,57,58,57,184,119, 214,245,116,83,82,49,80,79,78,77,44,75,41,40,39,38,37,36,34, 0 @@ -366,10 +368,10 @@ static int read_old_huffman_tables(HYuvContext *s){ GetBitContext gb; int i; - init_get_bits(&gb, classic_shift_luma, sizeof(classic_shift_luma)*8); + init_get_bits(&gb, classic_shift_luma, classic_shift_luma_table_size*8); if(read_len_table(s->len[0], &gb)<0) return -1; - init_get_bits(&gb, classic_shift_chroma, sizeof(classic_shift_chroma)*8); + init_get_bits(&gb, classic_shift_chroma, classic_shift_chroma_table_size*8); if(read_len_table(s->len[1], &gb)<0) return -1; From 88c3cc019c8f3ebb9a41ce49c4b7ee6242836849 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Thu, 8 Mar 2012 17:09:27 -0800 Subject: [PATCH 125/154] cook: expand dither_tab[], and make sure indexes into it don't overflow. Fixes overflows in accessing dither_tab[]. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 442c3a8cb1785d74f8e2d7ab35b1862b7088436b) Signed-off-by: Reinhard Tartler --- libavcodec/cook.c | 6 +++++- libavcodec/cookdata.h | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/libavcodec/cook.c b/libavcodec/cook.c index dc4c2ab170..7c499f0c93 100644 --- a/libavcodec/cook.c +++ b/libavcodec/cook.c @@ -507,7 +507,11 @@ static inline void expand_category(COOKContext *q, int *category, { int i; for (i = 0; i < q->num_vectors; i++) - ++category[category_index[i]]; + { + int idx = category_index[i]; + if (++category[idx] >= FF_ARRAY_ELEMS(dither_tab)) + --category[idx]; + } } /** diff --git a/libavcodec/cookdata.h b/libavcodec/cookdata.h index e8d6ebfcb3..6825a4459c 100644 --- a/libavcodec/cookdata.h +++ b/libavcodec/cookdata.h @@ -36,8 +36,8 @@ static const int expbits_tab[8] = { 52,47,43,37,29,22,16,0, }; -static const float dither_tab[8] = { - 0.0, 0.0, 0.0, 0.0, 0.0, 0.176777, 0.25, 0.707107, +static const float dither_tab[9] = { + 0.0, 0.0, 0.0, 0.0, 0.0, 0.176777, 0.25, 0.707107, 1.0 }; static const float quant_centroid_tab[7][14] = { From b9482a6efdac8d8c31ce93ce9393f20eb029865d Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Tue, 6 Mar 2012 13:45:32 -0800 Subject: [PATCH 126/154] cook: extend channel uncoupling tables so the full bit range is covered. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 37cc8600d0313838cab5b886b9d373e5819aa24f) Signed-off-by: Reinhard Tartler --- libavcodec/cook.c | 4 ++-- libavcodec/cookdata.h | 27 ++++++++++++++++++++++----- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/libavcodec/cook.c b/libavcodec/cook.c index 7c499f0c93..d869c4200e 100644 --- a/libavcodec/cook.c +++ b/libavcodec/cook.c @@ -836,8 +836,8 @@ static void joint_decode(COOKContext *q, COOKSubpacket *p, float *mlt_buffer1, cpl_tmp = cplband[i]; idx -= decouple_tab[cpl_tmp]; cplscale = q->cplscales[p->js_vlc_bits - 2]; // choose decoupler table - f1 = cplscale[decouple_tab[cpl_tmp]]; - f2 = cplscale[idx - 1]; + f1 = cplscale[decouple_tab[cpl_tmp] + 1]; + f2 = cplscale[idx]; q->decouple(q, p, i, f1, f2, decode_buffer, mlt_buffer1, mlt_buffer2); idx = (1 << p->js_vlc_bits) - 1; } diff --git a/libavcodec/cookdata.h b/libavcodec/cookdata.h index 6825a4459c..c4c26fae5f 100644 --- a/libavcodec/cookdata.h +++ b/libavcodec/cookdata.h @@ -510,23 +510,37 @@ static const int cplband[51] = { 19, }; -static const float cplscale2[3] = { +// The 1 and 0 at the beginning/end are to prevent overflows with +// bitstream-read indexes. E.g. if n_bits=5, we can access any +// index from [1, (1< Date: Sat, 10 Mar 2012 17:51:28 -0800 Subject: [PATCH 127/154] cook: error out on quant_index values outside [-63, 63] range. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 97e48b2f541396ef6e8816a555bac1bb993d7a6a) Signed-off-by: Reinhard Tartler --- libavcodec/cook.c | 50 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 36 insertions(+), 14 deletions(-) diff --git a/libavcodec/cook.c b/libavcodec/cook.c index d869c4200e..a835442b6b 100644 --- a/libavcodec/cook.c +++ b/libavcodec/cook.c @@ -366,8 +366,8 @@ static void decode_gain_info(GetBitContext *gb, int *gaininfo) * @param q pointer to the COOKContext * @param quant_index_table pointer to the array */ -static void decode_envelope(COOKContext *q, COOKSubpacket *p, - int *quant_index_table) +static int decode_envelope(COOKContext *q, COOKSubpacket *p, + int *quant_index_table) { int i, j, vlc_index; @@ -388,7 +388,15 @@ static void decode_envelope(COOKContext *q, COOKSubpacket *p, j = get_vlc2(&q->gb, q->envelope_quant_index[vlc_index - 1].table, q->envelope_quant_index[vlc_index - 1].bits, 2); quant_index_table[i] = quant_index_table[i - 1] + j - 12; // differential encoding + if (quant_index_table[i] > 63 || quant_index_table[i] < -63) { + av_log(q->avctx, AV_LOG_ERROR, + "Invalid quantizer %d at position %d, outside [-63, 63] range\n", + quant_index_table[i], i); + return AVERROR_INVALIDDATA; + } } + + return 0; } /** @@ -639,20 +647,24 @@ static void decode_vectors(COOKContext *q, COOKSubpacket *p, int *category, * @param q pointer to the COOKContext * @param mlt_buffer pointer to mlt coefficients */ -static void mono_decode(COOKContext *q, COOKSubpacket *p, float *mlt_buffer) +static int mono_decode(COOKContext *q, COOKSubpacket *p, float *mlt_buffer) { int category_index[128]; int quant_index_table[102]; int category[128]; + int res; memset(&category, 0, sizeof(category)); memset(&category_index, 0, sizeof(category_index)); - decode_envelope(q, p, quant_index_table); + if ((res = decode_envelope(q, p, quant_index_table)) < 0) + return res; q->num_vectors = get_bits(&q->gb, p->log2_numvector_size); categorize(q, p, quant_index_table, category, category_index); expand_category(q, category, category_index); decode_vectors(q, p, category, quant_index_table, mlt_buffer); + + return 0; } @@ -802,10 +814,10 @@ static void decouple_float(COOKContext *q, * @param mlt_buffer1 pointer to left channel mlt coefficients * @param mlt_buffer2 pointer to right channel mlt coefficients */ -static void joint_decode(COOKContext *q, COOKSubpacket *p, float *mlt_buffer1, - float *mlt_buffer2) +static int joint_decode(COOKContext *q, COOKSubpacket *p, float *mlt_buffer1, + float *mlt_buffer2) { - int i, j; + int i, j, res; int decouple_tab[SUBBAND_SIZE]; float *decode_buffer = q->decode_buffer_0; int idx, cpl_tmp; @@ -819,7 +831,8 @@ static void joint_decode(COOKContext *q, COOKSubpacket *p, float *mlt_buffer1, memset(mlt_buffer1, 0, 1024 * sizeof(*mlt_buffer1)); memset(mlt_buffer2, 0, 1024 * sizeof(*mlt_buffer2)); decouple_info(q, p, decouple_tab); - mono_decode(q, p, decode_buffer); + if ((res = mono_decode(q, p, decode_buffer)) < 0) + return res; /* The two channels are stored interleaved in decode_buffer. */ for (i = 0; i < p->js_subband_start; i++) { @@ -841,6 +854,8 @@ static void joint_decode(COOKContext *q, COOKSubpacket *p, float *mlt_buffer1, q->decouple(q, p, i, f1, f2, decode_buffer, mlt_buffer1, mlt_buffer2); idx = (1 << p->js_vlc_bits) - 1; } + + return 0; } /** @@ -913,10 +928,11 @@ static inline void mlt_compensate_output(COOKContext *q, float *decode_buffer, * @param inbuffer pointer to the inbuffer * @param outbuffer pointer to the outbuffer */ -static void decode_subpacket(COOKContext *q, COOKSubpacket *p, - const uint8_t *inbuffer, float *outbuffer) +static int decode_subpacket(COOKContext *q, COOKSubpacket *p, + const uint8_t *inbuffer, float *outbuffer) { int sub_packet_size = p->size; + int res; /* packet dump */ // for (i = 0; i < sub_packet_size ; i++) // av_log(q->avctx, AV_LOG_ERROR, "%02x", inbuffer[i]); @@ -925,13 +941,16 @@ static void decode_subpacket(COOKContext *q, COOKSubpacket *p, decode_bytes_and_gain(q, p, inbuffer, &p->gains1); if (p->joint_stereo) { - joint_decode(q, p, q->decode_buffer_1, q->decode_buffer_2); + if ((res = joint_decode(q, p, q->decode_buffer_1, q->decode_buffer_2)) < 0) + return res; } else { - mono_decode(q, p, q->decode_buffer_1); + if ((res = mono_decode(q, p, q->decode_buffer_1)) < 0) + return res; if (p->num_channels == 2) { decode_bytes_and_gain(q, p, inbuffer + sub_packet_size / 2, &p->gains2); - mono_decode(q, p, q->decode_buffer_2); + if ((res = mono_decode(q, p, q->decode_buffer_2)) < 0) + return res; } } @@ -945,6 +964,8 @@ static void decode_subpacket(COOKContext *q, COOKSubpacket *p, else mlt_compensate_output(q, q->decode_buffer_2, &p->gains2, p->mono_previous_buffer2, outbuffer, p->ch_idx + 1); + + return 0; } @@ -1000,7 +1021,8 @@ static int cook_decode_frame(AVCodecContext *avctx, void *data, i, q->subpacket[i].size, q->subpacket[i].joint_stereo, offset, avctx->block_align); - decode_subpacket(q, &q->subpacket[i], buf + offset, samples); + if ((ret = decode_subpacket(q, &q->subpacket[i], buf + offset, samples)) < 0) + return ret; offset += q->subpacket[i].size; chidx += q->subpacket[i].num_channels; av_log(avctx, AV_LOG_DEBUG, "subpacket[%i] %i %i\n", From 26521d87ba22fe1bb49f1f0796c7227017064e7f Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Sun, 11 Mar 2012 07:28:54 -0700 Subject: [PATCH 128/154] dsicinvideo: validate buffer offset before copying pixels. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit c95fefa0420be9cc0f09a95041acf11114aaacd0) Signed-off-by: Reinhard Tartler --- libavcodec/dsicinav.c | 40 +++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/libavcodec/dsicinav.c b/libavcodec/dsicinav.c index 37d39f5405..a379531613 100644 --- a/libavcodec/dsicinav.c +++ b/libavcodec/dsicinav.c @@ -146,11 +146,11 @@ static int cin_decode_huffman(const unsigned char *src, int src_size, unsigned c return dst_cur - dst; } -static void cin_decode_lzss(const unsigned char *src, int src_size, unsigned char *dst, int dst_size) +static int cin_decode_lzss(const unsigned char *src, int src_size, unsigned char *dst, int dst_size) { uint16_t cmd; int i, sz, offset, code; - unsigned char *dst_end = dst + dst_size; + unsigned char *dst_end = dst + dst_size, *dst_start = dst; const unsigned char *src_end = src + src_size; while (src < src_end && dst < dst_end) { @@ -161,6 +161,8 @@ static void cin_decode_lzss(const unsigned char *src, int src_size, unsigned cha } else { cmd = AV_RL16(src); src += 2; offset = cmd >> 4; + if ((int) (dst - dst_start) < offset + 1) + return AVERROR_INVALIDDATA; sz = (cmd & 0xF) + 2; /* don't use memcpy/memmove here as the decoding routine (ab)uses */ /* buffer overlappings to repeat bytes in the destination */ @@ -172,6 +174,8 @@ static void cin_decode_lzss(const unsigned char *src, int src_size, unsigned cha } } } + + return 0; } static void cin_decode_rle(const unsigned char *src, int src_size, unsigned char *dst, int dst_size) @@ -201,13 +205,7 @@ static int cinvideo_decode_frame(AVCodecContext *avctx, const uint8_t *buf = avpkt->data; int buf_size = avpkt->size; CinVideoContext *cin = avctx->priv_data; - int i, y, palette_type, palette_colors_count, bitmap_frame_type, bitmap_frame_size; - - cin->frame.buffer_hints = FF_BUFFER_HINTS_VALID | FF_BUFFER_HINTS_PRESERVE | FF_BUFFER_HINTS_REUSABLE; - if (avctx->reget_buffer(avctx, &cin->frame)) { - av_log(cin->avctx, AV_LOG_ERROR, "delphinecinvideo: reget_buffer() failed to allocate a frame\n"); - return -1; - } + int i, y, palette_type, palette_colors_count, bitmap_frame_type, bitmap_frame_size, res = 0; palette_type = buf[0]; palette_colors_count = AV_RL16(buf+1); @@ -233,8 +231,6 @@ static int cinvideo_decode_frame(AVCodecContext *avctx, bitmap_frame_size -= 4; } } - memcpy(cin->frame.data[1], cin->palette, sizeof(cin->palette)); - cin->frame.palette_has_changed = 1; /* note: the decoding routines below assumes that surface.width = surface.pitch */ switch (bitmap_frame_type) { @@ -267,17 +263,31 @@ static int cinvideo_decode_frame(AVCodecContext *avctx, cin->bitmap_table[CIN_CUR_BMP], cin->bitmap_size); break; case 38: - cin_decode_lzss(buf, bitmap_frame_size, - cin->bitmap_table[CIN_CUR_BMP], cin->bitmap_size); + res = cin_decode_lzss(buf, bitmap_frame_size, + cin->bitmap_table[CIN_CUR_BMP], + cin->bitmap_size); + if (res < 0) + return res; break; case 39: - cin_decode_lzss(buf, bitmap_frame_size, - cin->bitmap_table[CIN_CUR_BMP], cin->bitmap_size); + res = cin_decode_lzss(buf, bitmap_frame_size, + cin->bitmap_table[CIN_CUR_BMP], + cin->bitmap_size); + if (res < 0) + return res; cin_apply_delta_data(cin->bitmap_table[CIN_PRE_BMP], cin->bitmap_table[CIN_CUR_BMP], cin->bitmap_size); break; } + cin->frame.buffer_hints = FF_BUFFER_HINTS_VALID | FF_BUFFER_HINTS_PRESERVE | FF_BUFFER_HINTS_REUSABLE; + if (avctx->reget_buffer(avctx, &cin->frame)) { + av_log(cin->avctx, AV_LOG_ERROR, "delphinecinvideo: reget_buffer() failed to allocate a frame\n"); + return -1; + } + + memcpy(cin->frame.data[1], cin->palette, sizeof(cin->palette)); + cin->frame.palette_has_changed = 1; for (y = 0; y < cin->avctx->height; ++y) memcpy(cin->frame.data[0] + (cin->avctx->height - 1 - y) * cin->frame.linesize[0], cin->bitmap_table[CIN_CUR_BMP] + y * cin->avctx->width, From 8f881885c2325ce83f114437b97c2e0d6001cd7d Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Thu, 8 Mar 2012 16:32:46 -0800 Subject: [PATCH 129/154] xxan: don't read before start of buffer in av_memcpy_backptr(). Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit f1279e286b00e99f343adb51e251f036a3df6f32) Signed-off-by: Reinhard Tartler --- libavcodec/xxan.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libavcodec/xxan.c b/libavcodec/xxan.c index 58c80c05fa..12a261b49f 100644 --- a/libavcodec/xxan.c +++ b/libavcodec/xxan.c @@ -129,7 +129,8 @@ static int xan_unpack(uint8_t *dest, const int dest_len, if (size + size2 > dest_end - dest) break; } - if (src + size > src_end || dest + size + size2 > dest_end) + if (src + size > src_end || dest + size + size2 > dest_end || + dest - orig_dest + size < back) return -1; bytestream_get_buffer(&src, dest, size); dest += size; From a43f4bd601e905e3f04c47293a642ac541d727f3 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Thu, 8 Mar 2012 16:32:47 -0800 Subject: [PATCH 130/154] xxan: convert to bytestream2 API. Protects against overreads. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 55188278169c3a1838334d7aa47a1f7a40741690) Signed-off-by: Reinhard Tartler --- libavcodec/xxan.c | 117 ++++++++++++++++++++++------------------------ 1 file changed, 56 insertions(+), 61 deletions(-) diff --git a/libavcodec/xxan.c b/libavcodec/xxan.c index 12a261b49f..fb8fb526ac 100644 --- a/libavcodec/xxan.c +++ b/libavcodec/xxan.c @@ -35,6 +35,7 @@ typedef struct XanContext { uint8_t *y_buffer; uint8_t *scratch_buffer; int buffer_size; + GetByteContext gb; } XanContext; static av_cold int xan_decode_init(AVCodecContext *avctx) @@ -58,29 +59,29 @@ static av_cold int xan_decode_init(AVCodecContext *avctx) return 0; } -static int xan_unpack_luma(const uint8_t *src, const int src_size, +static int xan_unpack_luma(XanContext *s, uint8_t *dst, const int dst_size) { int tree_size, eof; - const uint8_t *tree; int bits, mask; int tree_root, node; const uint8_t *dst_end = dst + dst_size; - const uint8_t *src_end = src + src_size; + GetByteContext tree = s->gb; + int start_off = bytestream2_tell(&tree); - tree_size = *src++; - eof = *src++; - tree = src - eof * 2 - 2; + tree_size = bytestream2_get_byte(&s->gb); + eof = bytestream2_get_byte(&s->gb); tree_root = eof + tree_size; - src += tree_size * 2; + bytestream2_skip(&s->gb, tree_size * 2); node = tree_root; - bits = *src++; + bits = bytestream2_get_byte(&s->gb); mask = 0x80; for (;;) { int bit = !!(bits & mask); mask >>= 1; - node = tree[node*2 + bit]; + bytestream2_seek(&tree, start_off + node*2 + bit - eof * 2, SEEK_SET); + node = bytestream2_get_byte(&tree); if (node == eof) break; if (node < eof) { @@ -90,49 +91,51 @@ static int xan_unpack_luma(const uint8_t *src, const int src_size, node = tree_root; } if (!mask) { - bits = *src++; - if (src > src_end) + if (bytestream2_get_bytes_left(&s->gb) <= 0) break; + bits = bytestream2_get_byteu(&s->gb); mask = 0x80; } } - return dst != dst_end; + return dst != dst_end ? AVERROR_INVALIDDATA : 0; } /* almost the same as in xan_wc3 decoder */ -static int xan_unpack(uint8_t *dest, const int dest_len, - const uint8_t *src, const int src_len) +static int xan_unpack(XanContext *s, + uint8_t *dest, const int dest_len) { uint8_t opcode; int size; uint8_t *orig_dest = dest; - const uint8_t *src_end = src + src_len; const uint8_t *dest_end = dest + dest_len; while (dest < dest_end) { - opcode = *src++; + if (bytestream2_get_bytes_left(&s->gb) <= 0) + return AVERROR_INVALIDDATA; + + opcode = bytestream2_get_byteu(&s->gb); if (opcode < 0xe0) { int size2, back; if ((opcode & 0x80) == 0) { size = opcode & 3; - back = ((opcode & 0x60) << 3) + *src++ + 1; + back = ((opcode & 0x60) << 3) + bytestream2_get_byte(&s->gb) + 1; size2 = ((opcode & 0x1c) >> 2) + 3; } else if ((opcode & 0x40) == 0) { - size = *src >> 6; - back = (bytestream_get_be16(&src) & 0x3fff) + 1; + size = bytestream2_peek_byte(&s->gb) >> 6; + back = (bytestream2_get_be16(&s->gb) & 0x3fff) + 1; size2 = (opcode & 0x3f) + 4; } else { size = opcode & 3; - back = ((opcode & 0x10) << 12) + bytestream_get_be16(&src) + 1; - size2 = ((opcode & 0x0c) << 6) + *src++ + 5; + back = ((opcode & 0x10) << 12) + bytestream2_get_be16(&s->gb) + 1; + size2 = ((opcode & 0x0c) << 6) + bytestream2_get_byte(&s->gb) + 5; if (size + size2 > dest_end - dest) break; } - if (src + size > src_end || dest + size + size2 > dest_end || + if (dest + size + size2 > dest_end || dest - orig_dest + size < back) return -1; - bytestream_get_buffer(&src, dest, size); + bytestream2_get_buffer(&s->gb, dest, size); dest += size; av_memcpy_backptr(dest, back, size2); dest += size2; @@ -140,9 +143,9 @@ static int xan_unpack(uint8_t *dest, const int dest_len, int finish = opcode >= 0xfc; size = finish ? opcode & 3 : ((opcode & 0x1f) << 2) + 4; - if (src + size > src_end || dest + size > dest_end) + if (dest_end - dest < size) return -1; - bytestream_get_buffer(&src, dest, size); + bytestream2_get_buffer(&s->gb, dest, size); dest += size; if (finish) break; @@ -151,38 +154,35 @@ static int xan_unpack(uint8_t *dest, const int dest_len, return dest - orig_dest; } -static int xan_decode_chroma(AVCodecContext *avctx, AVPacket *avpkt) +static int xan_decode_chroma(AVCodecContext *avctx, unsigned chroma_off) { - const uint8_t *buf = avpkt->data; XanContext *s = avctx->priv_data; uint8_t *U, *V; - unsigned chroma_off; int val, uval, vval; int i, j; const uint8_t *src, *src_end; const uint8_t *table; int mode, offset, dec_size; - chroma_off = AV_RL32(buf + 4); if (!chroma_off) return 0; - if (chroma_off + 10 >= avpkt->size) { + if (chroma_off + 4 >= bytestream2_get_bytes_left(&s->gb)) { av_log(avctx, AV_LOG_ERROR, "Invalid chroma block position\n"); return -1; } - src = avpkt->data + 4 + chroma_off; - table = src + 2; - mode = bytestream_get_le16(&src); - offset = bytestream_get_le16(&src) * 2; + bytestream2_seek(&s->gb, chroma_off + 4, SEEK_SET); + mode = bytestream2_get_le16(&s->gb); + table = s->gb.buffer; + offset = bytestream2_get_le16(&s->gb) * 2; - if (src - avpkt->data >= avpkt->size - offset) { + if (offset >= bytestream2_get_bytes_left(&s->gb)) { av_log(avctx, AV_LOG_ERROR, "Invalid chroma block offset\n"); return -1; } + bytestream2_skip(&s->gb, offset); memset(s->scratch_buffer, 0, s->buffer_size); - dec_size = xan_unpack(s->scratch_buffer, s->buffer_size, src + offset, - avpkt->size - offset - (src - avpkt->data)); + dec_size = xan_unpack(s, s->scratch_buffer, s->buffer_size); if (dec_size < 0) { av_log(avctx, AV_LOG_ERROR, "Chroma unpacking failed\n"); return -1; @@ -234,32 +234,27 @@ static int xan_decode_chroma(AVCodecContext *avctx, AVPacket *avpkt) return 0; } -static int xan_decode_frame_type0(AVCodecContext *avctx, AVPacket *avpkt) +static int xan_decode_frame_type0(AVCodecContext *avctx) { - const uint8_t *buf = avpkt->data; XanContext *s = avctx->priv_data; uint8_t *ybuf, *prev_buf, *src = s->scratch_buffer; unsigned chroma_off, corr_off; - int cur, last, size; + int cur, last; int i, j; int ret; - corr_off = AV_RL32(buf + 8); - chroma_off = AV_RL32(buf + 4); + chroma_off = bytestream2_get_le32(&s->gb); + corr_off = bytestream2_get_le32(&s->gb); - if ((ret = xan_decode_chroma(avctx, avpkt)) != 0) + if ((ret = xan_decode_chroma(avctx, chroma_off)) != 0) return ret; - size = avpkt->size - 4; - if (corr_off >= avpkt->size) { + if (corr_off >= (s->gb.buffer_end - s->gb.buffer_start)) { av_log(avctx, AV_LOG_WARNING, "Ignoring invalid correction block position\n"); corr_off = 0; } - if (corr_off) - size = corr_off; - if (chroma_off) - size = FFMIN(size, chroma_off); - ret = xan_unpack_luma(buf + 12, size, src, s->buffer_size >> 1); + bytestream2_seek(&s->gb, 12, SEEK_SET); + ret = xan_unpack_luma(s, src, s->buffer_size >> 1); if (ret) { av_log(avctx, AV_LOG_ERROR, "Luma decoding failed\n"); return ret; @@ -295,12 +290,11 @@ static int xan_decode_frame_type0(AVCodecContext *avctx, AVPacket *avpkt) if (corr_off) { int corr_end, dec_size; - corr_end = avpkt->size; + corr_end = (s->gb.buffer_end - s->gb.buffer_start); if (chroma_off > corr_off) corr_end = chroma_off; - dec_size = xan_unpack(s->scratch_buffer, s->buffer_size, - avpkt->data + 8 + corr_off, - corr_end - corr_off); + bytestream2_seek(&s->gb, 8 + corr_off, SEEK_SET); + dec_size = xan_unpack(s, s->scratch_buffer, s->buffer_size); if (dec_size < 0) dec_size = 0; for (i = 0; i < dec_size; i++) @@ -319,19 +313,19 @@ static int xan_decode_frame_type0(AVCodecContext *avctx, AVPacket *avpkt) return 0; } -static int xan_decode_frame_type1(AVCodecContext *avctx, AVPacket *avpkt) +static int xan_decode_frame_type1(AVCodecContext *avctx) { - const uint8_t *buf = avpkt->data; XanContext *s = avctx->priv_data; uint8_t *ybuf, *src = s->scratch_buffer; int cur, last; int i, j; int ret; - if ((ret = xan_decode_chroma(avctx, avpkt)) != 0) + if ((ret = xan_decode_chroma(avctx, bytestream2_get_le32(&s->gb))) != 0) return ret; - ret = xan_unpack_luma(buf + 16, avpkt->size - 16, src, + bytestream2_seek(&s->gb, 16, SEEK_SET); + ret = xan_unpack_luma(s, src, s->buffer_size >> 1); if (ret) { av_log(avctx, AV_LOG_ERROR, "Luma decoding failed\n"); @@ -381,13 +375,14 @@ static int xan_decode_frame(AVCodecContext *avctx, return ret; } - ftype = AV_RL32(avpkt->data); + bytestream2_init(&s->gb, avpkt->data, avpkt->size); + ftype = bytestream2_get_le32(&s->gb); switch (ftype) { case 0: - ret = xan_decode_frame_type0(avctx, avpkt); + ret = xan_decode_frame_type0(avctx); break; case 1: - ret = xan_decode_frame_type1(avctx, avpkt); + ret = xan_decode_frame_type1(avctx); break; default: av_log(avctx, AV_LOG_ERROR, "Unknown frame type %d\n", ftype); From c65eadee5d200b3ed2106548e8d0cace3db5e97f Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Sat, 10 Mar 2012 11:57:17 -0800 Subject: [PATCH 131/154] xxan: protect against chroma LUT overreads. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit f77bfa837636a99a4034d31916a76f7d1688cf5a) Signed-off-by: Reinhard Tartler --- libavcodec/xxan.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/libavcodec/xxan.c b/libavcodec/xxan.c index fb8fb526ac..0a37d48f6b 100644 --- a/libavcodec/xxan.c +++ b/libavcodec/xxan.c @@ -162,7 +162,7 @@ static int xan_decode_chroma(AVCodecContext *avctx, unsigned chroma_off) int i, j; const uint8_t *src, *src_end; const uint8_t *table; - int mode, offset, dec_size; + int mode, offset, dec_size, table_size; if (!chroma_off) return 0; @@ -171,9 +171,11 @@ static int xan_decode_chroma(AVCodecContext *avctx, unsigned chroma_off) return -1; } bytestream2_seek(&s->gb, chroma_off + 4, SEEK_SET); - mode = bytestream2_get_le16(&s->gb); - table = s->gb.buffer; - offset = bytestream2_get_le16(&s->gb) * 2; + mode = bytestream2_get_le16(&s->gb); + table = s->gb.buffer; + table_size = bytestream2_get_le16(&s->gb); + offset = table_size * 2; + table_size += 1; if (offset >= bytestream2_get_bytes_left(&s->gb)) { av_log(avctx, AV_LOG_ERROR, "Invalid chroma block offset\n"); @@ -196,7 +198,7 @@ static int xan_decode_chroma(AVCodecContext *avctx, unsigned chroma_off) for (j = 0; j < avctx->height >> 1; j++) { for (i = 0; i < avctx->width >> 1; i++) { val = *src++; - if (val) { + if (val && val < table_size) { val = AV_RL16(table + (val << 1)); uval = (val >> 3) & 0xF8; vval = (val >> 8) & 0xF8; @@ -216,7 +218,7 @@ static int xan_decode_chroma(AVCodecContext *avctx, unsigned chroma_off) for (j = 0; j < avctx->height >> 2; j++) { for (i = 0; i < avctx->width >> 1; i += 2) { val = *src++; - if (val) { + if (val && val < table_size) { val = AV_RL16(table + (val << 1)); uval = (val >> 3) & 0xF8; vval = (val >> 8) & 0xF8; From 7bb97a61dfb65a3825e17c2dc1e0e693b5607ec6 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Sat, 10 Mar 2012 14:28:08 -0800 Subject: [PATCH 132/154] mpc: pad mpc_CC/SCF[] tables to allow for negative indices. MPC8 allows indices of mpc_CC up to -1, and mpc_SCF up to -6, thus pad the tables by that much on the left end. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit d7eabd50425a61b31e90c763a0c3e4316a725404) Signed-off-by: Reinhard Tartler --- libavcodec/mpc.c | 6 +++--- libavcodec/mpc7.c | 4 ++-- libavcodec/mpcdata.h | 10 +++++++--- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/libavcodec/mpc.c b/libavcodec/mpc.c index 4573860525..6b15a33e5a 100644 --- a/libavcodec/mpc.c +++ b/libavcodec/mpc.c @@ -78,13 +78,13 @@ void ff_mpc_dequantize_and_synth(MPCContext * c, int maxband, void *data, int ch for(ch = 0; ch < 2; ch++){ if(bands[i].res[ch]){ j = 0; - mul = mpc_CC[bands[i].res[ch]] * mpc_SCF[bands[i].scf_idx[ch][0]]; + mul = mpc_CC[bands[i].res[ch] + 1] * mpc_SCF[bands[i].scf_idx[ch][0]+6]; for(; j < 12; j++) c->sb_samples[ch][j][i] = mul * c->Q[ch][j + off]; - mul = mpc_CC[bands[i].res[ch]] * mpc_SCF[bands[i].scf_idx[ch][1]]; + mul = mpc_CC[bands[i].res[ch] + 1] * mpc_SCF[bands[i].scf_idx[ch][1]+6]; for(; j < 24; j++) c->sb_samples[ch][j][i] = mul * c->Q[ch][j + off]; - mul = mpc_CC[bands[i].res[ch]] * mpc_SCF[bands[i].scf_idx[ch][2]]; + mul = mpc_CC[bands[i].res[ch] + 1] * mpc_SCF[bands[i].scf_idx[ch][2]+6]; for(; j < 36; j++) c->sb_samples[ch][j][i] = mul * c->Q[ch][j + off]; } diff --git a/libavcodec/mpc7.c b/libavcodec/mpc7.c index 290ecfb385..b678afd1a2 100644 --- a/libavcodec/mpc7.c +++ b/libavcodec/mpc7.c @@ -193,7 +193,7 @@ static int get_scale_idx(GetBitContext *gb, int ref) int t = get_vlc2(gb, dscf_vlc.table, MPC7_DSCF_BITS, 1) - 7; if (t == 8) return get_bits(gb, 6); - return ref + t; + return av_clip_uintp2(ref + t, 7); } static int mpc7_decode_frame(AVCodecContext * avctx, void *data, @@ -234,7 +234,7 @@ static int mpc7_decode_frame(AVCodecContext * avctx, void *data, int t = 4; if(i) t = get_vlc2(&gb, hdr_vlc.table, MPC7_HDR_BITS, 1) - 5; if(t == 4) bands[i].res[ch] = get_bits(&gb, 4); - else bands[i].res[ch] = bands[i-1].res[ch] + t; + else bands[i].res[ch] = av_clip(bands[i-1].res[ch] + t, 0, 17); } if(bands[i].res[0] || bands[i].res[1]){ diff --git a/libavcodec/mpcdata.h b/libavcodec/mpcdata.h index 397dad59d8..15724f3b74 100644 --- a/libavcodec/mpcdata.h +++ b/libavcodec/mpcdata.h @@ -22,13 +22,17 @@ #ifndef AVCODEC_MPCDATA_H #define AVCODEC_MPCDATA_H -static const float mpc_CC[18] = { - 65536.0000, 21845.3333, 13107.2000, 9362.2857, 7281.7778, 4369.0667, 2114.0645, +static const float mpc_CC[18+1] = { + 111.285962475327f, // 32768/2/255*sqrt(3) + 65536.0000 /* this value is never used */, + 21845.3333, 13107.2000, 9362.2857, 7281.7778, 4369.0667, 2114.0645, 1040.2539, 516.0315, 257.0039, 128.2505, 64.0626, 32.0156, 16.0039, 8.0010, 4.0002, 2.0001, 1.0000 }; -static const float mpc_SCF[128] = { +static const float mpc_SCF[128+6] = { + 920.016296386718750000, 766.355773925781250000, 638.359558105468750000, + 531.741149902343750000, 442.930114746093750000, 368.952209472656250000, 307.330047607421875000, 255.999984741210937500, 213.243041992187500000, 177.627334594726562500, 147.960128784179687500, 123.247924804687500000, 102.663139343261718750, 85.516410827636718750, 71.233520507812500000, 59.336143493652343750, 49.425861358642578125, 41.170787811279296875, From d94256d36cc789788a68c6b35d31481c4b16fdd3 Mon Sep 17 00:00:00 2001 From: Janne Grunau Date: Mon, 12 Mar 2012 22:01:02 +0100 Subject: [PATCH 133/154] Revert "h264: clear trailing bits in partially parsed NAL units" This reverts commit 729ebb2f185244b0ff06d48edbbbbb02ceb4ed4e. There was an off-by-one error in the bit mask calculation clearing actually the last valid bit and causing http://bugzilla.libav.org/show_bug.cgi?id=227 The broken sample (Mr_MrsSmith-h264_aac.mp4) the commit was fixing does not work after correcting the off-by-one error. CC: libav-stable@libav.org (cherry picked from commit 8a6037c3900875ccab8d553d2cc659bdef2c9d0e) Signed-off-by: Reinhard Tartler --- libavcodec/h264.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/libavcodec/h264.c b/libavcodec/h264.c index 449c634cfe..46e6c72832 100644 --- a/libavcodec/h264.c +++ b/libavcodec/h264.c @@ -3763,7 +3763,7 @@ static int decode_nal_units(H264Context *h, const uint8_t *buf, int buf_size){ int consumed; int dst_length; int bit_length; - uint8_t *ptr; + const uint8_t *ptr; int i, nalsize = 0; int err; @@ -3813,9 +3813,6 @@ static int decode_nal_units(H264Context *h, const uint8_t *buf, int buf_size){ } if (h->is_avc && (nalsize != consumed) && nalsize){ - // set trailing bits in the last partial byte to zero - if (bit_length & 7) - ptr[bit_length >> 3] = ptr[bit_length >> 3] & (0xff << 8 - (bit_length & 7)); av_log(h->s.avctx, AV_LOG_DEBUG, "AVC: Consumed only %d bytes instead of %d\n", consumed, nalsize); } From 666bd5848a92f82ee97ad0869dd1f8c7edb9f214 Mon Sep 17 00:00:00 2001 From: Anton Khirnov Date: Mon, 12 Mar 2012 17:42:57 +0100 Subject: [PATCH 134/154] avconv: link '-passlogfile' option to libx264 'stats' AVOption. Fixes bug 204. CC: libav-stable@libav.org (cherry picked from commit 6e8be949f12734f38d360aad0f5c503a0f9606fa) Signed-off-by: Reinhard Tartler --- avconv.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/avconv.c b/avconv.c index 46f3090755..4a61fa52fc 100644 --- a/avconv.c +++ b/avconv.c @@ -2459,6 +2459,9 @@ static int transcode_init(OutputFile *output_files, snprintf(logfilename, sizeof(logfilename), "%s-%d.log", pass_logfilename_prefix ? pass_logfilename_prefix : DEFAULT_PASS_LOGFILENAME_PREFIX, i); + if (!strcmp(ost->enc->name, "libx264")) { + av_dict_set(&ost->opts, "stats", logfilename, AV_DICT_DONT_OVERWRITE); + } else { if (codec->flags & CODEC_FLAG_PASS1) { f = fopen(logfilename, "wb"); if (!f) { @@ -2477,6 +2480,7 @@ static int transcode_init(OutputFile *output_files, } codec->stats_in = logbuffer; } + } } } if (codec->codec_type == AVMEDIA_TYPE_VIDEO) { From a15adb18faecaaa984e4a6ef6732f4ea4be2418c Mon Sep 17 00:00:00 2001 From: Anton Khirnov Date: Mon, 12 Mar 2012 17:43:48 +0100 Subject: [PATCH 135/154] avconv: reindent CC: libav-stable@libav.org (cherry picked from commit 64334ddbbc7fce490c895c54106291d0b128e830) Signed-off-by: Reinhard Tartler --- avconv.c | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/avconv.c b/avconv.c index 4a61fa52fc..dcc0935ed7 100644 --- a/avconv.c +++ b/avconv.c @@ -2462,24 +2462,24 @@ static int transcode_init(OutputFile *output_files, if (!strcmp(ost->enc->name, "libx264")) { av_dict_set(&ost->opts, "stats", logfilename, AV_DICT_DONT_OVERWRITE); } else { - if (codec->flags & CODEC_FLAG_PASS1) { - f = fopen(logfilename, "wb"); - if (!f) { - av_log(NULL, AV_LOG_FATAL, "Cannot write log file '%s' for pass-1 encoding: %s\n", - logfilename, strerror(errno)); - exit_program(1); + if (codec->flags & CODEC_FLAG_PASS1) { + f = fopen(logfilename, "wb"); + if (!f) { + av_log(NULL, AV_LOG_FATAL, "Cannot write log file '%s' for pass-1 encoding: %s\n", + logfilename, strerror(errno)); + exit_program(1); + } + ost->logfile = f; + } else { + char *logbuffer; + size_t logbuffer_size; + if (cmdutils_read_file(logfilename, &logbuffer, &logbuffer_size) < 0) { + av_log(NULL, AV_LOG_FATAL, "Error reading log file '%s' for pass-2 encoding\n", + logfilename); + exit_program(1); + } + codec->stats_in = logbuffer; } - ost->logfile = f; - } else { - char *logbuffer; - size_t logbuffer_size; - if (cmdutils_read_file(logfilename, &logbuffer, &logbuffer_size) < 0) { - av_log(NULL, AV_LOG_FATAL, "Error reading log file '%s' for pass-2 encoding\n", - logfilename); - exit_program(1); - } - codec->stats_in = logbuffer; - } } } } From f6257cf4b710eed9f05d9dcbca853d236d3cdd56 Mon Sep 17 00:00:00 2001 From: Anton Khirnov Date: Mon, 12 Mar 2012 17:09:22 +0100 Subject: [PATCH 136/154] libx264: fix help text for slice-max-size option. CC: libav-stable@libav.org (cherry picked from commit 9d5c131ecec75fcfb1b4b56f74f2b2756bf0027a) Signed-off-by: Reinhard Tartler --- libavcodec/libx264.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavcodec/libx264.c b/libavcodec/libx264.c index b3581f168f..96c9588ece 100644 --- a/libavcodec/libx264.c +++ b/libavcodec/libx264.c @@ -552,7 +552,7 @@ static const AVOption options[] = { { "spatial", NULL, 0, AV_OPT_TYPE_CONST, { X264_DIRECT_PRED_SPATIAL }, 0, 0, VE, "direct-pred" }, { "temporal", NULL, 0, AV_OPT_TYPE_CONST, { X264_DIRECT_PRED_TEMPORAL }, 0, 0, VE, "direct-pred" }, { "auto", NULL, 0, AV_OPT_TYPE_CONST, { X264_DIRECT_PRED_AUTO }, 0, 0, VE, "direct-pred" }, - { "slice-max-size","Constant quantization parameter rate control method",OFFSET(slice_max_size), AV_OPT_TYPE_INT, {-1 }, -1, INT_MAX, VE }, + { "slice-max-size","Limit the size of each slice in bytes", OFFSET(slice_max_size),AV_OPT_TYPE_INT, {-1 }, -1, INT_MAX, VE }, { NULL }, }; From 6548cb25782d05c1ea52bb6904c2b2b398079b8b Mon Sep 17 00:00:00 2001 From: Anton Khirnov Date: Mon, 12 Mar 2012 17:20:20 +0100 Subject: [PATCH 137/154] libx264: add 'stats' private option for setting 2pass stats filename. x264 always opens the file itself with fopen, so we cannot use the standard lavc stats mechanism. CC: libav-stable@libav.org (cherry picked from commit d533e395e14d403948ca2424efbcee92429ef8e1) Signed-off-by: Reinhard Tartler --- libavcodec/libx264.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libavcodec/libx264.c b/libavcodec/libx264.c index 96c9588ece..ac0f6b6d05 100644 --- a/libavcodec/libx264.c +++ b/libavcodec/libx264.c @@ -66,6 +66,7 @@ typedef struct X264Context { char *partitions; int direct_pred; int slice_max_size; + char *stats; } X264Context; static void X264_log(void *p, int level, const char *fmt, va_list args) @@ -379,6 +380,7 @@ static av_cold int X264_init(AVCodecContext *avctx) PARSE_X264_OPT("psy-rd", psy_rd); PARSE_X264_OPT("deblock", deblock); PARSE_X264_OPT("partitions", partitions); + PARSE_X264_OPT("stats", stats); if (x4->psy >= 0) x4->params.analyse.b_psy = x4->psy; if (x4->rc_lookahead >= 0) @@ -553,6 +555,7 @@ static const AVOption options[] = { { "temporal", NULL, 0, AV_OPT_TYPE_CONST, { X264_DIRECT_PRED_TEMPORAL }, 0, 0, VE, "direct-pred" }, { "auto", NULL, 0, AV_OPT_TYPE_CONST, { X264_DIRECT_PRED_AUTO }, 0, 0, VE, "direct-pred" }, { "slice-max-size","Limit the size of each slice in bytes", OFFSET(slice_max_size),AV_OPT_TYPE_INT, {-1 }, -1, INT_MAX, VE }, + { "stats", "Filename for 2 pass stats", OFFSET(stats), AV_OPT_TYPE_STRING, { 0 }, 0, 0, VE }, { NULL }, }; From de0ff4ce69c311a2879e10143f1cc2c4945f3ef0 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Mon, 12 Mar 2012 18:26:50 -0700 Subject: [PATCH 138/154] h264: Fix invalid interlaced/progressive MB combinations for direct mode prediction. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org Signed-off-by: Ronald S. Bultje (cherry picked from commit 758ec111538ccd487686e8677aa754ee4d82beaa) Signed-off-by: Reinhard Tartler --- libavcodec/h264_direct.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/libavcodec/h264_direct.c b/libavcodec/h264_direct.c index a953728a12..4f70ff83e9 100644 --- a/libavcodec/h264_direct.c +++ b/libavcodec/h264_direct.c @@ -252,6 +252,10 @@ static void pred_spatial_direct_motion(H264Context * const h, int *mb_type){ mb_type_col[1] = h->ref_list[1][0].f.mb_type[mb_xy + s->mb_stride]; b8_stride = 2+4*s->mb_stride; b4_stride *= 6; + if (IS_INTERLACED(mb_type_col[0]) != IS_INTERLACED(mb_type_col[1])) { + mb_type_col[0] &= ~MB_TYPE_INTERLACED; + mb_type_col[1] &= ~MB_TYPE_INTERLACED; + } sub_mb_type |= MB_TYPE_16x16|MB_TYPE_DIRECT2; /* B_SUB_8x8 */ if( (mb_type_col[0] & MB_TYPE_16x16_OR_INTRA) @@ -438,6 +442,10 @@ static void pred_temp_direct_motion(H264Context * const h, int *mb_type){ mb_type_col[1] = h->ref_list[1][0].f.mb_type[mb_xy + s->mb_stride]; b8_stride = 2+4*s->mb_stride; b4_stride *= 6; + if (IS_INTERLACED(mb_type_col[0]) != IS_INTERLACED(mb_type_col[1])) { + mb_type_col[0] &= ~MB_TYPE_INTERLACED; + mb_type_col[1] &= ~MB_TYPE_INTERLACED; + } sub_mb_type = MB_TYPE_16x16|MB_TYPE_P0L0|MB_TYPE_P0L1|MB_TYPE_DIRECT2; /* B_SUB_8x8 */ From e4e4d92641df31a3c2d5213ac18b9fd5b0c38833 Mon Sep 17 00:00:00 2001 From: Paul B Mahol Date: Wed, 14 Mar 2012 03:02:02 +0000 Subject: [PATCH 139/154] jvdec: unbreak video decoding The safe bitstream reader broke it since the buffer size was specified in bytes instead of bits. Signed-off-by: Janne Grunau CC: libav-stable@libav.org (cherry picked from commit a1c036e961a32f7208e7315dabfa0ee99d779edb) Signed-off-by: Reinhard Tartler --- libavcodec/jvdec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavcodec/jvdec.c b/libavcodec/jvdec.c index 5249764347..f2c97526c0 100644 --- a/libavcodec/jvdec.c +++ b/libavcodec/jvdec.c @@ -150,7 +150,7 @@ static int decode_frame(AVCodecContext *avctx, if (video_type == 0 || video_type == 1) { GetBitContext gb; - init_get_bits(&gb, buf, FFMIN(video_size, (buf_end - buf) * 8)); + init_get_bits(&gb, buf, 8 * FFMIN(video_size, buf_end - buf)); for (j = 0; j < avctx->height; j += 8) for (i = 0; i < avctx->width; i += 8) From f6778f58d4eb9cf47a42506b239586b5d17f84c4 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Tue, 6 Mar 2012 15:15:42 -0800 Subject: [PATCH 140/154] algmm: convert to bytestream2 API. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit a55d5bdc6e28a2cfefc440d792de5cc4f02377e2) Signed-off-by: Reinhard Tartler --- libavcodec/mmvideo.c | 89 +++++++++++++++++++++++++------------------- 1 file changed, 51 insertions(+), 38 deletions(-) diff --git a/libavcodec/mmvideo.c b/libavcodec/mmvideo.c index 9e82ef94f9..501371ad52 100644 --- a/libavcodec/mmvideo.c +++ b/libavcodec/mmvideo.c @@ -33,6 +33,7 @@ #include "libavutil/intreadwrite.h" #include "avcodec.h" +#include "bytestream.h" #define MM_PREAMBLE_SIZE 6 @@ -48,6 +49,7 @@ typedef struct MmContext { AVCodecContext *avctx; AVFrame frame; int palette[AVPALETTE_COUNT]; + GetByteContext gb; } MmContext; static av_cold int mm_decode_init(AVCodecContext *avctx) @@ -63,40 +65,40 @@ static av_cold int mm_decode_init(AVCodecContext *avctx) return 0; } -static void mm_decode_pal(MmContext *s, const uint8_t *buf, const uint8_t *buf_end) +static int mm_decode_pal(MmContext *s) { int i; - buf += 4; - for (i=0; i<128 && buf+2palette[i] = AV_RB24(buf); + + bytestream2_skip(&s->gb, 4); + for (i = 0; i < 128; i++) { + s->palette[i] = bytestream2_get_be24(&s->gb); s->palette[i+128] = s->palette[i]<<2; - buf += 3; } + + return 0; } /** * @param half_horiz Half horizontal resolution (0 or 1) * @param half_vert Half vertical resolution (0 or 1) */ -static void mm_decode_intra(MmContext * s, int half_horiz, int half_vert, const uint8_t *buf, int buf_size) +static int mm_decode_intra(MmContext * s, int half_horiz, int half_vert) { int i, x, y; i=0; x=0; y=0; - while(igb) > 0) { int run_length, color; if (y >= s->avctx->height) - return; + return 0; - if (buf[i] & 0x80) { + color = bytestream2_get_byte(&s->gb); + if (color & 0x80) { run_length = 1; - color = buf[i]; - i++; }else{ - run_length = (buf[i] & 0x7f) + 2; - color = buf[i+1]; - i+=2; + run_length = (color & 0x7f) + 2; + color = bytestream2_get_byte(&s->gb); } if (half_horiz) @@ -114,23 +116,28 @@ static void mm_decode_intra(MmContext * s, int half_horiz, int half_vert, const y += 1 + half_vert; } } + + return 0; } /* * @param half_horiz Half horizontal resolution (0 or 1) * @param half_vert Half vertical resolution (0 or 1) */ -static void mm_decode_inter(MmContext * s, int half_horiz, int half_vert, const uint8_t *buf, int buf_size) +static int mm_decode_inter(MmContext * s, int half_horiz, int half_vert) { - const int data_ptr = 2 + AV_RL16(&buf[0]); - int d, r, y; - d = data_ptr; r = 2; y = 0; + int data_off = bytestream2_get_le16(&s->gb), y; + GetByteContext data_ptr; - while(r < data_ptr) { + if (bytestream2_get_bytes_left(&s->gb) < data_off) + return AVERROR_INVALIDDATA; + + bytestream2_init(&data_ptr, s->gb.buffer + data_off, bytestream2_get_bytes_left(&s->gb) - data_off); + while (s->gb.buffer < data_ptr.buffer_start) { int i, j; - int length = buf[r] & 0x7f; - int x = buf[r+1] + ((buf[r] & 0x80) << 1); - r += 2; + int length = bytestream2_get_byte(&s->gb); + int x = bytestream2_get_byte(&s->gb) + ((length & 0x80) << 1); + length &= 0x7F; if (length==0) { y += x; @@ -138,13 +145,14 @@ static void mm_decode_inter(MmContext * s, int half_horiz, int half_vert, const } if (y + half_vert >= s->avctx->height) - return; + return 0; for(i=0; igb); for(j=0; j<8; j++) { - int replace = (buf[r+i] >> (7-j)) & 1; + int replace = (replace_array >> (7-j)) & 1; if (replace) { - int color = buf[d]; + int color = bytestream2_get_byte(&data_ptr); s->frame.data[0][y*s->frame.linesize[0] + x] = color; if (half_horiz) s->frame.data[0][y*s->frame.linesize[0] + x + 1] = color; @@ -153,15 +161,15 @@ static void mm_decode_inter(MmContext * s, int half_horiz, int half_vert, const if (half_horiz) s->frame.data[0][(y+1)*s->frame.linesize[0] + x + 1] = color; } - d++; } x += 1 + half_horiz; } } - r += length; y += 1 + half_vert; } + + return 0; } static int mm_decode_frame(AVCodecContext *avctx, @@ -171,12 +179,14 @@ static int mm_decode_frame(AVCodecContext *avctx, const uint8_t *buf = avpkt->data; int buf_size = avpkt->size; MmContext *s = avctx->priv_data; - const uint8_t *buf_end = buf+buf_size; - int type; + int type, res; + if (buf_size < MM_PREAMBLE_SIZE) + return AVERROR_INVALIDDATA; type = AV_RL16(&buf[0]); buf += MM_PREAMBLE_SIZE; buf_size -= MM_PREAMBLE_SIZE; + bytestream2_init(&s->gb, buf, buf_size); if (avctx->reget_buffer(avctx, &s->frame) < 0) { av_log(avctx, AV_LOG_ERROR, "reget_buffer() failed\n"); @@ -184,16 +194,19 @@ static int mm_decode_frame(AVCodecContext *avctx, } switch(type) { - case MM_TYPE_PALETTE : mm_decode_pal(s, buf, buf_end); return buf_size; - case MM_TYPE_INTRA : mm_decode_intra(s, 0, 0, buf, buf_size); break; - case MM_TYPE_INTRA_HH : mm_decode_intra(s, 1, 0, buf, buf_size); break; - case MM_TYPE_INTRA_HHV : mm_decode_intra(s, 1, 1, buf, buf_size); break; - case MM_TYPE_INTER : mm_decode_inter(s, 0, 0, buf, buf_size); break; - case MM_TYPE_INTER_HH : mm_decode_inter(s, 1, 0, buf, buf_size); break; - case MM_TYPE_INTER_HHV : mm_decode_inter(s, 1, 1, buf, buf_size); break; - default : - return -1; + case MM_TYPE_PALETTE : res = mm_decode_pal(s); return buf_size; + case MM_TYPE_INTRA : res = mm_decode_intra(s, 0, 0); break; + case MM_TYPE_INTRA_HH : res = mm_decode_intra(s, 1, 0); break; + case MM_TYPE_INTRA_HHV : res = mm_decode_intra(s, 1, 1); break; + case MM_TYPE_INTER : res = mm_decode_inter(s, 0, 0); break; + case MM_TYPE_INTER_HH : res = mm_decode_inter(s, 1, 0); break; + case MM_TYPE_INTER_HHV : res = mm_decode_inter(s, 1, 1); break; + default: + res = AVERROR_INVALIDDATA; + break; } + if (res < 0) + return res; memcpy(s->frame.data[1], s->palette, AVPALETTE_SIZE); From ddb1149e250faceb91c220b2c032b27e60c1b417 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Tue, 6 Mar 2012 14:18:32 -0800 Subject: [PATCH 141/154] tgq: convert to bytestream2 API. This protects against input buffer overreads. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 1255eed533b4069db7f205601953ca54c0dc42c9) Signed-off-by: Reinhard Tartler --- libavcodec/eatgq.c | 56 +++++++++++++++++++++++----------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/libavcodec/eatgq.c b/libavcodec/eatgq.c index e82ed32470..5be1f55ba3 100644 --- a/libavcodec/eatgq.c +++ b/libavcodec/eatgq.c @@ -43,6 +43,7 @@ typedef struct TgqContext { ScanTable scantable; int qtable[64]; DECLARE_ALIGNED(16, DCTELEM, block)[6][64]; + GetByteContext gb; } TgqContext; static av_cold int tgq_decode_init(AVCodecContext *avctx){ @@ -141,39 +142,36 @@ static void tgq_idct_put_mb_dconly(TgqContext *s, int mb_x, int mb_y, const int8 } } -static void tgq_decode_mb(TgqContext *s, int mb_y, int mb_x, const uint8_t **bs, const uint8_t *buf_end){ +static void tgq_decode_mb(TgqContext *s, int mb_y, int mb_x){ int mode; int i; int8_t dc[6]; - mode = bytestream_get_byte(bs); - if (mode>buf_end-*bs) { - av_log(s->avctx, AV_LOG_ERROR, "truncated macroblock\n"); - return; - } - + mode = bytestream2_get_byte(&s->gb); if (mode>12) { GetBitContext gb; - init_get_bits(&gb, *bs, mode*8); + init_get_bits(&gb, s->gb.buffer, FFMIN(s->gb.buffer_end - s->gb.buffer, mode) * 8); for(i=0; i<6; i++) tgq_decode_block(s, s->block[i], &gb); tgq_idct_put_mb(s, s->block, mb_x, mb_y); + bytestream2_skip(&s->gb, mode); }else{ if (mode==3) { - memset(dc, (*bs)[0], 4); - dc[4] = (*bs)[1]; - dc[5] = (*bs)[2]; + memset(dc, bytestream2_get_byte(&s->gb), 4); + dc[4] = bytestream2_get_byte(&s->gb); + dc[5] = bytestream2_get_byte(&s->gb); }else if (mode==6) { - memcpy(dc, *bs, 6); + bytestream2_get_buffer(&s->gb, dc, 6); }else if (mode==12) { - for(i=0; i<6; i++) - dc[i] = (*bs)[i*2]; + for (i = 0; i < 6; i++) { + dc[i] = bytestream2_get_byte(&s->gb); + bytestream2_skip(&s->gb, 1); + } }else{ av_log(s->avctx, AV_LOG_ERROR, "unsupported mb mode %i\n", mode); } tgq_idct_put_mb_dconly(s, mb_x, mb_y, dc); } - *bs += mode; } static void tgq_calculate_qtable(TgqContext *s, int quant){ @@ -193,28 +191,30 @@ static int tgq_decode_frame(AVCodecContext *avctx, AVPacket *avpkt){ const uint8_t *buf = avpkt->data; int buf_size = avpkt->size; - const uint8_t *buf_start = buf; - const uint8_t *buf_end = buf + buf_size; TgqContext *s = avctx->priv_data; int x,y; - int big_endian = AV_RL32(&buf[4]) > 0x000FFFFF; - buf += 8; - if(8>buf_end-buf) { + if (buf_size < 16) { av_log(avctx, AV_LOG_WARNING, "truncated header\n"); return -1; } - s->width = big_endian ? AV_RB16(&buf[0]) : AV_RL16(&buf[0]); - s->height = big_endian ? AV_RB16(&buf[2]) : AV_RL16(&buf[2]); + bytestream2_init(&s->gb, buf + 8, buf_size - 8); + if (big_endian) { + s->width = bytestream2_get_be16u(&s->gb); + s->height = bytestream2_get_be16u(&s->gb); + } else { + s->width = bytestream2_get_le16u(&s->gb); + s->height = bytestream2_get_le16u(&s->gb); + } if (s->avctx->width!=s->width || s->avctx->height!=s->height) { avcodec_set_dimensions(s->avctx, s->width, s->height); if (s->frame.data[0]) avctx->release_buffer(avctx, &s->frame); } - tgq_calculate_qtable(s, buf[4]); - buf += 8; + tgq_calculate_qtable(s, bytestream2_get_byteu(&s->gb)); + bytestream2_skip(&s->gb, 3); if (!s->frame.data[0]) { s->frame.key_frame = 1; @@ -226,14 +226,14 @@ static int tgq_decode_frame(AVCodecContext *avctx, } } - for (y=0; y<(avctx->height+15)/16; y++) - for (x=0; x<(avctx->width+15)/16; x++) - tgq_decode_mb(s, y, x, &buf, buf_end); + for (y = 0; y < FFALIGN(avctx->height, 16) >> 4; y++) + for (x = 0; x < FFALIGN(avctx->width, 16) >> 4; x++) + tgq_decode_mb(s, y, x); *data_size = sizeof(AVFrame); *(AVFrame*)data = s->frame; - return buf-buf_start; + return avpkt->size; } static av_cold int tgq_decode_end(AVCodecContext *avctx){ From 9a66cdbc16806dc61272a81c4ea261d44cd2d41a Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Wed, 29 Feb 2012 14:44:37 -0800 Subject: [PATCH 142/154] smc: port to bytestream2 API. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 8febcb9fc178926687ee19d32d2b3150da899867) Signed-off-by: Reinhard Tartler --- libavcodec/smc.c | 74 +++++++++++++++++++++--------------------------- 1 file changed, 32 insertions(+), 42 deletions(-) diff --git a/libavcodec/smc.c b/libavcodec/smc.c index f4a0b6a6a9..2bd3176f8e 100644 --- a/libavcodec/smc.c +++ b/libavcodec/smc.c @@ -34,6 +34,7 @@ #include "libavutil/intreadwrite.h" #include "avcodec.h" +#include "bytestream.h" #define CPAIR 2 #define CQUAD 4 @@ -46,8 +47,7 @@ typedef struct SmcContext { AVCodecContext *avctx; AVFrame frame; - const unsigned char *buf; - int size; + GetByteContext gb; /* SMC color tables */ unsigned char color_pairs[COLORS_PER_TABLE * CPAIR]; @@ -58,7 +58,7 @@ typedef struct SmcContext { } SmcContext; #define GET_BLOCK_COUNT() \ - (opcode & 0x10) ? (1 + s->buf[stream_ptr++]) : 1 + (opcode & 0x0F); + (opcode & 0x10) ? (1 + bytestream2_get_byte(&s->gb)) : 1 + (opcode & 0x0F); #define ADVANCE_BLOCK() \ { \ @@ -82,8 +82,8 @@ static void smc_decode_stream(SmcContext *s) int height = s->avctx->height; int stride = s->frame.linesize[0]; int i; - int stream_ptr = 0; int chunk_size; + int buf_size = (int) (s->gb.buffer_end - s->gb.buffer_start); unsigned char opcode; int n_blocks; unsigned int color_flags; @@ -113,24 +113,18 @@ static void smc_decode_stream(SmcContext *s) /* make the palette available */ memcpy(s->frame.data[1], s->pal, AVPALETTE_SIZE); - chunk_size = AV_RB32(&s->buf[stream_ptr]) & 0x00FFFFFF; - stream_ptr += 4; - if (chunk_size != s->size) + bytestream2_skip(&s->gb, 1); + chunk_size = bytestream2_get_be24(&s->gb); + if (chunk_size != buf_size) av_log(s->avctx, AV_LOG_INFO, "warning: MOV chunk size != encoded chunk size (%d != %d); using MOV chunk size\n", - chunk_size, s->size); + chunk_size, buf_size); - chunk_size = s->size; + chunk_size = buf_size; total_blocks = ((s->avctx->width + 3) / 4) * ((s->avctx->height + 3) / 4); /* traverse through the blocks */ while (total_blocks) { /* sanity checks */ - /* make sure stream ptr hasn't gone out of bounds */ - if (stream_ptr > chunk_size) { - av_log(s->avctx, AV_LOG_INFO, "SMC decoder just went out of bounds (stream ptr = %d, chunk size = %d)\n", - stream_ptr, chunk_size); - return; - } /* make sure the row pointer hasn't gone wild */ if (row_ptr >= image_size) { av_log(s->avctx, AV_LOG_INFO, "SMC decoder just went out of bounds (row ptr = %d, height = %d)\n", @@ -138,7 +132,7 @@ static void smc_decode_stream(SmcContext *s) return; } - opcode = s->buf[stream_ptr++]; + opcode = bytestream2_get_byte(&s->gb); switch (opcode & 0xF0) { /* skip n blocks */ case 0x00: @@ -158,7 +152,7 @@ static void smc_decode_stream(SmcContext *s) if ((row_ptr == 0) && (pixel_ptr == 0)) { av_log(s->avctx, AV_LOG_INFO, "encountered repeat block opcode (%02X) but no blocks rendered yet\n", opcode & 0xF0); - break; + return; } /* figure out where the previous block started */ @@ -192,7 +186,7 @@ static void smc_decode_stream(SmcContext *s) if ((row_ptr == 0) && (pixel_ptr < 2 * 4)) { av_log(s->avctx, AV_LOG_INFO, "encountered repeat block opcode (%02X) but not enough blocks rendered yet\n", opcode & 0xF0); - break; + return; } /* figure out where the previous 2 blocks started */ @@ -233,7 +227,7 @@ static void smc_decode_stream(SmcContext *s) case 0x60: case 0x70: n_blocks = GET_BLOCK_COUNT(); - pixel = s->buf[stream_ptr++]; + pixel = bytestream2_get_byte(&s->gb); while (n_blocks--) { block_ptr = row_ptr + pixel_ptr; @@ -257,7 +251,7 @@ static void smc_decode_stream(SmcContext *s) /* fetch the next 2 colors from bytestream and store in next * available entry in the color pair table */ for (i = 0; i < CPAIR; i++) { - pixel = s->buf[stream_ptr++]; + pixel = bytestream2_get_byte(&s->gb); color_table_index = CPAIR * color_pair_index + i; s->color_pairs[color_table_index] = pixel; } @@ -268,11 +262,10 @@ static void smc_decode_stream(SmcContext *s) if (color_pair_index == COLORS_PER_TABLE) color_pair_index = 0; } else - color_table_index = CPAIR * s->buf[stream_ptr++]; + color_table_index = CPAIR * bytestream2_get_byte(&s->gb); while (n_blocks--) { - color_flags = AV_RB16(&s->buf[stream_ptr]); - stream_ptr += 2; + color_flags = bytestream2_get_be16(&s->gb); flag_mask = 0x8000; block_ptr = row_ptr + pixel_ptr; for (pixel_y = 0; pixel_y < 4; pixel_y++) { @@ -300,7 +293,7 @@ static void smc_decode_stream(SmcContext *s) /* fetch the next 4 colors from bytestream and store in next * available entry in the color quad table */ for (i = 0; i < CQUAD; i++) { - pixel = s->buf[stream_ptr++]; + pixel = bytestream2_get_byte(&s->gb); color_table_index = CQUAD * color_quad_index + i; s->color_quads[color_table_index] = pixel; } @@ -311,11 +304,10 @@ static void smc_decode_stream(SmcContext *s) if (color_quad_index == COLORS_PER_TABLE) color_quad_index = 0; } else - color_table_index = CQUAD * s->buf[stream_ptr++]; + color_table_index = CQUAD * bytestream2_get_byte(&s->gb); while (n_blocks--) { - color_flags = AV_RB32(&s->buf[stream_ptr]); - stream_ptr += 4; + color_flags = bytestream2_get_be32(&s->gb); /* flag mask actually acts as a bit shift count here */ flag_mask = 30; block_ptr = row_ptr + pixel_ptr; @@ -342,7 +334,7 @@ static void smc_decode_stream(SmcContext *s) /* fetch the next 8 colors from bytestream and store in next * available entry in the color octet table */ for (i = 0; i < COCTET; i++) { - pixel = s->buf[stream_ptr++]; + pixel = bytestream2_get_byte(&s->gb); color_table_index = COCTET * color_octet_index + i; s->color_octets[color_table_index] = pixel; } @@ -353,7 +345,7 @@ static void smc_decode_stream(SmcContext *s) if (color_octet_index == COLORS_PER_TABLE) color_octet_index = 0; } else - color_table_index = COCTET * s->buf[stream_ptr++]; + color_table_index = COCTET * bytestream2_get_byte(&s->gb); while (n_blocks--) { /* @@ -363,15 +355,12 @@ static void smc_decode_stream(SmcContext *s) flags_a = xx012456, flags_b = xx89A37B */ /* build the color flags */ - color_flags_a = - ((AV_RB16(s->buf + stream_ptr ) & 0xFFF0) << 8) | - (AV_RB16(s->buf + stream_ptr + 2) >> 4); - color_flags_b = - ((AV_RB16(s->buf + stream_ptr + 4) & 0xFFF0) << 8) | - ((s->buf[stream_ptr + 1] & 0x0F) << 8) | - ((s->buf[stream_ptr + 3] & 0x0F) << 4) | - (s->buf[stream_ptr + 5] & 0x0F); - stream_ptr += 6; + int val1 = bytestream2_get_be16(&s->gb); + int val2 = bytestream2_get_be16(&s->gb); + int val3 = bytestream2_get_be16(&s->gb); + color_flags_a = ((val1 & 0xFFF0) << 8) | (val2 >> 4); + color_flags_b = ((val3 & 0xFFF0) << 8) | + ((val1 & 0x0F) << 8) | ((val2 & 0x0F) << 4) | (val3 & 0x0F); color_flags = color_flags_a; /* flag mask actually acts as a bit shift count here */ @@ -403,7 +392,7 @@ static void smc_decode_stream(SmcContext *s) block_ptr = row_ptr + pixel_ptr; for (pixel_y = 0; pixel_y < 4; pixel_y++) { for (pixel_x = 0; pixel_x < 4; pixel_x++) { - pixels[block_ptr++] = s->buf[stream_ptr++]; + pixels[block_ptr++] = bytestream2_get_byte(&s->gb); } block_ptr += row_inc; } @@ -412,10 +401,12 @@ static void smc_decode_stream(SmcContext *s) break; case 0xF0: - av_log(s->avctx, AV_LOG_INFO, "0xF0 opcode seen in SMC chunk (contact the developers)\n"); + av_log_missing_feature(s->avctx, "0xF0 opcode", 1); break; } } + + return; } static av_cold int smc_decode_init(AVCodecContext *avctx) @@ -439,8 +430,7 @@ static int smc_decode_frame(AVCodecContext *avctx, SmcContext *s = avctx->priv_data; const uint8_t *pal = av_packet_get_side_data(avpkt, AV_PKT_DATA_PALETTE, NULL); - s->buf = buf; - s->size = buf_size; + bytestream2_init(&s->gb, buf, buf_size); s->frame.reference = 1; s->frame.buffer_hints = FF_BUFFER_HINTS_VALID | FF_BUFFER_HINTS_PRESERVE | From 568a474a0831a6224ae1886187fd7f4a74328215 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Tue, 6 Mar 2012 15:58:35 -0800 Subject: [PATCH 143/154] roqvideo: convert to bytestream2 API. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit cdf15771621bce7959b3e53b21426c5ba747e17b) Signed-off-by: Reinhard Tartler --- libavcodec/roqvideo.h | 4 +-- libavcodec/roqvideodec.c | 69 ++++++++++++++++++++-------------------- 2 files changed, 37 insertions(+), 36 deletions(-) diff --git a/libavcodec/roqvideo.h b/libavcodec/roqvideo.h index a1ff10af7d..4e6e5bb7db 100644 --- a/libavcodec/roqvideo.h +++ b/libavcodec/roqvideo.h @@ -24,6 +24,7 @@ #include "libavutil/lfg.h" #include "avcodec.h" +#include "bytestream.h" #include "dsputil.h" typedef struct { @@ -53,8 +54,7 @@ typedef struct RoqContext { roq_cell cb2x2[256]; roq_qcell cb4x4[256]; - const unsigned char *buf; - int size; + GetByteContext gb; int width, height; /* Encoder only data */ diff --git a/libavcodec/roqvideodec.c b/libavcodec/roqvideodec.c index 527ba51c99..0bf00cf380 100644 --- a/libavcodec/roqvideodec.c +++ b/libavcodec/roqvideodec.c @@ -38,16 +38,15 @@ static void roqvideo_decode_frame(RoqContext *ri) unsigned int chunk_id = 0, chunk_arg = 0; unsigned long chunk_size = 0; int i, j, k, nv1, nv2, vqflg = 0, vqflg_pos = -1; - int vqid, bpos, xpos, ypos, xp, yp, x, y, mx, my; + int vqid, xpos, ypos, xp, yp, x, y, mx, my; int frame_stats[2][4] = {{0},{0}}; roq_qcell *qcell; - const unsigned char *buf = ri->buf; - const unsigned char *buf_end = ri->buf + ri->size; + int64_t chunk_start; - while (buf < buf_end) { - chunk_id = bytestream_get_le16(&buf); - chunk_size = bytestream_get_le32(&buf); - chunk_arg = bytestream_get_le16(&buf); + while (bytestream2_get_bytes_left(&ri->gb) > 0) { + chunk_id = bytestream2_get_le16(&ri->gb); + chunk_size = bytestream2_get_le32(&ri->gb); + chunk_arg = bytestream2_get_le16(&ri->gb); if(chunk_id == RoQ_QUAD_VQ) break; @@ -57,25 +56,26 @@ static void roqvideo_decode_frame(RoqContext *ri) if((nv2 = chunk_arg & 0xff) == 0 && nv1 * 6 < chunk_size) nv2 = 256; for(i = 0; i < nv1; i++) { - ri->cb2x2[i].y[0] = *buf++; - ri->cb2x2[i].y[1] = *buf++; - ri->cb2x2[i].y[2] = *buf++; - ri->cb2x2[i].y[3] = *buf++; - ri->cb2x2[i].u = *buf++; - ri->cb2x2[i].v = *buf++; + ri->cb2x2[i].y[0] = bytestream2_get_byte(&ri->gb); + ri->cb2x2[i].y[1] = bytestream2_get_byte(&ri->gb); + ri->cb2x2[i].y[2] = bytestream2_get_byte(&ri->gb); + ri->cb2x2[i].y[3] = bytestream2_get_byte(&ri->gb); + ri->cb2x2[i].u = bytestream2_get_byte(&ri->gb); + ri->cb2x2[i].v = bytestream2_get_byte(&ri->gb); } for(i = 0; i < nv2; i++) for(j = 0; j < 4; j++) - ri->cb4x4[i].idx[j] = *buf++; + ri->cb4x4[i].idx[j] = bytestream2_get_byte(&ri->gb); } } - bpos = xpos = ypos = 0; - while(bpos < chunk_size) { + chunk_start = bytestream2_tell(&ri->gb); + xpos = ypos = 0; + while (bytestream2_tell(&ri->gb) < chunk_start + chunk_size) { for (yp = ypos; yp < ypos + 16; yp += 8) for (xp = xpos; xp < xpos + 16; xp += 8) { if (vqflg_pos < 0) { - vqflg = buf[bpos++]; vqflg |= (buf[bpos++] << 8); + vqflg = bytestream2_get_le16(&ri->gb); vqflg_pos = 7; } vqid = (vqflg >> (vqflg_pos * 2)) & 0x3; @@ -85,13 +85,15 @@ static void roqvideo_decode_frame(RoqContext *ri) switch(vqid) { case RoQ_ID_MOT: break; - case RoQ_ID_FCC: - mx = 8 - (buf[bpos] >> 4) - ((signed char) (chunk_arg >> 8)); - my = 8 - (buf[bpos++] & 0xf) - ((signed char) chunk_arg); + case RoQ_ID_FCC: { + int byte = bytestream2_get_byte(&ri->gb); + mx = 8 - (byte >> 4) - ((signed char) (chunk_arg >> 8)); + my = 8 - (byte & 0xf) - ((signed char) chunk_arg); ff_apply_motion_8x8(ri, xp, yp, mx, my); break; + } case RoQ_ID_SLD: - qcell = ri->cb4x4 + buf[bpos++]; + qcell = ri->cb4x4 + bytestream2_get_byte(&ri->gb); ff_apply_vector_4x4(ri, xp, yp, ri->cb2x2 + qcell->idx[0]); ff_apply_vector_4x4(ri, xp+4, yp, ri->cb2x2 + qcell->idx[1]); ff_apply_vector_4x4(ri, xp, yp+4, ri->cb2x2 + qcell->idx[2]); @@ -104,8 +106,7 @@ static void roqvideo_decode_frame(RoqContext *ri) if(k & 0x02) y += 4; if (vqflg_pos < 0) { - vqflg = buf[bpos++]; - vqflg |= (buf[bpos++] << 8); + vqflg = bytestream2_get_le16(&ri->gb); vqflg_pos = 7; } vqid = (vqflg >> (vqflg_pos * 2)) & 0x3; @@ -114,24 +115,25 @@ static void roqvideo_decode_frame(RoqContext *ri) switch(vqid) { case RoQ_ID_MOT: break; - case RoQ_ID_FCC: - mx = 8 - (buf[bpos] >> 4) - ((signed char) (chunk_arg >> 8)); - my = 8 - (buf[bpos++] & 0xf) - ((signed char) chunk_arg); + case RoQ_ID_FCC: { + int byte = bytestream2_get_byte(&ri->gb); + mx = 8 - (byte >> 4) - ((signed char) (chunk_arg >> 8)); + my = 8 - (byte & 0xf) - ((signed char) chunk_arg); ff_apply_motion_4x4(ri, x, y, mx, my); break; + } case RoQ_ID_SLD: - qcell = ri->cb4x4 + buf[bpos++]; + qcell = ri->cb4x4 + bytestream2_get_byte(&ri->gb); ff_apply_vector_2x2(ri, x, y, ri->cb2x2 + qcell->idx[0]); ff_apply_vector_2x2(ri, x+2, y, ri->cb2x2 + qcell->idx[1]); ff_apply_vector_2x2(ri, x, y+2, ri->cb2x2 + qcell->idx[2]); ff_apply_vector_2x2(ri, x+2, y+2, ri->cb2x2 + qcell->idx[3]); break; case RoQ_ID_CCC: - ff_apply_vector_2x2(ri, x, y, ri->cb2x2 + buf[bpos]); - ff_apply_vector_2x2(ri, x+2, y, ri->cb2x2 + buf[bpos+1]); - ff_apply_vector_2x2(ri, x, y+2, ri->cb2x2 + buf[bpos+2]); - ff_apply_vector_2x2(ri, x+2, y+2, ri->cb2x2 + buf[bpos+3]); - bpos += 4; + ff_apply_vector_2x2(ri, x, y, ri->cb2x2 + bytestream2_get_byte(&ri->gb)); + ff_apply_vector_2x2(ri, x+2, y, ri->cb2x2 + bytestream2_get_byte(&ri->gb)); + ff_apply_vector_2x2(ri, x, y+2, ri->cb2x2 + bytestream2_get_byte(&ri->gb)); + ff_apply_vector_2x2(ri, x+2, y+2, ri->cb2x2 + bytestream2_get_byte(&ri->gb)); break; } } @@ -185,8 +187,7 @@ static int roq_decode_frame(AVCodecContext *avctx, av_picture_copy((AVPicture*)s->current_frame, (AVPicture*)s->last_frame, avctx->pix_fmt, avctx->width, avctx->height); - s->buf = buf; - s->size = buf_size; + bytestream2_init(&s->gb, buf, buf_size); roqvideo_decode_frame(s); *data_size = sizeof(AVFrame); From d26e47bf6c7df8b4d74dc2ba818d17e6e2fa839f Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Wed, 7 Mar 2012 16:16:20 -0800 Subject: [PATCH 144/154] png: convert to bytestream2 API. Protects against overreads in the input buffer. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 4c25269cedd042abcb823c42d33609564861c374) Signed-off-by: Reinhard Tartler --- libavcodec/pngdec.c | 69 ++++++++++++++++++++------------------------- 1 file changed, 30 insertions(+), 39 deletions(-) diff --git a/libavcodec/pngdec.c b/libavcodec/pngdec.c index a40cebbb83..94eb6ebeed 100644 --- a/libavcodec/pngdec.c +++ b/libavcodec/pngdec.c @@ -35,9 +35,7 @@ typedef struct PNGDecContext { DSPContext dsp; - const uint8_t *bytestream; - const uint8_t *bytestream_start; - const uint8_t *bytestream_end; + GetByteContext gb; AVFrame picture1, picture2; AVFrame *current_picture, *last_picture; @@ -362,12 +360,9 @@ static void png_handle_row(PNGDecContext *s) static int png_decode_idat(PNGDecContext *s, int length) { int ret; - s->zstream.avail_in = length; - s->zstream.next_in = s->bytestream; - s->bytestream += length; - - if(s->bytestream > s->bytestream_end) - return -1; + s->zstream.avail_in = FFMIN(length, bytestream2_get_bytes_left(&s->gb)); + s->zstream.next_in = s->gb.buffer; + bytestream2_skip(&s->gb, length); /* decode one line if possible */ while (s->zstream.avail_in > 0) { @@ -403,15 +398,13 @@ static int decode_frame(AVCodecContext *avctx, avctx->coded_frame= s->current_picture; p = s->current_picture; - s->bytestream_start= - s->bytestream= buf; - s->bytestream_end= buf + buf_size; - /* check signature */ - if (memcmp(s->bytestream, ff_pngsig, 8) != 0 && - memcmp(s->bytestream, ff_mngsig, 8) != 0) + if (buf_size < 8 || + memcmp(buf, ff_pngsig, 8) != 0 && + memcmp(buf, ff_mngsig, 8) != 0) return -1; - s->bytestream+= 8; + + bytestream2_init(&s->gb, buf + 8, buf_size - 8); s->y= s->state=0; // memset(s, 0, sizeof(PNGDecContext)); @@ -423,14 +416,12 @@ static int decode_frame(AVCodecContext *avctx, if (ret != Z_OK) return -1; for(;;) { - int tag32; - if (s->bytestream >= s->bytestream_end) + if (bytestream2_get_bytes_left(&s->gb) <= 0) goto fail; - length = bytestream_get_be32(&s->bytestream); + length = bytestream2_get_be32(&s->gb); if (length > 0x7fffffff) goto fail; - tag32 = bytestream_get_be32(&s->bytestream); - tag = av_bswap32(tag32); + tag = bytestream2_get_le32(&s->gb); av_dlog(avctx, "png: tag=%c%c%c%c length=%u\n", (tag & 0xff), ((tag >> 8) & 0xff), @@ -440,18 +431,18 @@ static int decode_frame(AVCodecContext *avctx, case MKTAG('I', 'H', 'D', 'R'): if (length != 13) goto fail; - s->width = bytestream_get_be32(&s->bytestream); - s->height = bytestream_get_be32(&s->bytestream); + s->width = bytestream2_get_be32(&s->gb); + s->height = bytestream2_get_be32(&s->gb); if(av_image_check_size(s->width, s->height, 0, avctx)){ s->width= s->height= 0; goto fail; } - s->bit_depth = *s->bytestream++; - s->color_type = *s->bytestream++; - s->compression_type = *s->bytestream++; - s->filter_type = *s->bytestream++; - s->interlace_type = *s->bytestream++; - s->bytestream += 4; /* crc */ + s->bit_depth = bytestream2_get_byte(&s->gb); + s->color_type = bytestream2_get_byte(&s->gb); + s->compression_type = bytestream2_get_byte(&s->gb); + s->filter_type = bytestream2_get_byte(&s->gb); + s->interlace_type = bytestream2_get_byte(&s->gb); + bytestream2_skip(&s->gb, 4); /* crc */ s->state |= PNG_IHDR; av_dlog(avctx, "width=%d height=%d depth=%d color_type=%d compression_type=%d filter_type=%d interlace_type=%d\n", s->width, s->height, s->bit_depth, s->color_type, @@ -547,7 +538,7 @@ static int decode_frame(AVCodecContext *avctx, s->state |= PNG_IDAT; if (png_decode_idat(s, length) < 0) goto fail; - s->bytestream += 4; /* crc */ + bytestream2_skip(&s->gb, 4); /* crc */ break; case MKTAG('P', 'L', 'T', 'E'): { @@ -558,16 +549,16 @@ static int decode_frame(AVCodecContext *avctx, /* read the palette */ n = length / 3; for(i=0;ibytestream++; - g = *s->bytestream++; - b = *s->bytestream++; + r = bytestream2_get_byte(&s->gb); + g = bytestream2_get_byte(&s->gb); + b = bytestream2_get_byte(&s->gb); s->palette[i] = (0xff << 24) | (r << 16) | (g << 8) | b; } for(;i<256;i++) { s->palette[i] = (0xff << 24); } s->state |= PNG_PLTE; - s->bytestream += 4; /* crc */ + bytestream2_skip(&s->gb, 4); /* crc */ } break; case MKTAG('t', 'R', 'N', 'S'): @@ -580,21 +571,21 @@ static int decode_frame(AVCodecContext *avctx, !(s->state & PNG_PLTE)) goto skip_tag; for(i=0;ibytestream++; + v = bytestream2_get_byte(&s->gb); s->palette[i] = (s->palette[i] & 0x00ffffff) | (v << 24); } - s->bytestream += 4; /* crc */ + bytestream2_skip(&s->gb, 4); /* crc */ } break; case MKTAG('I', 'E', 'N', 'D'): if (!(s->state & PNG_ALLIMAGE)) goto fail; - s->bytestream += 4; /* crc */ + bytestream2_skip(&s->gb, 4); /* crc */ goto exit_loop; default: /* skip tag */ skip_tag: - s->bytestream += length + 4; + bytestream2_skip(&s->gb, length + 4); break; } } @@ -619,7 +610,7 @@ static int decode_frame(AVCodecContext *avctx, *picture= *s->current_picture; *data_size = sizeof(AVFrame); - ret = s->bytestream - s->bytestream_start; + ret = bytestream2_tell(&s->gb); the_end: inflateEnd(&s->zstream); av_free(crow_buf_base); From 48f0eeb2e519882da9fe156abaa95cc808b67a8b Mon Sep 17 00:00:00 2001 From: Alex Converse Date: Sun, 4 Mar 2012 17:53:50 -0800 Subject: [PATCH 145/154] Replace computations of remaining bits with calls to get_bits_left(). (cherry picked from commit 3574a85ce57366ba7429edef93d5cad8640fb68c) Signed-off-by: Reinhard Tartler --- libavcodec/escape124.c | 2 +- libavcodec/h261dec.c | 2 +- libavcodec/h263dec.c | 2 +- libavcodec/h264.c | 6 +++--- libavcodec/h264_ps.c | 4 ++-- libavcodec/h264_sei.c | 2 +- libavcodec/huffyuv.c | 4 ++-- libavcodec/ituh263dec.c | 4 ++-- libavcodec/mjpegdec.c | 9 ++++----- libavcodec/vp6.c | 2 +- 10 files changed, 18 insertions(+), 19 deletions(-) diff --git a/libavcodec/escape124.c b/libavcodec/escape124.c index f6d7c8268e..9efafdbf12 100644 --- a/libavcodec/escape124.c +++ b/libavcodec/escape124.c @@ -49,7 +49,7 @@ typedef struct Escape124Context { } Escape124Context; static int can_safely_read(GetBitContext* gb, int bits) { - return get_bits_count(gb) + bits <= gb->size_in_bits; + return get_bits_left(gb) >= bits; } /** diff --git a/libavcodec/h261dec.c b/libavcodec/h261dec.c index 66ea4be2a1..0be0134f01 100644 --- a/libavcodec/h261dec.c +++ b/libavcodec/h261dec.c @@ -265,7 +265,7 @@ static int h261_decode_mb(H261Context *h){ while( h->mba_diff == MBA_STUFFING ); // stuffing if ( h->mba_diff < 0 ){ - if ( get_bits_count(&s->gb) + 7 >= s->gb.size_in_bits ) + if (get_bits_left(&s->gb) <= 7) return SLICE_END; av_log(s->avctx, AV_LOG_ERROR, "illegal mba at %d %d\n", s->mb_x, s->mb_y); diff --git a/libavcodec/h263dec.c b/libavcodec/h263dec.c index ba0ea4f9f9..55562148cf 100644 --- a/libavcodec/h263dec.c +++ b/libavcodec/h263dec.c @@ -650,7 +650,7 @@ retry: ret = decode_slice(s); while(s->mb_ymb_height){ if(s->msmpeg4_version){ - if(s->slice_height==0 || s->mb_x!=0 || (s->mb_y%s->slice_height)!=0 || get_bits_count(&s->gb) > s->gb.size_in_bits) + if(s->slice_height==0 || s->mb_x!=0 || (s->mb_y%s->slice_height)!=0 || get_bits_left(&s->gb)<0) break; }else{ int prev_x=s->mb_x, prev_y=s->mb_y; diff --git a/libavcodec/h264.c b/libavcodec/h264.c index 46e6c72832..d09c4aca2e 100644 --- a/libavcodec/h264.c +++ b/libavcodec/h264.c @@ -3666,7 +3666,7 @@ static int decode_slice(struct AVCodecContext *avctx, void *arg){ if(s->mb_y >= s->mb_height){ tprintf(s->avctx, "slice end %d %d\n", get_bits_count(&s->gb), s->gb.size_in_bits); - if(get_bits_count(&s->gb) == s->gb.size_in_bits ) { + if (get_bits_left(&s->gb) == 0) { ff_er_add_slice(s, s->resync_mb_x, s->resync_mb_y, s->mb_x-1, s->mb_y, ER_MB_END&part_mask); return 0; @@ -3678,9 +3678,9 @@ static int decode_slice(struct AVCodecContext *avctx, void *arg){ } } - if(get_bits_count(&s->gb) >= s->gb.size_in_bits && s->mb_skip_run<=0){ + if (get_bits_left(&s->gb) <= 0 && s->mb_skip_run <= 0){ tprintf(s->avctx, "slice end %d %d\n", get_bits_count(&s->gb), s->gb.size_in_bits); - if(get_bits_count(&s->gb) == s->gb.size_in_bits ){ + if (get_bits_left(&s->gb) == 0) { ff_er_add_slice(s, s->resync_mb_x, s->resync_mb_y, s->mb_x-1, s->mb_y, ER_MB_END&part_mask); if (s->mb_x > lf_x_start) loop_filter(h, lf_x_start, s->mb_x); diff --git a/libavcodec/h264_ps.c b/libavcodec/h264_ps.c index 76bf116a3f..287702c7c4 100644 --- a/libavcodec/h264_ps.c +++ b/libavcodec/h264_ps.c @@ -227,8 +227,8 @@ static inline int decode_vui_parameters(H264Context *h, SPS *sps){ sps->num_reorder_frames= get_ue_golomb(&s->gb); get_ue_golomb(&s->gb); /*max_dec_frame_buffering*/ - if(s->gb.size_in_bits < get_bits_count(&s->gb)){ - av_log(h->s.avctx, AV_LOG_ERROR, "Overread VUI by %d bits\n", get_bits_count(&s->gb) - s->gb.size_in_bits); + if (get_bits_left(&s->gb) < 0) { + av_log(h->s.avctx, AV_LOG_ERROR, "Overread VUI by %d bits\n", -get_bits_left(&s->gb)); sps->num_reorder_frames=0; sps->bitstream_restriction_flag= 0; } diff --git a/libavcodec/h264_sei.c b/libavcodec/h264_sei.c index 4f52bbe969..2e5fb65f0d 100644 --- a/libavcodec/h264_sei.c +++ b/libavcodec/h264_sei.c @@ -164,7 +164,7 @@ static int decode_buffering_period(H264Context *h){ int ff_h264_decode_sei(H264Context *h){ MpegEncContext * const s = &h->s; - while(get_bits_count(&s->gb) + 16 < s->gb.size_in_bits){ + while (get_bits_left(&s->gb) > 16) { int size, type; type=0; diff --git a/libavcodec/huffyuv.c b/libavcodec/huffyuv.c index 2e1db043ea..a173a13d87 100644 --- a/libavcodec/huffyuv.c +++ b/libavcodec/huffyuv.c @@ -720,7 +720,7 @@ static void decode_422_bitstream(HYuvContext *s, int count){ count/=2; if(count >= (get_bits_left(&s->gb))/(31*4)){ - for(i=0; igb) < s->gb.size_in_bits; i++){ + for (i = 0; i < count && get_bits_left(&s->gb) > 0; i++) { READ_2PIX(s->temp[0][2*i ], s->temp[1][i], 1); READ_2PIX(s->temp[0][2*i+1], s->temp[2][i], 2); } @@ -738,7 +738,7 @@ static void decode_gray_bitstream(HYuvContext *s, int count){ count/=2; if(count >= (get_bits_left(&s->gb))/(31*2)){ - for(i=0; igb) < s->gb.size_in_bits; i++){ + for (i = 0; i < count && get_bits_left(&s->gb) > 0; i++) { READ_2PIX(s->temp[0][2*i ], s->temp[0][2*i+1], 0); } }else{ diff --git a/libavcodec/ituh263dec.c b/libavcodec/ituh263dec.c index 148bb33a36..3d82e5c382 100644 --- a/libavcodec/ituh263dec.c +++ b/libavcodec/ituh263dec.c @@ -852,8 +852,8 @@ end: { int v= show_bits(&s->gb, 16); - if(get_bits_count(&s->gb) + 16 > s->gb.size_in_bits){ - v>>= get_bits_count(&s->gb) + 16 - s->gb.size_in_bits; + if (get_bits_left(&s->gb) < 16) { + v >>= 16 - get_bits_left(&s->gb); } if(v==0) diff --git a/libavcodec/mjpegdec.c b/libavcodec/mjpegdec.c index 49d334bb7e..a7950287e2 100644 --- a/libavcodec/mjpegdec.c +++ b/libavcodec/mjpegdec.c @@ -858,9 +858,9 @@ static int mjpeg_decode_scan(MJpegDecodeContext *s, int nb_components, int Ah, if (s->restart_interval && !s->restart_count) s->restart_count = s->restart_interval; - if (get_bits_count(&s->gb)>s->gb.size_in_bits) { + if (get_bits_left(&s->gb) < 0) { av_log(s->avctx, AV_LOG_ERROR, "overread %d\n", - get_bits_count(&s->gb) - s->gb.size_in_bits); + -get_bits_left(&s->gb)); return -1; } for (i = 0; i < nb_components; i++) { @@ -1151,7 +1151,7 @@ static int mjpeg_decode_app(MJpegDecodeContext *s) len = get_bits(&s->gb, 16); if (len < 5) return -1; - if (8 * len + get_bits_count(&s->gb) > s->gb.size_in_bits) + if (8 * len > get_bits_left(&s->gb)) return -1; id = get_bits_long(&s->gb, 32); @@ -1292,8 +1292,7 @@ out: static int mjpeg_decode_com(MJpegDecodeContext *s) { int len = get_bits(&s->gb, 16); - if (len >= 2 && - 8 * len - 16 + get_bits_count(&s->gb) <= s->gb.size_in_bits) { + if (len >= 2 && 8 * len - 16 <= get_bits_left(&s->gb)) { char *cbuf = av_malloc(len - 1); if (cbuf) { int i; diff --git a/libavcodec/vp6.c b/libavcodec/vp6.c index e4783c6e84..91377015eb 100644 --- a/libavcodec/vp6.c +++ b/libavcodec/vp6.c @@ -387,7 +387,7 @@ static void vp6_parse_coeff_huffman(VP56Context *s) if (coeff_idx) break; } else { - if (get_bits_count(&s->gb) >= s->gb.size_in_bits) + if (get_bits_left(&s->gb) <= 0) return; coeff = get_vlc2(&s->gb, vlc_coeff->table, 9, 3); if (coeff == 0) { From a81a6d9c80448cfbba0d2bcdd681bf971c7055a4 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Sat, 1 Oct 2011 17:41:28 +0200 Subject: [PATCH 146/154] h264: improve parsing of broken AVC SPS Parsing the entire NAL as SPS fixes decoding of some AVC bitstreams with broken escaping. Since the size of the NAL unit is known and checked against the buffer end we can parse it entirely without buffer overreads. Fixes playback of http://streams.videolan.org/streams/mp4/Mr_MrsSmith-h264_aac.mp4 Signed-off-by: Janne Grunau (cherry picked from commit 3aa661ec561d7a20812b84b353b0d7855ac346c8) Signed-off-by: Reinhard Tartler --- libavcodec/h264.c | 9 ++++++++- libavcodec/h264_ps.c | 5 ++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/libavcodec/h264.c b/libavcodec/h264.c index d09c4aca2e..95a672b962 100644 --- a/libavcodec/h264.c +++ b/libavcodec/h264.c @@ -3924,7 +3924,14 @@ static int decode_nal_units(H264Context *h, const uint8_t *buf, int buf_size){ break; case NAL_SPS: init_get_bits(&s->gb, ptr, bit_length); - ff_h264_decode_seq_parameter_set(h); + if (ff_h264_decode_seq_parameter_set(h) < 0 && + h->is_avc && (nalsize != consumed) && nalsize) { + av_log(h->s.avctx, AV_LOG_DEBUG, "SPS decoding failure, " + "try parsing the coomplete NAL\n"); + init_get_bits(&s->gb, buf + buf_index + 1 - consumed, + 8 * (nalsize - 1)); + ff_h264_decode_seq_parameter_set(h); + } if (s->flags& CODEC_FLAG_LOW_DELAY || (h->sps.bitstream_restriction_flag && !h->sps.num_reorder_frames)) diff --git a/libavcodec/h264_ps.c b/libavcodec/h264_ps.c index 287702c7c4..276eb77d1d 100644 --- a/libavcodec/h264_ps.c +++ b/libavcodec/h264_ps.c @@ -228,7 +228,6 @@ static inline int decode_vui_parameters(H264Context *h, SPS *sps){ get_ue_golomb(&s->gb); /*max_dec_frame_buffering*/ if (get_bits_left(&s->gb) < 0) { - av_log(h->s.avctx, AV_LOG_ERROR, "Overread VUI by %d bits\n", -get_bits_left(&s->gb)); sps->num_reorder_frames=0; sps->bitstream_restriction_flag= 0; } @@ -238,6 +237,10 @@ static inline int decode_vui_parameters(H264Context *h, SPS *sps){ return -1; } } + if (get_bits_left(&s->gb) < 0) { + av_log(h->s.avctx, AV_LOG_ERROR, "Overread VUI by %d bits\n", -get_bits_left(&s->gb)); + return AVERROR_INVALIDDATA; + } return 0; } From 4d343a6f47931a43fe8e4c9288a5068c76b843e0 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Tue, 13 Mar 2012 16:26:44 -0700 Subject: [PATCH 147/154] h264: stricter reference limit enforcement. Progressive images can have only 16 references, error out if there are more, since the data is almost certainly corrupt, and the invalid value will lead to random crashes or invalid writes later on. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit e0febda22d0e0fab094a9c886b0e0f0f662df1ef) Signed-off-by: Reinhard Tartler --- libavcodec/h264.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/libavcodec/h264.c b/libavcodec/h264.c index 95a672b962..e0eb8e119d 100644 --- a/libavcodec/h264.c +++ b/libavcodec/h264.c @@ -3020,6 +3020,8 @@ static int decode_slice_header(H264Context *h, H264Context *h0){ h->ref_count[1]= h->pps.ref_count[1]; if(h->slice_type_nos != AV_PICTURE_TYPE_I){ + int max_refs = s->picture_structure == PICT_FRAME ? 16 : 32; + if(h->slice_type_nos == AV_PICTURE_TYPE_B){ h->direct_spatial_mv_pred= get_bits1(&s->gb); } @@ -3029,13 +3031,14 @@ static int decode_slice_header(H264Context *h, H264Context *h0){ h->ref_count[0]= get_ue_golomb(&s->gb) + 1; if(h->slice_type_nos==AV_PICTURE_TYPE_B) h->ref_count[1]= get_ue_golomb(&s->gb) + 1; - - if(h->ref_count[0]-1 > 32-1 || h->ref_count[1]-1 > 32-1){ - av_log(h->s.avctx, AV_LOG_ERROR, "reference overflow\n"); - h->ref_count[0]= h->ref_count[1]= 1; - return -1; - } } + + if (h->ref_count[0] > max_refs || h->ref_count[1] > max_refs) { + av_log(h->s.avctx, AV_LOG_ERROR, "reference overflow\n"); + h->ref_count[0] = h->ref_count[1] = 1; + return AVERROR_INVALIDDATA; + } + if(h->slice_type_nos == AV_PICTURE_TYPE_B) h->list_count= 2; else From c999a8ed65797f26a8505af2d6ef203cb603b08e Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Tue, 13 Mar 2012 15:21:07 -0700 Subject: [PATCH 148/154] h264: increase reference poc list from 16 to 32. Interlaced images can have 32 references (16 per field), so limiting the array size to 16 leads to invalid writes. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 48cbe4b092113eae0b3e5d6a08b59027f913a884) Signed-off-by: Reinhard Tartler --- libavcodec/mpegvideo.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavcodec/mpegvideo.h b/libavcodec/mpegvideo.h index 3473e6d8f7..06be735301 100644 --- a/libavcodec/mpegvideo.h +++ b/libavcodec/mpegvideo.h @@ -124,7 +124,7 @@ typedef struct Picture{ int pic_id; /**< h264 pic_num (short -> no wrap version of pic_num, pic_num & max_pic_num; long -> long_pic_num) */ int long_ref; ///< 1->long term reference 0->short term reference - int ref_poc[2][2][16]; ///< h264 POCs of the frames used as reference (FIXME need per slice) + int ref_poc[2][2][32]; ///< h264 POCs of the frames used as reference (FIXME need per slice) int ref_count[2][2]; ///< number of entries in ref_poc (FIXME need per slice) int mbaff; ///< h264 1 -> MBAFF frame 0-> not MBAFF int field_picture; ///< whether or not the picture was encoded in separate fields From 6e5c07f4c81317d728bfcba5f46b4ef46de9857f Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Tue, 13 Mar 2012 12:28:35 -0700 Subject: [PATCH 149/154] xa_adpcm: limit filter to prevent xa_adpcm_table[] array bounds overruns. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org (cherry picked from commit 86020073dbb9a3a9d1fbb76345b2ca29ba1f13d2) Signed-off-by: Reinhard Tartler --- libavcodec/adpcm.c | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/libavcodec/adpcm.c b/libavcodec/adpcm.c index a540a9b12f..a2947329eb 100644 --- a/libavcodec/adpcm.c +++ b/libavcodec/adpcm.c @@ -260,8 +260,9 @@ static inline short adpcm_yamaha_expand_nibble(ADPCMChannelStatus *c, unsigned c return c->predictor; } -static void xa_decode(short *out, const unsigned char *in, - ADPCMChannelStatus *left, ADPCMChannelStatus *right, int inc) +static int xa_decode(AVCodecContext *avctx, + short *out, const unsigned char *in, + ADPCMChannelStatus *left, ADPCMChannelStatus *right, int inc) { int i, j; int shift,filter,f0,f1; @@ -272,6 +273,12 @@ static void xa_decode(short *out, const unsigned char *in, shift = 12 - (in[4+i*2] & 15); filter = in[4+i*2] >> 4; + if (filter > 4) { + av_log(avctx, AV_LOG_ERROR, + "Invalid XA-ADPCM filter %d (max. allowed is 4)\n", + filter); + return AVERROR_INVALIDDATA; + } f0 = xa_adpcm_table[filter][0]; f1 = xa_adpcm_table[filter][1]; @@ -299,7 +306,12 @@ static void xa_decode(short *out, const unsigned char *in, shift = 12 - (in[5+i*2] & 15); filter = in[5+i*2] >> 4; - + if (filter > 4) { + av_log(avctx, AV_LOG_ERROR, + "Invalid XA-ADPCM filter %d (max. allowed is 4)\n", + filter); + return AVERROR_INVALIDDATA; + } f0 = xa_adpcm_table[filter][0]; f1 = xa_adpcm_table[filter][1]; @@ -323,6 +335,8 @@ static void xa_decode(short *out, const unsigned char *in, left->sample2 = s_2; } } + + return 0; } /** @@ -782,8 +796,9 @@ static int adpcm_decode_frame(AVCodecContext *avctx, void *data, break; case CODEC_ID_ADPCM_XA: while (buf_size >= 128) { - xa_decode(samples, src, &c->status[0], &c->status[1], - avctx->channels); + if ((ret = xa_decode(avctx, samples, src, &c->status[0], + &c->status[1], avctx->channels)) < 0) + return ret; src += 128; samples += 28 * 8; buf_size -= 128; From c9e95636a893225e2a5a42ba6e1e3cf6bfd59f2b Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Sat, 10 Mar 2012 00:08:32 +0100 Subject: [PATCH 150/154] snow: reject unsupported chroma shifts. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org Signed-off-by: Ronald S. Bultje (cherry picked from commit c9837954e7b968d44f82e7cdb7618e9f523b196c) Signed-off-by: Reinhard Tartler --- libavcodec/snowdec.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/libavcodec/snowdec.c b/libavcodec/snowdec.c index 2b6f6e11c7..d9f61bc7ca 100644 --- a/libavcodec/snowdec.c +++ b/libavcodec/snowdec.c @@ -327,6 +327,11 @@ static int decode_header(SnowContext *s){ return -1; } + if (s->chroma_h_shift != 1 || s->chroma_v_shift != 1) { + av_log(s->avctx, AV_LOG_ERROR, "Invalid chroma shift\n"); + return AVERROR_PATCHWELCOME; + } + s->qlog += get_symbol(&s->c, s->header_state, 1); s->mv_scale += get_symbol(&s->c, s->header_state, 1); s->qbias += get_symbol(&s->c, s->header_state, 1); From ce15406e78fd213b420dae68c5015803d9716c51 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Fri, 2 Mar 2012 20:53:00 +0100 Subject: [PATCH 151/154] snow: check reference frame indices. Fixes NULL ptr dereference Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org Signed-off-by: Ronald S. Bultje (cherry picked from commit 1f8ff2b13cbfef790385818664ed12e763e7c75b) Signed-off-by: Reinhard Tartler --- libavcodec/snowdec.c | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/libavcodec/snowdec.c b/libavcodec/snowdec.c index d9f61bc7ca..70c5d4afc7 100644 --- a/libavcodec/snowdec.c +++ b/libavcodec/snowdec.c @@ -132,7 +132,7 @@ static inline void decode_subband_slice_buffered(SnowContext *s, SubBand *b, sli return; } -static void decode_q_branch(SnowContext *s, int level, int x, int y){ +static int decode_q_branch(SnowContext *s, int level, int x, int y){ const int w= s->b_width << s->block_max_depth; const int rem_depth= s->block_max_depth - level; const int index= (x + y*w) << rem_depth; @@ -142,10 +142,11 @@ static void decode_q_branch(SnowContext *s, int level, int x, int y){ const BlockNode *tl = y && x ? &s->block[index-w-1] : left; const BlockNode *tr = y && trxblock[index-w+(1<level + 2*top->level + tl->level + tr->level; + int res; if(s->keyframe){ set_blocks(s, level, x, y, null_block.color[0], null_block.color[1], null_block.color[2], null_block.mx, null_block.my, null_block.ref, BLOCK_INTRA); - return; + return 0; } if(level==s->block_max_depth || get_rac(&s->c, &s->block_state[4 + s_context])){ @@ -168,17 +169,23 @@ static void decode_q_branch(SnowContext *s, int level, int x, int y){ }else{ if(s->ref_frames > 1) ref= get_symbol(&s->c, &s->block_state[128 + 1024 + 32*ref_context], 0); + if (ref >= s->ref_frames) { + av_log(s->avctx, AV_LOG_ERROR, "Invalid ref\n"); + return AVERROR_INVALIDDATA; + } pred_mv(s, &mx, &my, ref, left, top, tr); mx+= get_symbol(&s->c, &s->block_state[128 + 32*(mx_context + 16*!!ref)], 1); my+= get_symbol(&s->c, &s->block_state[128 + 32*(my_context + 16*!!ref)], 1); } set_blocks(s, level, x, y, l, cb, cr, mx, my, ref, type); }else{ - decode_q_branch(s, level+1, 2*x+0, 2*y+0); - decode_q_branch(s, level+1, 2*x+1, 2*y+0); - decode_q_branch(s, level+1, 2*x+0, 2*y+1); - decode_q_branch(s, level+1, 2*x+1, 2*y+1); + if ((res = decode_q_branch(s, level+1, 2*x+0, 2*y+0)) < 0 || + (res = decode_q_branch(s, level+1, 2*x+1, 2*y+0)) < 0 || + (res = decode_q_branch(s, level+1, 2*x+0, 2*y+1)) < 0 || + (res = decode_q_branch(s, level+1, 2*x+1, 2*y+1)) < 0) + return res; } + return 0; } static void dequantize_slice_buffered(SnowContext *s, slice_buffer * sb, SubBand *b, IDWTELEM *src, int stride, int start_y, int end_y){ @@ -354,16 +361,19 @@ static av_cold int decode_init(AVCodecContext *avctx) return 0; } -static void decode_blocks(SnowContext *s){ +static int decode_blocks(SnowContext *s){ int x, y; int w= s->b_width; int h= s->b_height; + int res; for(y=0; ydebug&FF_DEBUG_PICT_INFO) av_log(avctx, AV_LOG_ERROR, "keyframe:%d qlog:%d\n", s->keyframe, s->qlog); - decode_blocks(s); + if ((res = decode_blocks(s)) < 0) + return res; for(plane_index=0; plane_index<3; plane_index++){ Plane *p= &s->plane[plane_index]; From b5947324758ae2a99ebf910ad425fdde69b23935 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Wed, 7 Mar 2012 11:06:20 -0800 Subject: [PATCH 152/154] dca: don't use av_clip_uintp2(). The argument is not a literal, thus causing the ARM v6 or later builds to break. Signed-off-by: Janne Grunau --- libavcodec/dca.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavcodec/dca.c b/libavcodec/dca.c index 1bd31c9ac4..70cf6044f8 100644 --- a/libavcodec/dca.c +++ b/libavcodec/dca.c @@ -644,7 +644,7 @@ static inline int get_scale(GetBitContext *gb, int level, int value, int log2ran if (level < 5) { /* huffman encoded */ value += get_bitalloc(gb, &dca_scalefactor, level); - value = av_clip_uintp2(value, log2range); + value = av_clip(value, 0, (1 << log2range) - 1); } else if (level < 8) { if (level + 1 > log2range) { skip_bits(gb, level + 1 - log2range); From 1ee0cd1ad77ab96ca4573ea4b3937df7e138c8d5 Mon Sep 17 00:00:00 2001 From: Kostya Shishkov Date: Wed, 7 Mar 2012 20:07:17 +0100 Subject: [PATCH 153/154] dca: include libavutil/mathematics.h for possibly missing M_SQRT1_2 Signed-off-by: Janne Grunau --- libavcodec/dca.c | 1 + 1 file changed, 1 insertion(+) diff --git a/libavcodec/dca.c b/libavcodec/dca.c index 70cf6044f8..a83d082118 100644 --- a/libavcodec/dca.c +++ b/libavcodec/dca.c @@ -29,6 +29,7 @@ #include "libavutil/common.h" #include "libavutil/intmath.h" #include "libavutil/intreadwrite.h" +#include "libavutil/mathematics.h" #include "libavutil/audioconvert.h" #include "avcodec.h" #include "dsputil.h" From 5effcfa76792470677a1f6bc9aa73347a87ef720 Mon Sep 17 00:00:00 2001 From: Reinhard Tartler Date: Thu, 15 Mar 2012 08:57:33 +0100 Subject: [PATCH 154/154] Update Changelog for the 0.8.1 Release --- Changelog | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/Changelog b/Changelog index f1530749de..cb04ee4992 100644 --- a/Changelog +++ b/Changelog @@ -1,6 +1,36 @@ Entries are sorted chronologically from oldest to youngest within each release, releases are sorted from youngest to oldest. +version 0.8.1: + +- Several bugs and crashes have been fixed in the following codecs: AAC, + AC-3, ADPCM, AMR (both NB and WB), ATRAC3, CAVC, Cook, camstudio, DCA, + DPCM, DSI CIN, DV, EA TGQ, FLAC, fraps, G.722 (both encoder and + decoder), H.264, huvffyuv, BB JV decoder, Indeo 3, KGV1, LCL, the + libx264 wrapper, MJPEG, mp3on4, Musepack, MPEG1/2, PNG, QDM2, Qt RLE, + ROQ, RV10, RV30/RV34/RV40, shorten, smacker, subrip, SVQ3, TIFF, + Truemotion2, TTA, VC1, VMware Screen codec, Vorbis, VP5, VP6, WMA, + Westwood SNDx, XXAN. + +- This release additionally updates the following codecs to the + bytestream2 API, and therefore benefit from additional overflow + checks: XXAN, ALG MM, TQG, SMC, Qt SMC, ROQ, PNG + +- Several bugs and crashes have been fixed in the following formats: + AIFF, ASF, DV, Matroska, NSV, MOV, MPEG-TS, Smacker, Sony OpenMG, RM, + SWF. + +- Libswscale has an potential overflow for large image size fixed. + +- The following APIs have been added: + + avcodec_is_open() + avformat_get_riff_video_tags() + avformat_get_riff_audio_tags() + + Please see the file doc/APIchanges and the Doxygen documentation for + further information. + version 0.8: