From ff712a262d317f5bd6fc9552cd837508e584a565 Mon Sep 17 00:00:00 2001 From: Xi Wang Date: Fri, 15 Mar 2013 06:59:22 -0400 Subject: [PATCH] lzo: fix overflow checking in copy_backptr() The check `src > dst' in the form `&c->out[-back] > c->out' invokes pointer overflow, which is undefined behavior in C. Remove the check. Also replace `&c->out[-back] < c->out_start' with a safe form `c->out - c->out_start < back' to avoid overflow. CC: libav-stable@libav.org Signed-off-by: Xi Wang Signed-off-by: Luca Barbato (cherry picked from commit ca6c3f2c53be70aa3c38e8f1292809db89ea1ba6) Conflicts: libavutil/lzo.c --- libavutil/lzo.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libavutil/lzo.c b/libavutil/lzo.c index 3642308100..1c65e2f6ed 100644 --- a/libavutil/lzo.c +++ b/libavutil/lzo.c @@ -118,10 +118,10 @@ static inline void memcpy_backptr(uint8_t *dst, int back, int cnt); * cnt > back is valid, this will copy the bytes we just copied, * thus creating a repeating pattern with a period length of back. */ -static inline void copy_backptr(LZOContext *c, int back, int cnt) { - register const uint8_t *src = &c->out[-back]; - register uint8_t *dst = c->out; - if (src < c->out_start || src > dst) { +static inline void copy_backptr(LZOContext *c, int back, int cnt) +{ + register uint8_t *dst = c->out; + if (dst - c->out_start < back) { c->error |= AV_LZO_INVALID_BACKPTR; return; }