diff --git a/libavcodec/mjpegdec.c b/libavcodec/mjpegdec.c
index 3082031ba4..b829ab682b 100644
--- a/libavcodec/mjpegdec.c
+++ b/libavcodec/mjpegdec.c
@@ -1678,15 +1678,6 @@ int ff_mjpeg_decode_sos(MJpegDecodeContext *s, const uint8_t *mb_bitmask,
         return -1;
     }
 
-    if (reference) {
-        if (reference->width  != s->picture_ptr->width  ||
-            reference->height != s->picture_ptr->height ||
-            reference->format != s->picture_ptr->format) {
-            av_log(s->avctx, AV_LOG_ERROR, "Reference mismatching\n");
-            return AVERROR_INVALIDDATA;
-        }
-    }
-
     /* XXX: verify len field validity */
     len = get_bits(&s->gb, 16);
     nb_components = get_bits(&s->gb, 8);
diff --git a/libavcodec/mxpegdec.c b/libavcodec/mxpegdec.c
index 9a5110a814..8861a3969d 100644
--- a/libavcodec/mxpegdec.c
+++ b/libavcodec/mxpegdec.c
@@ -179,6 +179,12 @@ static int mxpeg_check_dimensions(MXpegDecodeContext *s, MJpegDecodeContext *jpg
                 return AVERROR(EINVAL);
             }
         }
+        if (reference_ptr->width  != jpg->picture_ptr->width  ||
+            reference_ptr->height != jpg->picture_ptr->height ||
+            reference_ptr->format != jpg->picture_ptr->format) {
+            av_log(jpg->avctx, AV_LOG_ERROR, "Reference mismatching\n");
+            return AVERROR_INVALIDDATA;
+        }
     }
 
     return 0;