diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c
index cbeefb71d6..32289112a8 100644
--- a/fftools/ffmpeg.c
+++ b/fftools/ffmpeg.c
@@ -400,6 +400,7 @@ static void frame_data_free(void *opaque, uint8_t *data)
 {
     FrameData *fd = (FrameData *)data;
 
+    av_frame_side_data_free(&fd->side_data, &fd->nb_side_data);
     avcodec_parameters_free(&fd->par_enc);
 
     av_free(data);
@@ -429,6 +430,8 @@ static int frame_data_ensure(AVBufferRef **dst, int writable)
 
             memcpy(fd, fd_src, sizeof(*fd));
             fd->par_enc = NULL;
+            fd->side_data = NULL;
+            fd->nb_side_data = 0;
 
             if (fd_src->par_enc) {
                 int ret = 0;
@@ -444,6 +447,16 @@ static int frame_data_ensure(AVBufferRef **dst, int writable)
                 }
             }
 
+            if (fd_src->nb_side_data) {
+                int ret = clone_side_data(&fd->side_data, &fd->nb_side_data,
+                                          fd_src->side_data, fd_src->nb_side_data, 0);
+                if (ret < 0) {
+                    av_buffer_unref(dst);
+                    av_buffer_unref(&src);
+                    return ret;
+                }
+            }
+
             av_buffer_unref(&src);
         } else {
             fd->dec.frame_num = UINT64_MAX;
diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h
index 63da1d14df..8f665f3432 100644
--- a/fftools/ffmpeg.h
+++ b/fftools/ffmpeg.h
@@ -698,6 +698,9 @@ typedef struct FrameData {
     int64_t wallclock[LATENCY_PROBE_NB];
 
     AVCodecParameters *par_enc;
+
+    AVFrameSideData   **side_data;
+    int                 nb_side_data;
 } FrameData;
 
 extern InputFile   **input_files;
diff --git a/fftools/ffmpeg_enc.c b/fftools/ffmpeg_enc.c
index 84e7e0ca0e..8f07a10848 100644
--- a/fftools/ffmpeg_enc.c
+++ b/fftools/ffmpeg_enc.c
@@ -205,19 +205,10 @@ int enc_open(void *opaque, const AVFrame *frame)
         av_assert0(frame->opaque_ref);
         fd = (FrameData*)frame->opaque_ref->data;
 
-        for (int i = 0; i < frame->nb_side_data; i++) {
-            const AVSideDataDescriptor *desc = av_frame_side_data_desc(frame->side_data[i]->type);
-
-            if (!(desc->props & AV_SIDE_DATA_PROP_GLOBAL))
-                continue;
-
-            ret = av_frame_side_data_clone(&enc_ctx->decoded_side_data,
-                                           &enc_ctx->nb_decoded_side_data,
-                                           frame->side_data[i],
-                                           AV_FRAME_SIDE_DATA_FLAG_UNIQUE);
-            if (ret < 0)
-                return ret;
-        }
+        ret = clone_side_data(&enc_ctx->decoded_side_data, &enc_ctx->nb_decoded_side_data,
+                              fd->side_data, fd->nb_side_data, AV_FRAME_SIDE_DATA_FLAG_UNIQUE);
+        if (ret < 0)
+            return ret;
     }
 
     if (ist)
diff --git a/fftools/ffmpeg_filter.c b/fftools/ffmpeg_filter.c
index a23bcbda95..1e1c423789 100644
--- a/fftools/ffmpeg_filter.c
+++ b/fftools/ffmpeg_filter.c
@@ -2129,7 +2129,8 @@ static int ifilter_parameters_from_frame(InputFilter *ifilter, const AVFrame *fr
     for (int i = 0; i < frame->nb_side_data; i++) {
         const AVSideDataDescriptor *desc = av_frame_side_data_desc(frame->side_data[i]->type);
 
-        if (!(desc->props & AV_SIDE_DATA_PROP_GLOBAL))
+        if (!(desc->props & AV_SIDE_DATA_PROP_GLOBAL) ||
+            frame->side_data[i]->type == AV_FRAME_DATA_DISPLAYMATRIX)
             continue;
 
         ret = av_frame_side_data_clone(&ifp->side_data,
@@ -2502,16 +2503,17 @@ static int close_output(OutputFilterPriv *ofp, FilterGraphThread *fgt)
             if (ret < 0)
                 return ret;
         }
-        av_frame_side_data_free(&frame->side_data, &frame->nb_side_data);
-        ret = clone_side_data(&frame->side_data, &frame->nb_side_data,
-                              ofp->side_data, ofp->nb_side_data, 0);
-        if (ret < 0)
-            return ret;
 
         fd = frame_data(frame);
         if (!fd)
             return AVERROR(ENOMEM);
 
+        av_frame_side_data_free(&fd->side_data, &fd->nb_side_data);
+        ret = clone_side_data(&fd->side_data, &fd->nb_side_data,
+                              ofp->side_data, ofp->nb_side_data, 0);
+        if (ret < 0)
+            return ret;
+
         fd->frame_rate_filter = ofp->fps.framerate;
 
         av_assert0(!frame->buf[0]);
@@ -2666,6 +2668,14 @@ static int fg_output_step(OutputFilterPriv *ofp, FilterGraphThread *fgt,
         return AVERROR(ENOMEM);
     }
 
+    av_frame_side_data_free(&fd->side_data, &fd->nb_side_data);
+    if (!fgt->got_frame) {
+        ret = clone_side_data(&fd->side_data, &fd->nb_side_data,
+                              ofp->side_data, ofp->nb_side_data, 0);
+        if (ret < 0)
+            return ret;
+    }
+
     fd->wallclock[LATENCY_PROBE_FILTER_POST] = av_gettime_relative();
 
     // only use bits_per_raw_sample passed through from the decoder