diff --git a/libavcodec/h264.c b/libavcodec/h264.c
index a56f900a50..40c4ea1efe 100644
--- a/libavcodec/h264.c
+++ b/libavcodec/h264.c
@@ -1023,7 +1023,8 @@ again:
 #endif
                 } else
                     context_count++;
-            }
+            } else
+                sl->ref_count[0] = sl->ref_count[1] = 0;
             break;
         case NAL_DPA:
         case NAL_DPB: