diff --git a/libavcodec/h264_refs.c b/libavcodec/h264_refs.c
index 0ccbe62bcb..fc59a9b37b 100644
--- a/libavcodec/h264_refs.c
+++ b/libavcodec/h264_refs.c
@@ -63,20 +63,22 @@ static int split_field_copy(Picture *dest, Picture *src,
     return match;
 }
 
-static int build_def_list(Picture *def, Picture **in, int len, int is_long, int sel){
+static int build_def_list(Picture *def, int def_len,
+                          Picture **in, int len, int is_long, int sel)
+{
     int i[2]={0};
     int index=0;
 
-    while(i[0]<len || i[1]<len){
+    while ((i[0] < len || i[1] < len) && index < def_len) {
         while (i[0] < len && !(in[ i[0] ] && (in[ i[0] ]->f.reference & sel)))
             i[0]++;
         while (i[1] < len && !(in[ i[1] ] && (in[ i[1] ]->f.reference & (sel^3))))
             i[1]++;
-        if(i[0] < len){
+        if (i[0] < len && index < def_len) {
             in[ i[0] ]->pic_id= is_long ? i[0] : in[ i[0] ]->frame_num;
             split_field_copy(&def[index++], in[ i[0]++ ], sel  , 1);
         }
-        if(i[1] < len){
+        if (i[1] < len && index < def_len) {
             in[ i[1] ]->pic_id= is_long ? i[1] : in[ i[1] ]->frame_num;
             split_field_copy(&def[index++], in[ i[1]++ ], sel^3, 0);
         }
@@ -124,9 +126,12 @@ int ff_h264_fill_default_ref_list(H264Context *h){
             len= add_sorted(sorted    , h->short_ref, h->short_ref_count, cur_poc, 1^list);
             len+=add_sorted(sorted+len, h->short_ref, h->short_ref_count, cur_poc, 0^list);
             assert(len<=32);
-            len= build_def_list(h->default_ref_list[list]    , sorted     , len, 0, s->picture_structure);
-            len+=build_def_list(h->default_ref_list[list]+len, h->long_ref, 16 , 1, s->picture_structure);
-            assert(len<=32);
+
+            len  = build_def_list(h->default_ref_list[list], FF_ARRAY_ELEMS(h->default_ref_list[0]),
+                                  sorted, len, 0, s->picture_structure);
+            len += build_def_list(h->default_ref_list[list] + len,
+                                  FF_ARRAY_ELEMS(h->default_ref_list[0]) - len,
+                                  h->long_ref, 16, 1, s->picture_structure);
 
             if(len < h->ref_count[list])
                 memset(&h->default_ref_list[list][len], 0, sizeof(Picture)*(h->ref_count[list] - len));
@@ -139,9 +144,12 @@ int ff_h264_fill_default_ref_list(H264Context *h){
                 FFSWAP(Picture, h->default_ref_list[1][0], h->default_ref_list[1][1]);
         }
     }else{
-        len = build_def_list(h->default_ref_list[0]    , h->short_ref, h->short_ref_count, 0, s->picture_structure);
-        len+= build_def_list(h->default_ref_list[0]+len, h-> long_ref, 16                , 1, s->picture_structure);
-        assert(len <= 32);
+        len  = build_def_list(h->default_ref_list[0], FF_ARRAY_ELEMS(h->default_ref_list[0]),
+                              h->short_ref, h->short_ref_count, 0, s->picture_structure);
+        len += build_def_list(h->default_ref_list[0] + len,
+                              FF_ARRAY_ELEMS(h->default_ref_list[0]) - len,
+                              h-> long_ref, 16, 1, s->picture_structure);
+
         if(len < h->ref_count[0])
             memset(&h->default_ref_list[0][len], 0, sizeof(Picture)*(h->ref_count[0] - len));
     }