diff --git a/libavcodec/h264.c b/libavcodec/h264.c
index 6f49dd28f2..54f618653d 100644
--- a/libavcodec/h264.c
+++ b/libavcodec/h264.c
@@ -3039,8 +3039,11 @@ static int decode_slice_header(H264Context *h, H264Context *h0)
             h->list_count = 2;
         else
             h->list_count = 1;
-    } else
+    } else {
         h->list_count = 0;
+        h->ref_count[0] = h->ref_count[1] = 0;
+    }
+
 
     max_refs = s->picture_structure == PICT_FRAME ? 16 : 32;