diff --git a/decoder/ih264d_defs.h b/decoder/ih264d_defs.h
index 2758a59..73604f1 100644
--- a/decoder/ih264d_defs.h
+++ b/decoder/ih264d_defs.h
@@ -45,6 +45,11 @@
 
 #define FMT_CONV_NUM_ROWS       16
 
+/** Decoder currently has an additional latency of 2 pictures when
+  * returning output for display
+  */
+#define DISPLAY_LATENCY         2
+
 /** Bit manipulation macros */
 #define CHECKBIT(a,i) ((a) &  (1 << i))
 #define CLEARBIT(a,i) ((a) &= ~(1 << i))
diff --git a/decoder/ih264d_parse_headers.c b/decoder/ih264d_parse_headers.c
index 674f4c3..793f604 100644
--- a/decoder/ih264d_parse_headers.c
+++ b/decoder/ih264d_parse_headers.c
@@ -1091,6 +1091,23 @@ WORD32 ih264d_parse_sps(dec_struct_t *ps_dec, dec_bit_stream_t *ps_bitstrm)
     /*--------------------------------------------------------------------*/
     /* All initializations to ps_dec are beyond this point                */
     /*--------------------------------------------------------------------*/
+    {
+        WORD32 reorder_depth = ih264d_get_dpb_size(ps_seq);
+        if((1 == ps_seq->u1_vui_parameters_present_flag) &&
+           (1 == ps_seq->s_vui.u1_bitstream_restriction_flag))
+        {
+            reorder_depth = ps_seq->s_vui.u4_num_reorder_frames + 1;
+        }
+
+        if (reorder_depth > H264_MAX_REF_PICS)
+        {
+            return ERROR_INV_SPS_PPS_T;
+        }
+
+        if(ps_seq->u1_frame_mbs_only_flag != 1)
+            reorder_depth *= 2;
+        ps_dec->i4_reorder_depth = reorder_depth + DISPLAY_LATENCY;
+    }
     ps_dec->u2_disp_height = i4_cropped_ht;
     ps_dec->u2_disp_width = i4_cropped_wd;
 
diff --git a/decoder/ih264d_utils.c b/decoder/ih264d_utils.c
index 141a111..0893c3f 100644
--- a/decoder/ih264d_utils.c
+++ b/decoder/ih264d_utils.c
@@ -770,7 +770,6 @@ WORD32 ih264d_init_pic(dec_struct_t *ps_dec,
             else
                 ps_dec->i4_display_delay = ps_seq->s_vui.u4_num_reorder_frames * 2 + 2;
         }
-        ps_dec->i4_reorder_depth = ps_dec->i4_display_delay;
 
         if(IVD_DECODE_FRAME_OUT == ps_dec->e_frm_out_mode)
             ps_dec->i4_display_delay = 0;