diff --git a/encoder/ih264e_encode.c b/encoder/ih264e_encode.c
index 041148b..c027321 100644
--- a/encoder/ih264e_encode.c
+++ b/encoder/ih264e_encode.c
@@ -304,6 +304,12 @@ WORD32 ih264e_encode(iv_obj_t *ps_codec_obj, void *pv_api_ip, void *pv_api_op)
     }
 #endif /*LOGO_EN*/
 
+    /* In case of alt ref and B pics we will have non reference frame in stream */
+    if (ps_codec->s_cfg.u4_enable_alt_ref || ps_codec->s_cfg.u4_num_bframes)
+    {
+        ps_codec->i4_non_ref_frames_in_stream = 1;
+    }
+
     if (ps_codec->i4_encode_api_call_cnt == 0)
     {
         /********************************************************************/
diff --git a/encoder/ih264e_utils.c b/encoder/ih264e_utils.c
index b8514a8..b339143 100644
--- a/encoder/ih264e_utils.c
+++ b/encoder/ih264e_utils.c
@@ -1360,12 +1360,6 @@ IH264E_ERROR_T ih264e_pic_init(codec_t *ps_codec, inp_buf_t *ps_inp_buf)
      *   a) Set up alt ref too
      **************************************************************************/
 
-    /* In case of alt ref and B pics we will have non reference frame in stream */
-    if (ps_codec->s_cfg.u4_enable_alt_ref || ps_codec->s_cfg.u4_num_bframes)
-    {
-        ps_codec->i4_non_ref_frames_in_stream = 1;
-    }
-
     /* Check and set if the current frame is reference or not */
     ps_codec->u4_is_curr_frm_ref = 0;