From 358b09305aeeb440d3f5e16cc335df0734136052 Mon Sep 17 00:00:00 2001 From: Shivaansh Agrawal Date: Wed, 22 Jul 2020 13:11:55 +0530 Subject: [PATCH] decoder: fix integer overflow when setting i4_prev_max_display_seq reset ps_dec->i4_prev_max_display_seq if out of int32 range to avoid overflow Bug: 143791121 Bug: 143791161 Test: POC in bug description Change-Id: I3d8df556b003a7c739716bb33262ab3a6ca7b2d9 --- decoder/ih264d_parse_slice.c | 12 ++++++------ decoder/ih264d_utils.c | 20 ++++++++++---------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/decoder/ih264d_parse_slice.c b/decoder/ih264d_parse_slice.c index 927f1c0..d807f11 100644 --- a/decoder/ih264d_parse_slice.c +++ b/decoder/ih264d_parse_slice.c @@ -826,8 +826,8 @@ WORD32 ih264d_end_of_pic_dispbuf_mgr(dec_struct_t * ps_dec) ps_cur_pic->u2_crop_offset_uv = ps_dec->u2_crop_offset_uv; ps_cur_pic->u1_pic_type = 0; { - UWORD64 i8_display_poc; - i8_display_poc = (UWORD64)ps_dec->i4_prev_max_display_seq + + WORD64 i8_display_poc; + i8_display_poc = (WORD64)ps_dec->i4_prev_max_display_seq + ps_dec->ps_cur_pic->i4_poc; if(IS_OUT_OF_RANGE_S32(i8_display_poc)) { @@ -1495,13 +1495,13 @@ WORD32 ih264d_parse_decode_slice(UWORD8 u1_is_idr_slice, /* IDR Picture or POC wrap around */ if(i4_poc == 0) { - UWORD64 u8_temp; - u8_temp = (UWORD64)ps_dec->i4_prev_max_display_seq + WORD64 i8_temp; + i8_temp = (WORD64)ps_dec->i4_prev_max_display_seq + ps_dec->i4_max_poc + ps_dec->u1_max_dec_frame_buffering + 1; /*If i4_prev_max_display_seq overflows integer range, reset it */ - ps_dec->i4_prev_max_display_seq = (u8_temp > 0x7fffffff)? - 0 : u8_temp; + ps_dec->i4_prev_max_display_seq = IS_OUT_OF_RANGE_S32(i8_temp)? + 0 : i8_temp; ps_dec->i4_max_poc = 0; } } diff --git a/decoder/ih264d_utils.c b/decoder/ih264d_utils.c index 1fe9a5b..141a111 100644 --- a/decoder/ih264d_utils.c +++ b/decoder/ih264d_utils.c @@ -1301,7 +1301,7 @@ void ih264d_release_display_bufs(dec_struct_t *ps_dec) WORD32 i4_min_poc; WORD32 i4_min_poc_buf_id; WORD32 i4_min_index; - UWORD64 u8_temp; + WORD64 i8_temp; dpb_manager_t *ps_dpb_mgr = ps_dec->ps_dpb_mgr; WORD32 (*i4_poc_buf_id_map)[3] = ps_dpb_mgr->ai4_poc_buf_id_map; @@ -1348,11 +1348,11 @@ void ih264d_release_display_bufs(dec_struct_t *ps_dec) } } ps_dpb_mgr->i1_poc_buf_id_entries = 0; - u8_temp = (UWORD64)ps_dec->i4_prev_max_display_seq + ps_dec->i4_max_poc + i8_temp = (WORD64)ps_dec->i4_prev_max_display_seq + ps_dec->i4_max_poc + ps_dec->u1_max_dec_frame_buffering + 1; /*If i4_prev_max_display_seq overflows integer range, reset it */ - ps_dec->i4_prev_max_display_seq = (u8_temp > 0x7fffffff)? - 0 : u8_temp; + ps_dec->i4_prev_max_display_seq = IS_OUT_OF_RANGE_S32(i8_temp)? + 0 : i8_temp; ps_dec->i4_max_poc = 0; } @@ -1624,13 +1624,13 @@ WORD32 ih264d_decode_gaps_in_frame_num(dec_struct_t *ps_dec, /* IDR Picture or POC wrap around */ if(i4_poc == 0) { - UWORD64 u8_temp; - u8_temp = (UWORD64)ps_dec->i4_prev_max_display_seq + WORD64 i8_temp; + i8_temp = (WORD64)ps_dec->i4_prev_max_display_seq + ps_dec->i4_max_poc + ps_dec->u1_max_dec_frame_buffering + 1; /*If i4_prev_max_display_seq overflows integer range, reset it */ - ps_dec->i4_prev_max_display_seq = (u8_temp > 0x7fffffff)? - 0 : u8_temp; + ps_dec->i4_prev_max_display_seq = IS_OUT_OF_RANGE_S32(i8_temp)? + 0 : i8_temp; ps_dec->i4_max_poc = 0; } @@ -1648,8 +1648,8 @@ WORD32 ih264d_decode_gaps_in_frame_num(dec_struct_t *ps_dec, } { - UWORD64 i8_display_poc; - i8_display_poc = (UWORD64)ps_dec->i4_prev_max_display_seq + + WORD64 i8_display_poc; + i8_display_poc = (WORD64)ps_dec->i4_prev_max_display_seq + i4_poc; if(IS_OUT_OF_RANGE_S32(i8_display_poc)) {