From a751406b915baccc6607ad06b870fac2425ea7dd Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Sun, 12 May 2024 00:13:58 +0200 Subject: [PATCH] avcodec/mpegvideo_enc: Fix potential overflow in RD Fixes: CID1500285 Unintentional integer overflow Sponsored-by: Sovereign Tech Fund Signed-off-by: Michael Niedermayer (cherry picked from commit b6b2b01025e016ce29e5add57305384a663edcfc) Signed-off-by: Michael Niedermayer --- libavcodec/mpegvideo_enc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libavcodec/mpegvideo_enc.c b/libavcodec/mpegvideo_enc.c index 7b468dde8e..1f37b6eb90 100644 --- a/libavcodec/mpegvideo_enc.c +++ b/libavcodec/mpegvideo_enc.c @@ -1474,7 +1474,7 @@ static int estimate_best_b_count(MpegEncContext *s) goto fail; } - rd += (out_size * lambda2) >> (FF_LAMBDA_SHIFT - 3); + rd += (out_size * (uint64_t)lambda2) >> (FF_LAMBDA_SHIFT - 3); } /* get the delayed frames */ @@ -1483,7 +1483,7 @@ static int estimate_best_b_count(MpegEncContext *s) ret = out_size; goto fail; } - rd += (out_size * lambda2) >> (FF_LAMBDA_SHIFT - 3); + rd += (out_size * (uint64_t)lambda2) >> (FF_LAMBDA_SHIFT - 3); rd += c->error[0] + c->error[1] + c->error[2];