From e6ef90db6cf9e00dcbef21216641cd335bd2ecd6 Mon Sep 17 00:00:00 2001 From: Cosmin Stejerean Date: Wed, 6 Dec 2023 18:39:32 +0800 Subject: [PATCH] avfilter/bwdif: account for chroma sub-sampling in min size calculation The current logic for detecting frames that are too small for the algorithm does not account for chroma sub-sampling, and so a sample where the luma plane is large enough, but the chroma planes are not will not be rejected. In that event, a heap overflow will occur. This change adjusts the logic to consider the chroma planes and makes the change to all three bwdif implementations. Fixes #10688 Signed-off-by: Cosmin Stejerean Reviewed-by: Thomas Mundt Signed-off-by: Philip Langdale (cherry picked from commit 737ede405b11a37fdd61d19cf25df296a0cb0b75) Signed-off-by: Michael Niedermayer --- libavfilter/vf_bwdif.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/libavfilter/vf_bwdif.c b/libavfilter/vf_bwdif.c index 7290c9a4a2..029bc92bd1 100644 --- a/libavfilter/vf_bwdif.c +++ b/libavfilter/vf_bwdif.c @@ -505,12 +505,13 @@ static int config_props(AVFilterLink *link) if(s->mode&1) link->frame_rate = av_mul_q(link->src->inputs[0]->frame_rate, (AVRational){2,1}); - if (link->w < 3 || link->h < 4) { - av_log(ctx, AV_LOG_ERROR, "Video of less than 3 columns or 4 lines is not supported\n"); + s->csp = av_pix_fmt_desc_get(link->format); + + if (AV_CEIL_RSHIFT(link->w, s->csp->log2_chroma_w) < 3 || AV_CEIL_RSHIFT(link->h, s->csp->log2_chroma_h) < 4) { + av_log(ctx, AV_LOG_ERROR, "Video with planes less than 3 columns or 4 lines is not supported\n"); return AVERROR(EINVAL); } - s->csp = av_pix_fmt_desc_get(link->format); if (s->csp->comp[0].depth > 8) { s->filter_intra = filter_intra_16bit; s->filter_line = filter_line_c_16bit;