swscale/output: Fix integer overflow in yuv2ya16_X_c_template() release/4.3
authorMichael Niedermayer <michael@niedermayer.cc>
Mon, 13 Oct 2025 12:32:45 +0000 (14:32 +0200)
committerJames Almer <jamrial@gmail.com>
Fri, 2 Jan 2026 21:58:51 +0000 (21:58 +0000)
Found-by: colod colod <colodcolod7@gmail.com>
Fixes: CVE-2025-63757

Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
(cherry picked from commit 0c6b7f9483a38657c9be824572b4c0c45d4d9fef)
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
(cherry picked from commit ac4caa33bae5841649c61d4f8a0608dfa59c4fa1)
Signed-off-by: Carlos Henrique Lima Melara <charlesmelara@riseup.net>
libswscale/output.c

index 3cb00b4c998b2c0b4a5c05770a3d5ea5e4469ca1..d9b16f3516a34cd6042c8598daac723e34d4f871 100644 (file)
@@ -909,7 +909,7 @@ yuv2ya16_X_c_template(SwsContext *c, const int16_t *lumFilter,
         int A = 0xffff;
 
         for (j = 0; j < lumFilterSize; j++)
-            Y += lumSrc[j][i] * lumFilter[j];
+            Y += lumSrc[j][i] * (unsigned)lumFilter[j];
 
         Y >>= 15;
         Y += (1<<3) + 0x8000;
@@ -918,7 +918,7 @@ yuv2ya16_X_c_template(SwsContext *c, const int16_t *lumFilter,
         if (hasAlpha) {
             A = -0x40000000 + (1<<14);
             for (j = 0; j < lumFilterSize; j++)
-                A += alpSrc[j][i] * lumFilter[j];
+                A += alpSrc[j][i] * (unsigned)lumFilter[j];
 
             A >>= 15;
             A += 0x8000;