avcodec/mlpdec: don't depend on context channel layout when setting substream masks
authorJames Almer <jamrial@gmail.com>
Mon, 27 Oct 2025 14:13:20 +0000 (11:13 -0300)
committerJames Almer <jamrial@gmail.com>
Tue, 28 Oct 2025 16:39:37 +0000 (13:39 -0300)
If avctx->ch_layout is unset (as it's allowed and even expeced by the
AV_CODEC_CAP_CHANNEL_CONF flag), the code setting substream masks will fail for
stereo and mono layouts unless a downmix channel was requested.
Fix this by deriving the mask with coded values only.

Fixes issue #20764.

Signed-off-by: James Almer <jamrial@gmail.com>
(cherry picked from commit 530ca627a325a0f77859a6a7d8baa05395767d56)

libavcodec/mlpdec.c

index e85dac36a7123b19fdc8163388756e5e206380bc..e69e4442d25afcad51d1d7910fbb5798f824dc53 100644 (file)
@@ -318,8 +318,10 @@ static av_cold int mlp_decode_init(AVCodecContext *avctx)
             av_channel_layout_uninit(&avctx->ch_layout);
             avctx->ch_layout = (AVChannelLayout)AV_CHANNEL_LAYOUT_5POINT1;
         }
-        else
+        else {
           av_log(avctx, AV_LOG_WARNING, "Invalid downmix layout\n");
+            av_channel_layout_uninit(&m->downmix_layout);
+        }
     }
 
     ff_thread_once(&init_static_once, init_static);
@@ -452,26 +454,22 @@ static int read_major_sync(MLPDecodeContext *m, GetBitContext *gb)
         }
         m->substream[1].mask = mh.channel_layout_thd_stream1;
         if (mh.channels_thd_stream1 == 2 &&
-            mh.channels_thd_stream2 == 2 &&
-            m->avctx->ch_layout.nb_channels == 2)
+            mh.channels_thd_stream2 == 2)
             m->substream[0].mask = AV_CH_LAYOUT_STEREO;
         if ((substr = (mh.num_substreams > 1)))
             m->substream[0].mask = AV_CH_LAYOUT_STEREO;
         if (mh.num_substreams == 1 &&
             mh.channels_thd_stream1 == 1 &&
-            mh.channels_thd_stream2 == 1 &&
-            m->avctx->ch_layout.nb_channels == 1)
+            mh.channels_thd_stream2 == 1)
             m->substream[0].mask = AV_CH_LAYOUT_MONO;
         if (mh.num_substreams > 2)
             if (mh.channel_layout_thd_stream2)
                 m->substream[2].mask = mh.channel_layout_thd_stream2;
             else
                 m->substream[2].mask = mh.channel_layout_thd_stream1;
-        if (m->avctx->ch_layout.nb_channels > 2)
-            if (mh.num_substreams > 2)
-                m->substream[1].mask = mh.channel_layout_thd_stream1;
-            else
-                m->substream[mh.num_substreams > 1].mask = mh.channel_layout_thd_stream2;
+        if (mh.num_substreams == 2 && (!m->downmix_layout.nb_channels ||
+                                        m->downmix_layout.nb_channels > 2))
+            m->substream[1].mask = mh.channel_layout_thd_stream2;
     }
 
     m->needs_reordering = mh.channel_arrangement >= 18 && mh.channel_arrangement <= 20;