libavcodec/dcadec: adjust the ch_layout when downmix is active

Applications making use of this codec with the `downmix` option are
segfaulting unless the `ch_layout` is overridden after `avcodec_open2`
as can be seen in projects like MythTV[1]

This patch fixes this by overriding the ch_layout as done in other
decoders such as AC3.

1: af6f362a14/mythtv/libs/libmythtv/decoders/avformatdecoder.cpp (L4607)

Signed-off-by: Geoffrey McRae <geoff@hostfission.com>
Signed-off-by: James Almer <jamrial@gmail.com>
This commit is contained in:
Geoffrey McRae 2023-11-26 15:33:22 +11:00 committed by James Almer
parent 47b2328076
commit a8677bcc8f

View File

@ -366,12 +366,19 @@ FF_ENABLE_DEPRECATION_WARNINGS
if (s->downmix_layout.nb_channels) {
if (!av_channel_layout_compare(&s->downmix_layout, &(AVChannelLayout)AV_CHANNEL_LAYOUT_STEREO) ||
!av_channel_layout_compare(&s->downmix_layout, &(AVChannelLayout)AV_CHANNEL_LAYOUT_STEREO_DOWNMIX))
!av_channel_layout_compare(&s->downmix_layout, &(AVChannelLayout)AV_CHANNEL_LAYOUT_STEREO_DOWNMIX)) {
s->request_channel_layout = DCA_SPEAKER_LAYOUT_STEREO;
else if (!av_channel_layout_compare(&s->downmix_layout, &(AVChannelLayout)AV_CHANNEL_LAYOUT_5POINT0))
av_channel_layout_uninit(&avctx->ch_layout);
avctx->ch_layout = (AVChannelLayout)AV_CHANNEL_LAYOUT_STEREO;
} else if (!av_channel_layout_compare(&s->downmix_layout, &(AVChannelLayout)AV_CHANNEL_LAYOUT_5POINT0)) {
s->request_channel_layout = DCA_SPEAKER_LAYOUT_5POINT0;
else if (!av_channel_layout_compare(&s->downmix_layout, &(AVChannelLayout)AV_CHANNEL_LAYOUT_5POINT1))
av_channel_layout_uninit(&avctx->ch_layout);
avctx->ch_layout = (AVChannelLayout)AV_CHANNEL_LAYOUT_5POINT0;
} else if (!av_channel_layout_compare(&s->downmix_layout, &(AVChannelLayout)AV_CHANNEL_LAYOUT_5POINT1)) {
s->request_channel_layout = DCA_SPEAKER_LAYOUT_5POINT1;
av_channel_layout_uninit(&avctx->ch_layout);
avctx->ch_layout = (AVChannelLayout)AV_CHANNEL_LAYOUT_5POINT1;
}
else
av_log(avctx, AV_LOG_WARNING, "Invalid downmix layout\n");
}