From bb4c9d0a8ead02f7d943c2bae3e36b30e605b30b Mon Sep 17 00:00:00 2001 From: Philip Langdale Date: Mon, 13 Nov 2017 21:11:27 -0800 Subject: [PATCH] avcodec: Don't assume separate u and v planes in ff_alloc_picture alloc_frame_buffer in ff_alloc_picture asserts that the linesize of planes 1 and 2 are the same. If the pixfmt has a single uv plane, like NV12, this won't be true. So, let's only do this check if there are more than 2 planes. We never hit this with previous hw formats because they don't set linesize to meaningful values, but the cuda hw format sets the values based on the underlying data layout. --- libavcodec/mpegpicture.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/libavcodec/mpegpicture.c b/libavcodec/mpegpicture.c index 9811a778b7..2b72346fb2 100644 --- a/libavcodec/mpegpicture.c +++ b/libavcodec/mpegpicture.c @@ -22,6 +22,7 @@ #include "libavutil/avassert.h" #include "libavutil/common.h" +#include "libavutil/pixdesc.h" #include "avcodec.h" #include "motion_est.h" @@ -155,7 +156,8 @@ static int alloc_frame_buffer(AVCodecContext *avctx, Picture *pic, return -1; } - if (pic->f->linesize[1] != pic->f->linesize[2]) { + if (av_pix_fmt_count_planes(pic->f->format) > 2 && + pic->f->linesize[1] != pic->f->linesize[2]) { av_log(avctx, AV_LOG_ERROR, "get_buffer() failed (uv stride mismatch)\n"); ff_mpeg_unref_picture(avctx, pic);