diff --git a/libavcodec/mpegpicture.c b/libavcodec/mpegpicture.c
index 06b6daa01a..aa882cf747 100644
--- a/libavcodec/mpegpicture.c
+++ b/libavcodec/mpegpicture.c
@@ -89,12 +89,16 @@ int ff_mpeg_framesize_alloc(AVCodecContext *avctx, MotionEstContext *me,
                             ScratchpadContext *sc, int linesize)
 {
 #   define EMU_EDGE_HEIGHT (4 * 70)
-    int alloc_size = FFALIGN(FFABS(linesize) + 64, 32);
+    int linesizeabs = FFABS(linesize);
+    int alloc_size = FFALIGN(linesizeabs + 64, 32);
+
+    if (linesizeabs <= sc->linesize)
+        return 0;
 
     if (avctx->hwaccel)
         return 0;
 
-    if (linesize < 24) {
+    if (linesizeabs < 24) {
         av_log(avctx, AV_LOG_ERROR, "Image too small, temporary buffers cannot function\n");
         return AVERROR_PATCHWELCOME;
     }
@@ -102,6 +106,9 @@ int ff_mpeg_framesize_alloc(AVCodecContext *avctx, MotionEstContext *me,
     if (av_image_check_size2(alloc_size, EMU_EDGE_HEIGHT, avctx->max_pixels, AV_PIX_FMT_NONE, 0, avctx) < 0)
         return AVERROR(ENOMEM);
 
+    av_freep(&sc->edge_emu_buffer);
+    av_freep(&me->scratchpad);
+
     // edge emu needs blocksize + filter length - 1
     // (= 17x17 for  halfpel / 21x21 for H.264)
     // VC-1 computes luma and chroma simultaneously and needs 19X19 + 9x9
@@ -110,9 +117,11 @@ int ff_mpeg_framesize_alloc(AVCodecContext *avctx, MotionEstContext *me,
     // we also use this buffer for encoding in encode_mb_internal() needig an additional 32 lines
     if (!FF_ALLOCZ_TYPED_ARRAY(sc->edge_emu_buffer, alloc_size * EMU_EDGE_HEIGHT) ||
         !FF_ALLOCZ_TYPED_ARRAY(me->scratchpad,      alloc_size * 4 * 16 * 2)) {
+        sc->linesize = 0;
         av_freep(&sc->edge_emu_buffer);
         return AVERROR(ENOMEM);
     }
+    sc->linesize = linesizeabs;
 
     me->temp            = me->scratchpad;
     sc->rd_scratchpad   = me->scratchpad;
@@ -149,9 +158,9 @@ static int handle_pic_linesizes(AVCodecContext *avctx, Picture *pic,
         return -1;
     }
 
-    if (!sc->edge_emu_buffer &&
-        (ret = ff_mpeg_framesize_alloc(avctx, me, sc,
-                                       pic->f->linesize[0])) < 0) {
+    ret = ff_mpeg_framesize_alloc(avctx, me, sc,
+                                  pic->f->linesize[0]);
+    if (ret < 0) {
         av_log(avctx, AV_LOG_ERROR,
                "get_buffer() failed to allocate context scratch buffers.\n");
         ff_mpeg_unref_picture(pic);
diff --git a/libavcodec/mpegpicture.h b/libavcodec/mpegpicture.h
index a457586be5..215e7388ef 100644
--- a/libavcodec/mpegpicture.h
+++ b/libavcodec/mpegpicture.h
@@ -38,6 +38,7 @@ typedef struct ScratchpadContext {
     uint8_t *rd_scratchpad;       ///< scratchpad for rate distortion mb decision
     uint8_t *obmc_scratchpad;
     uint8_t *b_scratchpad;        ///< scratchpad used for writing into write only buffers
+    int      linesize;            ///< linesize that the buffers in this context have been allocated for
 } ScratchpadContext;
 
 /**
diff --git a/libavcodec/mpegvideo.c b/libavcodec/mpegvideo.c
index 7af823b8bd..130ccb4c97 100644
--- a/libavcodec/mpegvideo.c
+++ b/libavcodec/mpegvideo.c
@@ -443,6 +443,7 @@ static void free_duplicate_context(MpegEncContext *s)
     s->sc.rd_scratchpad =
     s->sc.b_scratchpad =
     s->sc.obmc_scratchpad = NULL;
+    s->sc.linesize = 0;
 
     av_freep(&s->dct_error_sum);
     av_freep(&s->me.map);
@@ -464,12 +465,9 @@ static void free_duplicate_contexts(MpegEncContext *s)
 static void backup_duplicate_context(MpegEncContext *bak, MpegEncContext *src)
 {
 #define COPY(a) bak->a = src->a
-    COPY(sc.edge_emu_buffer);
+    COPY(sc);
     COPY(me.scratchpad);
     COPY(me.temp);
-    COPY(sc.rd_scratchpad);
-    COPY(sc.b_scratchpad);
-    COPY(sc.obmc_scratchpad);
     COPY(me.map);
     COPY(me.score_map);
     COPY(blocks);
@@ -503,9 +501,9 @@ int ff_update_duplicate_context(MpegEncContext *dst, const MpegEncContext *src)
         // exchange uv
         FFSWAP(void *, dst->pblocks[4], dst->pblocks[5]);
     }
-    if (!dst->sc.edge_emu_buffer &&
-        (ret = ff_mpeg_framesize_alloc(dst->avctx, &dst->me,
-                                       &dst->sc, dst->linesize)) < 0) {
+    ret = ff_mpeg_framesize_alloc(dst->avctx, &dst->me,
+                                  &dst->sc, dst->linesize);
+    if (ret < 0) {
         av_log(dst->avctx, AV_LOG_ERROR, "failed to allocate context "
                "scratch buffers.\n");
         return ret;
@@ -646,12 +644,9 @@ static void clear_context(MpegEncContext *s)
     s->ac_val[0] =
     s->ac_val[1] =
     s->ac_val[2] =NULL;
-    s->sc.edge_emu_buffer = NULL;
     s->me.scratchpad = NULL;
-    s->me.temp =
-    s->sc.rd_scratchpad =
-    s->sc.b_scratchpad =
-    s->sc.obmc_scratchpad = NULL;
+    s->me.temp = NULL;
+    memset(&s->sc, 0, sizeof(s->sc));
 
 
     s->bitstream_buffer = NULL;
diff --git a/libavcodec/mpegvideo_dec.c b/libavcodec/mpegvideo_dec.c
index 4353f1fd68..31403d9acc 100644
--- a/libavcodec/mpegvideo_dec.c
+++ b/libavcodec/mpegvideo_dec.c
@@ -167,18 +167,13 @@ do {\
     }
 
     // linesize-dependent scratch buffer allocation
-    if (!s->sc.edge_emu_buffer)
-        if (s1->linesize) {
-            if (ff_mpeg_framesize_alloc(s->avctx, &s->me,
-                                        &s->sc, s1->linesize) < 0) {
-                av_log(s->avctx, AV_LOG_ERROR, "Failed to allocate context "
-                       "scratch buffers.\n");
-                return AVERROR(ENOMEM);
-            }
-        } else {
-            av_log(s->avctx, AV_LOG_ERROR, "Context scratch buffers could not "
-                   "be allocated due to unknown size.\n");
-        }
+    ret = ff_mpeg_framesize_alloc(s->avctx, &s->me,
+                                  &s->sc, s1->linesize);
+    if (ret < 0) {
+        av_log(s->avctx, AV_LOG_ERROR, "Failed to allocate context "
+               "scratch buffers.\n");
+        return ret;
+    }
 
     // MPEG-2/interlacing info
     memcpy(&s->progressive_sequence, &s1->progressive_sequence,