diff --git a/libavformat/oggdec.c b/libavformat/oggdec.c
index 02d64d4995..926326a005 100644
--- a/libavformat/oggdec.c
+++ b/libavformat/oggdec.c
@@ -139,6 +139,7 @@ static int ogg_reset(AVFormatContext *s)
         os->nsegs = 0;
         os->segp = 0;
         os->incomplete = 0;
+        os->got_data = 0;
         if (start_pos <= s->data_offset) {
             os->lastpts = 0;
         }
@@ -205,6 +206,12 @@ static int ogg_new_stream(AVFormatContext *s, uint32_t serial)
     struct ogg_stream *os;
     size_t size;
 
+    if (ogg->state) {
+        av_log(s, AV_LOG_ERROR, "New streams are not supposed to be added "
+               "in between Ogg context save/restore operations.\n");
+        return AVERROR_BUG;
+    }
+
     /* Allocate and init a new Ogg Stream */
     if (av_size_mult(ogg->nstreams + 1, sizeof(*ogg->streams), &size) < 0 ||
         !(os = av_realloc(ogg->streams, size)))
@@ -248,6 +255,16 @@ static int ogg_new_buf(struct ogg *ogg, int idx)
     return 0;
 }
 
+static int data_packets_seen(const struct ogg *ogg)
+{
+    int i;
+
+    for (i = 0; i < ogg->nstreams; i++)
+        if (ogg->streams[i].got_data)
+            return 1;
+    return 0;
+}
+
 static int ogg_read_page(AVFormatContext *s, int *sid)
 {
     AVIOContext *bc = s->pb;
@@ -297,7 +314,7 @@ static int ogg_read_page(AVFormatContext *s, int *sid)
 
     idx = ogg_find_stream (ogg, serial);
     if (idx < 0){
-        if (ogg->headers)
+        if (data_packets_seen(ogg))
             idx = ogg_replace_stream(s, serial);
         else
             idx = ogg_new_stream(s, serial);
@@ -325,6 +342,9 @@ static int ogg_read_page(AVFormatContext *s, int *sid)
     for (i = 0; i < nsegs; i++)
         size += os->segments[i];
 
+    if (!(flags & OGG_FLAG_BOS))
+        os->got_data = 1;
+
     if (flags & OGG_FLAG_CONT || os->incomplete){
         if (!os->psize){
             // If this is the very first segment we started
diff --git a/libavformat/oggdec.h b/libavformat/oggdec.h
index aa94db5573..dd005fa61e 100644
--- a/libavformat/oggdec.h
+++ b/libavformat/oggdec.h
@@ -76,6 +76,7 @@ struct ogg_stream {
     int page_end;   ///< current packet is the last one completed in the page
     int keyframe_seek;
     int got_start;
+    int got_data;   ///< 1 if the stream got some data (non-initial packets), 0 otherwise
     void *private;
 };