diff --git a/libavformat/oggdec.c b/libavformat/oggdec.c
index 1b4553c73b..be2f03a47b 100644
--- a/libavformat/oggdec.c
+++ b/libavformat/oggdec.c
@@ -374,20 +374,20 @@ ogg_packet (AVFormatContext * s, int *str, int *dstart, int *dsize, int64_t *fpo
             os->segp = segp;
             os->psize = psize;
 
-            // We have reached the first non-header packet. All header
-            // packets must be complete before the first non-header
-            // one, so everything that follows must be non-header.
+            // We have reached the first non-header packet in this stream.
+            // Unfortunately more header packets may still follow for others,
+            // so we reset this later unless we are done with the headers
+            // for all streams.
             ogg->headers = 1;
 
             // Update the header state for all streams and
             // compute the data_offset.
-            s->data_offset = os->sync_pos;
+            if (!s->data_offset)
+                s->data_offset = os->sync_pos;
             for (i = 0; i < ogg->nstreams; i++) {
                 struct ogg_stream *cur_os = ogg->streams + i;
-                // Set stream header state to 0 if its last packet
-                // was a header.
                 if (cur_os->header > 0)
-                    cur_os->header = 0;
+                    ogg->headers = 0;
 
                 // if we have a partial non-header packet, its start is
                 // obviously at or after the data start