sub: change how subtitles are read

Slightly change how it is decided when a new packet should be read.
Switch to demux_read_packet_async(), and let the player "wait properly"
until required subtitle packets arrive, instead of blocking everything.
Move distinguishing the cases of passive and active reading into the
demuxer, where it belongs.

demux/demux.c

@@ -685,15 +685,33 @@ struct demux_packet *demux_read_packet(struct sh_stream *sh)
     return pkt;
 }
 
+// Sparse packets (Subtitles) interleaved with other non-sparse packets (video,
+// audio) should never be read actively, meaning the demuxer thread does not
+// try to exceed default readahead in order to find a new packet.
+static bool use_lazy_subtitle_reading(struct demux_stream *ds)
+{
+    if (ds->type != STREAM_SUB)
+        return false;
+    for (int n = 0; n < ds->in->num_streams; n++) {
+        struct demux_stream *s = ds->in->streams[n]->ds;
+        if (s->type != STREAM_SUB && s->selected && !s->eof)
+            return true;
+    }
+    return false;
+}
+
 // Poll the demuxer queue, and if there's a packet, return it. Otherwise, just
 // make the demuxer thread read packets for this stream, and if there's at
 // least one packet, call the wakeup callback.
-// Unlike demux_read_packet(), this always enables readahead (which means you
-// must not use it on interleaved subtitle streams).
+// Unlike demux_read_packet(), this always enables readahead (except for
+// interleaved subtitles).
 // Returns:
 //   < 0: EOF was reached, *out_pkt=NULL
 //  == 0: no new packet yet, but maybe later, *out_pkt=NULL
 //   > 0: new packet read, *out_pkt is set
+// Note: when reading interleaved subtitles, the demuxer won't try to forcibly
+// read ahead to get the next subtitle packet (as the next packet could be
+// minutes away). In this situation, this function will just return -1.
 int demux_read_packet_async(struct sh_stream *sh, struct demux_packet **out_pkt)
 {
     struct demux_stream *ds = sh ? sh->ds : NULL;
@@ -703,10 +721,14 @@ int demux_read_packet_async(struct sh_stream *sh, struct demux_packet **out_pkt)
         if (ds->in->threading) {
             pthread_mutex_lock(&ds->in->lock);
             *out_pkt = dequeue_packet(ds);
-            r = *out_pkt ? 1 : ((ds->eof || !ds->selected) ? -1 : 0);
-            ds->active = ds->selected; // enable readahead
-            ds->in->eof = false; // force retry
-            pthread_cond_signal(&ds->in->wakeup); // possibly read more
+            if (use_lazy_subtitle_reading(ds)) {
+                r = *out_pkt ? 1 : -1;
+            } else {
+                r = *out_pkt ? 1 : ((ds->eof || !ds->selected) ? -1 : 0);
+                ds->active = ds->selected; // enable readahead
+                ds->in->eof = false; // force retry
+                pthread_cond_signal(&ds->in->wakeup); // possibly read more
+            }
             pthread_mutex_unlock(&ds->in->lock);
         } else {
             *out_pkt = demux_read_packet(sh);

player/core.h

@@ -515,7 +515,7 @@ void reinit_subs(struct MPContext *mpctx, int order);
 void uninit_sub(struct MPContext *mpctx, int order);
 void uninit_sub_all(struct MPContext *mpctx);
 void update_osd_msg(struct MPContext *mpctx);
-void update_subtitles(struct MPContext *mpctx);
+bool update_subtitles(struct MPContext *mpctx, double video_pts);
 
 // video.c
 void reset_video_state(struct MPContext *mpctx);

player/playloop.c

@@ -1036,7 +1036,8 @@ void run_playloop(struct MPContext *mpctx)
     handle_dummy_ticks(mpctx);
 
     update_osd_msg(mpctx);
-    update_subtitles(mpctx);
+    if (!mpctx->video_out)
+        update_subtitles(mpctx, mpctx->playback_pts);
 
     handle_segment_switch(mpctx, end_is_new_segment);
 

player/sub.c

@@ -68,32 +68,14 @@ void uninit_sub_all(struct MPContext *mpctx)
     uninit_sub(mpctx, 1);
 }
 
-// When reading subtitles from a demuxer, and we read video or audio from the
-// demuxer, we should not explicitly read subtitle packets. (With external
-// subs, we have to.)
-static bool is_interleaved(struct MPContext *mpctx, struct track *track)
-{
-    if (track->is_external || !track->demuxer)
-        return false;
-
-    struct demuxer *demuxer = track->demuxer;
-    for (int t = 0; t < mpctx->num_tracks; t++) {
-        struct track *other = mpctx->tracks[t];
-        if (other != track && other->selected && other->demuxer == demuxer &&
-            (other->type == STREAM_VIDEO || other->type == STREAM_AUDIO))
-            return true;
-    }
-    return track->demuxer == mpctx->demuxer;
-}
-
-static void update_subtitle(struct MPContext *mpctx, int order)
+static bool update_subtitle(struct MPContext *mpctx, double video_pts, int order)
 {
     struct MPOpts *opts = mpctx->opts;
     struct track *track = mpctx->current_track[order][STREAM_SUB];
     struct dec_sub *dec_sub = mpctx->d_sub[order];
 
-    if (!track || !dec_sub)
-        return;
+    if (!track || !dec_sub || video_pts == MP_NOPTS_VALUE)
+        return true;
 
     if (mpctx->d_video) {
         struct mp_image_params params = mpctx->d_video->vfilter->override_params;
@@ -101,46 +83,26 @@ static void update_subtitle(struct MPContext *mpctx, int order)
             sub_control(dec_sub, SD_CTRL_SET_VIDEO_PARAMS, &params);
     }
 
-    double refpts_s = mpctx->playback_pts;
-    double curpts_s = refpts_s - opts->sub_delay;
+    video_pts -= opts->sub_delay;
 
-    if (!track->preloaded && track->stream) {
-        struct sh_stream *sh_stream = track->stream;
-        bool interleaved = is_interleaved(mpctx, track);
-
-        while (1) {
-            if (interleaved && !demux_has_packet(sh_stream))
-                break;
-            double subpts_s = demux_get_next_pts(sh_stream);
-            if (!demux_has_packet(sh_stream))
-                break;
-            if (subpts_s > curpts_s) {
-                MP_DBG(mpctx, "Sub early: c_pts=%5.3f s_pts=%5.3f\n",
-                       curpts_s, subpts_s);
-                // Often subs can be handled in advance
-                if (!sub_accepts_packet_in_advance(dec_sub))
-                    break;
-                // Try to avoid demuxing whole file at once
-                if (subpts_s > curpts_s + 1 && !interleaved)
-                    break;
-            }
-            struct demux_packet *pkt = demux_read_packet(sh_stream);
-            MP_DBG(mpctx, "Sub: c_pts=%5.3f s_pts=%5.3f duration=%5.3f len=%d\n",
-                   curpts_s, pkt->pts, pkt->duration, pkt->len);
-            sub_decode(dec_sub, pkt);
-            talloc_free(pkt);
-        }
+    if (!track->preloaded) {
+        if (!sub_read_packets(dec_sub, video_pts))
+            return false;
     }
 
     // Handle displaying subtitles on terminal; never done for secondary subs
     if (order == 0 && !mpctx->video_out)
-        term_osd_set_subs(mpctx, sub_get_text(dec_sub, curpts_s));
+        term_osd_set_subs(mpctx, sub_get_text(dec_sub, video_pts));
+
+    return true;
 }
 
-void update_subtitles(struct MPContext *mpctx)
+// Return true if the subtitles for the given PTS are ready; false if the player
+// should wait for new demuxer data, and then should retry.
+bool update_subtitles(struct MPContext *mpctx, double video_pts)
 {
-    update_subtitle(mpctx, 0);
-    update_subtitle(mpctx, 1);
+    return update_subtitle(mpctx, video_pts, 0) &
+           update_subtitle(mpctx, video_pts, 1);
 }
 
 static bool init_subdec(struct MPContext *mpctx, struct track *track)

player/video.c

@@ -1206,6 +1206,11 @@ void write_video(struct MPContext *mpctx, double endpts)
     mpctx->time_frame -= get_relative_time(mpctx);
     update_avsync_before_frame(mpctx);
 
+    if (!update_subtitles(mpctx, mpctx->next_frames[0]->pts)) {
+        MP_WARN(mpctx, "subt wait\n");
+        return;
+    }
+
     double time_frame = MPMAX(mpctx->time_frame, -1);
     int64_t pts = mp_time_us() + (int64_t)(time_frame * 1e6);
 
@@ -1262,7 +1267,6 @@ void write_video(struct MPContext *mpctx, double endpts)
 
     mpctx->osd_force_update = true;
     update_osd_msg(mpctx);
-    update_subtitles(mpctx);
 
     vo_queue_frame(vo, frame);
 

sub/dec_sub.c

@@ -49,6 +49,7 @@ struct dec_sub {
     struct MPOpts *opts;
 
     struct sh_stream *sh;
+    double last_pkt_pts;
 
     struct sd *sd;
 };
@@ -90,6 +91,7 @@ struct dec_sub *sub_create(struct mpv_global *global, struct demuxer *demuxer,
         sub->log = talloc_steal(sub, log),
         sub->opts = global->opts;
         sub->sh = sh;
+        sub->last_pkt_pts = MP_NOPTS_VALUE;
         mpthread_mutex_init_recursive(&sub->lock);
 
         sub->sd = talloc(NULL, struct sd);
@@ -116,20 +118,12 @@ struct dec_sub *sub_create(struct mpv_global *global, struct demuxer *demuxer,
     return NULL;
 }
 
-void sub_decode(struct dec_sub *sub, struct demux_packet *packet)
-{
-    pthread_mutex_lock(&sub->lock);
-    sub->sd->driver->decode(sub->sd, packet);
-    pthread_mutex_unlock(&sub->lock);
-}
-
 // Read all packets from the demuxer and decode/add them. Returns false if
 // there are circumstances which makes this not possible.
 bool sub_read_all_packets(struct dec_sub *sub)
 {
     pthread_mutex_lock(&sub->lock);
 
-    // Converters are assumed to always accept packets in advance
     if (!sub->sd->driver->accept_packets_in_advance) {
         pthread_mutex_unlock(&sub->lock);
         return false;
@@ -147,14 +141,40 @@ bool sub_read_all_packets(struct dec_sub *sub)
     return true;
 }
 
-bool sub_accepts_packet_in_advance(struct dec_sub *sub)
+// Read packets from the demuxer stream passed to sub_create(). Return true if
+// enough packets were read, false if the player should wait until the demuxer
+// signals new packets available (and then should retry).
+bool sub_read_packets(struct dec_sub *sub, double video_pts)
 {
-    bool res = true;
+    bool r = true;
     pthread_mutex_lock(&sub->lock);
-    if (sub->sd->driver->accepts_packet)
-        res &= sub->sd->driver->accepts_packet(sub->sd);
+    while (1) {
+        bool read_more = true;
+        if (sub->sd->driver->accepts_packet)
+            read_more = sub->sd->driver->accepts_packet(sub->sd);
+
+        if (!read_more)
+            break;
+
+        struct demux_packet *pkt;
+        int st = demux_read_packet_async(sub->sh, &pkt);
+        // Note: "wait" (st==0) happens with non-interleaved streams only, and
+        // then we should stop the playloop until a new enough packet has been
+        // seen (or the subtitle decoder's queue is full). This does not happen
+        // for interleaved subtitle streams, which never return "wait" when
+        // reading.
+        if (st <= 0) {
+            r = st < 0 || (sub->last_pkt_pts != MP_NOPTS_VALUE &&
+                           sub->last_pkt_pts >= video_pts);
+            break;
+        }
+
+        sub->sd->driver->decode(sub->sd, pkt);
+        sub->last_pkt_pts = pkt->pts;
+        talloc_free(pkt);
+    }
     pthread_mutex_unlock(&sub->lock);
-    return res;
+    return r;
 }
 
 // You must call sub_lock/sub_unlock if more than 1 thread access sub.
@@ -189,6 +209,7 @@ void sub_reset(struct dec_sub *sub)
     pthread_mutex_lock(&sub->lock);
     if (sub->sd->driver->reset)
         sub->sd->driver->reset(sub->sd);
+    sub->last_pkt_pts = MP_NOPTS_VALUE;
     pthread_mutex_unlock(&sub->lock);
 }
 

sub/dec_sub.h

@@ -29,8 +29,7 @@ void sub_lock(struct dec_sub *sub);
 void sub_unlock(struct dec_sub *sub);
 
 bool sub_read_all_packets(struct dec_sub *sub);
-bool sub_accepts_packet_in_advance(struct dec_sub *sub);
-void sub_decode(struct dec_sub *sub, struct demux_packet *packet);
+bool sub_read_packets(struct dec_sub *sub, double video_pts);
 void sub_get_bitmaps(struct dec_sub *sub, struct mp_osd_res dim, double pts,
                      struct sub_bitmaps *res);
 char *sub_get_text(struct dec_sub *sub, double pts);