video: add VO framedropping mode

This mostly uses the same idea as with vo_vdpau.c, but much simplified.

On X11, it tries to get the display framerate with XF86VM, and limits
the frequency of new video frames against it. Note that this is an old
extension, and is confirmed not to work correctly with multi-monitor
setups. But we're using it because it was already around (it is also
used by vo_vdpau).

This attempts to predict the next vsync event by using the time of the
last frame and the display FPS. Even if that goes completely wrong,
the results are still relatively good.

On other systems, or if the X11 code doesn't return a display FPS, a
framerate of 1000 is assumed. This is infinite for all practical
purposes, and means that only frames which are definitely too late are
dropped. This probably has worse results, but is still useful.

"--framedrop=yes" is basically replaced with "--framedrop=decoder". The
old framedropping mode is kept around, and should perhaps be improved.
Dropping on the decoder level is still useful if decoding itself is too
slow.

DOCS/man/input.rst

@@ -159,7 +159,9 @@ List of Input Commands
         Take a single screenshot.
     <each-frame>
         Take a screenshot each frame. Issue this command again to stop taking
-        screenshots.
+        screenshots. Note that you should disable framedropping when using
+        this mode - or you might receive duplicate images in cases when a
+        frame was dropped.
 
 ``screenshot_to_file "<filename>" [subtitles|video|window]``
     Take a screenshot and save it to a given file. The format of the file will
@@ -635,8 +637,12 @@ Property list
     disabled.
 
 ``drop-frame-count``
-    Frames dropped because they arrived to late. Unavailable if video
-    is disabled
+    Frames dropped because they arrived to late. Doesn't necessarily indicate
+    actual framedrops, just the number of times the decoder was asked to drop.
+    Unavailable if video is disabled
+
+``vo-drop-frame-count``
+    Frames dropped by VO (when using ``--framedrop=vo``).
 
 ``percent-pos`` (RW)
     Position in current file (0-100). The advantage over using this instead of

DOCS/man/options.rst

@@ -430,16 +430,39 @@ Video
     Do not sleep when outputting video frames. Useful for benchmarks when used
     with ``--no-audio.``
 
-``--framedrop=<no|yes>``
-    Skip displaying some frames to maintain A/V sync on slow systems. Video
-    filters are not applied to such frames. For B-frames even decoding is
-    skipped completely. May produce unwatchably choppy output.
+``--framedrop=<mode>``
+    Skip displaying some frames to maintain A/V sync on slow systems, or
+    playing high framerate video on video outputs that have an upper framerate
+    limit.
 
-    The ``--vd-lavc-framedrop`` option controls what frames to drop.
+    The argument selects the drop methods, and can be one of the following:
+
+    <no>
+        Disable any framedropping (default).
+    <vo>
+        Drop late frames on video output. This still decodes and filters all
+        frames, but doesn't render them on the VO. It tries to query the
+        display FPS (X11 only, not correct on multi-monitor systems), or
+        assumes infinite display FPS if that fails. Drops are indicated in
+        the terminal status line as ``D: `` field. If the decoder is too slow,
+        in theory all frames would have to be dropped (because all frames are
+        too late) - to avoid this, frame dropping stops if the effective
+        framerate is below 10 FPS.
+    <decoder>
+        Old, decoder-based framedrop mode. (This is the same as ``--framedrop=yes``
+        in mpv 0.5.x and before.) This tells the decoder to skip frames (unless
+        they are needed to decode future frames). May help with slow systems,
+        but can produce unwatchably choppy output, or even freeze the display
+        complete. Not recommended.
+        The ``--vd-lavc-framedrop`` option controls what frames to drop.
+    <decoder+vo>
+        Enable both modes. Not recommended.
 
     .. note::
 
-        Practical use of this feature is questionable. Disabled by default.
+        ``--vo=vdpau`` (also the default VO) always has the ``vo`` framedrop
+        mode enabled. It doesn't increment the ``D:`` field in the statusline
+        either.
 
 ``--hwdec=<api>``
     Specify the hardware video decoding API that should be used if possible.

options/options.c

@@ -458,7 +458,9 @@ const m_option_t mp_opts[] = {
 
     OPT_CHOICE("framedrop", frame_dropping, 0,
                ({"no", 0},
-                {"yes", 1})),
+                {"vo", 1},
+                {"decoder", 2},
+                {"decoder+vo", 3})),
 
     OPT_FLAG("untimed", untimed, M_OPT_FIXED),
 

player/command.c

@@ -386,6 +386,16 @@ static int mp_property_drop_frame_cnt(void *ctx, struct m_property *prop,
     return m_property_int_ro(action, arg, mpctx->drop_frame_cnt);
 }
 
+static int mp_property_vo_drop_frame_count(void *ctx, struct m_property *prop,
+                                           int action, void *arg)
+{
+    MPContext *mpctx = ctx;
+    if (!mpctx->d_video)
+        return M_PROPERTY_UNAVAILABLE;
+
+    return m_property_int_ro(action, arg, vo_get_drop_count(mpctx->video_out));
+}
+
 /// Current position in percent (RW)
 static int mp_property_percent_pos(void *ctx, struct m_property *prop,
                                    int action, void *arg)
@@ -2686,6 +2696,7 @@ static const struct m_property mp_properties[] = {
     {"avsync", mp_property_avsync},
     {"total-avsync-change", mp_property_total_avsync_change},
     {"drop-frame-count", mp_property_drop_frame_cnt},
+    {"vo-drop-frame-count", mp_property_vo_drop_frame_count},
     {"percent-pos", mp_property_percent_pos},
     {"time-start", mp_property_time_start},
     {"time-pos", mp_property_time_pos},

player/osd.c

@@ -39,6 +39,8 @@
 #include "demux/demux.h"
 #include "sub/osd.h"
 
+#include "video/out/vo.h"
+
 #include "core.h"
 #include "command.h"
 
@@ -219,8 +221,13 @@ void print_status(struct MPContext *mpctx)
 #endif
     {
         // VO stats
-        if (mpctx->d_video && mpctx->drop_frame_cnt)
-            saddf(&line, " Late: %d", mpctx->drop_frame_cnt);
+        if (mpctx->d_video) {
+            if (mpctx->drop_frame_cnt)
+                saddf(&line, " Late: %d", mpctx->drop_frame_cnt);
+            int64_t c = vo_get_drop_count(mpctx->video_out);
+            if (c > 0)
+                saddf(&line, " D: %"PRId64, c);
+        }
     }
 
     float cache = mp_get_cache_percent(mpctx);

player/playloop.c

@@ -66,11 +66,10 @@ void pause_player(struct MPContext *mpctx)
     mpctx->osd_function = 0;
     mpctx->paused_for_cache = false;
 
-    if (mpctx->video_out && mpctx->d_video && mpctx->video_out->config_ok)
-        vo_control(mpctx->video_out, VOCTRL_PAUSE, NULL);
-
     if (mpctx->ao && mpctx->d_audio)
-        ao_pause(mpctx->ao);    // pause audio, keep data if possible
+        ao_pause(mpctx->ao);
+    if (mpctx->video_out)
+        vo_set_paused(mpctx->video_out, true);
 
     // Only print status if there's actually a file being played.
     if (mpctx->num_sources)
@@ -97,8 +96,9 @@ void unpause_player(struct MPContext *mpctx)
 
     if (mpctx->ao && mpctx->d_audio)
         ao_resume(mpctx->ao);
-    if (mpctx->video_out && mpctx->d_video && mpctx->video_out->config_ok)
-        vo_control(mpctx->video_out, VOCTRL_RESUME, NULL);      // resume video
+    if (mpctx->video_out)
+        vo_set_paused(mpctx->video_out, false);
+
     (void)get_relative_time(mpctx);     // ignore time that passed during pause
 
 end:

player/video.c

@@ -278,8 +278,7 @@ int reinit_video_chain(struct MPContext *mpctx)
     vo_control(mpctx->video_out, saver_state ? VOCTRL_RESTORE_SCREENSAVER
                                              : VOCTRL_KILL_SCREENSAVER, NULL);
 
-    vo_control(mpctx->video_out, mpctx->paused ? VOCTRL_PAUSE
-                                               : VOCTRL_RESUME, NULL);
+    vo_set_paused(mpctx->video_out, mpctx->paused);
 
     mpctx->sync_audio_to_video = !sh->attached_picture;
     mpctx->vo_pts_history_seek_ts++;
@@ -343,7 +342,7 @@ static int check_framedrop(struct MPContext *mpctx)
         if (d < -mpctx->dropped_frames * frame_time - 0.100) {
             mpctx->drop_frame_cnt++;
             mpctx->dropped_frames++;
-            return mpctx->opts->frame_dropping;
+            return !!(mpctx->opts->frame_dropping & 2);
         } else
             mpctx->dropped_frames = 0;
     }
@@ -576,7 +575,8 @@ static int update_video(struct MPContext *mpctx, double endpts)
     }
 
 
-    bool vo_framedrop = !!mpctx->video_out->driver->flip_page_timed;
+    //bool vo_framedrop = !!mpctx->video_out->driver->flip_page_timed;
+    bool vo_framedrop = !!(mpctx->opts->frame_dropping & 1);
     int min_frames = vo_framedrop ? 2 : 1; // framedrop needs duration
 
     // Already enough video buffered?

video/out/vo.c

@@ -130,9 +130,16 @@ struct vo_internal {
 
     bool hasframe;
     bool request_redraw;
+    bool paused;
 
     int64_t flip_queue_offset; // queue flip events at most this much in advance
 
+    int64_t last_flip;
+    int64_t vsync_interval;
+    int64_t drop_count;
+    bool dropped_frame;             // the previous frame was dropped
+    struct mp_image *dropped_image; // used to possibly redraw the dropped frame
+
     int64_t wakeup_pts;             // time at which to pull frame from decoder
 
     bool rendering;                 // true if an image is being rendered
@@ -323,6 +330,11 @@ static void run_reconfig(void *p)
         vo->params = NULL;
     }
     forget_frames(vo); // implicitly synchronized
+
+    double display_fps = 1000.0; // assume infinite if unset
+    vo->driver->control(vo, VOCTRL_GET_DISPLAY_FPS, &display_fps);
+    vo->in->vsync_interval = 1e6 / display_fps;
+    MP_VERBOSE(vo, "Assuming %f FPS for framedrop.\n", display_fps);
 }
 
 int vo_reconfig(struct vo *vo, struct mp_image_params *params, int flags)
@@ -358,7 +370,9 @@ static void forget_frames(struct vo *vo)
 {
     struct vo_internal *in = vo->in;
     in->hasframe = false;
+    in->drop_count = 0;
     mp_image_unrefp(&in->frame_queued);
+    mp_image_unrefp(&in->dropped_image);
 }
 
 #ifndef __MINGW32__
@@ -496,6 +510,17 @@ void vo_wait_frame(struct vo *vo)
     pthread_mutex_unlock(&in->lock);
 }
 
+static int64_t prev_sync(struct vo *vo, int64_t ts)
+{
+    struct vo_internal *in = vo->in;
+
+    int64_t diff = (int64_t)(ts - in->last_flip);
+    int64_t offset = diff % MPMAX(in->vsync_interval, 1);
+    if (offset < 0)
+        offset += in->vsync_interval;
+    return ts - offset;
+}
+
 static bool render_frame(struct vo *vo)
 {
     struct vo_internal *in = vo->in;
@@ -510,33 +535,59 @@ static bool render_frame(struct vo *vo)
         return false;
     }
 
+    mp_image_unrefp(&in->dropped_image);
+
     in->rendering = true;
     in->frame_queued = NULL;
 
-    pthread_mutex_unlock(&in->lock);
+    // The next time a flip (probably) happens.
+    int64_t next_vsync = prev_sync(vo, mp_time_us()) + in->vsync_interval;
+    int64_t end_time = pts + duration;
 
-    vo->driver->draw_image(vo, img);
+    in->dropped_frame = end_time < next_vsync;
+    in->dropped_frame &= !!(vo->global->opts->frame_dropping & 1);
+    in->dropped_frame &= !(vo->driver->caps & VO_CAP_FRAMEDROP) &&
+                         !vo->driver->untimed && !vo->driver->encode;
+    // Even if we're hopelessly behind, rather degrade to 10 FPS playback,
+    // instead of just freezing the display forever.
+    in->dropped_frame &= mp_time_us() - in->last_flip < 100 * 1000;
 
-    int64_t target = pts - in->flip_queue_offset;
-    while (1) {
-        int64_t now = mp_time_us();
-        if (target <= now)
-            break;
-        mp_sleep_us(target - now);
+    if (in->dropped_frame) {
+        in->drop_count += 1;
+        in->dropped_image = img;
+    } else {
+        pthread_mutex_unlock(&in->lock);
+
+        vo->driver->draw_image(vo, img);
+
+        int64_t target = pts - in->flip_queue_offset;
+        while (1) {
+            int64_t now = mp_time_us();
+            if (target <= now)
+                break;
+            mp_sleep_us(target - now);
+        }
+
+        if (vo->driver->flip_page_timed)
+            vo->driver->flip_page_timed(vo, pts, duration);
+        else
+            vo->driver->flip_page(vo);
+
+        in->last_flip = mp_time_us();
+
+        MP_DBG(vo, "phase: %ld\n", (long)(in->last_flip % in->vsync_interval));
+
+        pthread_mutex_lock(&in->lock);
     }
 
-    if (vo->driver->flip_page_timed)
-        vo->driver->flip_page_timed(vo, pts, duration);
-    else
-        vo->driver->flip_page(vo);
-
     vo->want_redraw = false;
 
-    pthread_mutex_lock(&in->lock);
     in->request_redraw = false;
     in->rendering = false;
+
     pthread_cond_signal(&in->wakeup); // for vo_wait_frame()
     mp_input_wakeup(vo->input_ctx);
+
     pthread_mutex_unlock(&in->lock);
 
     return true;
@@ -546,15 +597,28 @@ static void do_redraw(struct vo *vo)
 {
     struct vo_internal *in = vo->in;
 
-    pthread_mutex_lock(&in->lock);
-    in->request_redraw = false;
-    pthread_mutex_unlock(&in->lock);
-
     vo->want_redraw = false;
 
-    if (!vo->config_ok || vo->driver->control(vo, VOCTRL_REDRAW_FRAME, NULL) < 1)
+    pthread_mutex_lock(&in->lock);
+    in->request_redraw = false;
+    bool skip = !in->paused && in->dropped_frame;
+    struct mp_image *img = in->dropped_image;
+    if (!skip) {
+        in->dropped_image = NULL;
+        in->dropped_frame = false;
+    }
+    pthread_mutex_unlock(&in->lock);
+
+    if (!vo->config_ok || skip)
         return;
 
+    if (img) {
+        vo->driver->draw_image(vo, img);
+    } else {
+        if (vo->driver->control(vo, VOCTRL_REDRAW_FRAME, NULL) < 1)
+            return;
+    }
+
     if (vo->driver->flip_page_timed)
         vo->driver->flip_page_timed(vo, 0, -1);
     else
@@ -601,6 +665,27 @@ static void *vo_thread(void *ptr)
     return NULL;
 }
 
+void vo_set_paused(struct vo *vo, bool paused)
+{
+    struct vo_internal *in = vo->in;
+    pthread_mutex_lock(&in->lock);
+    if (in->paused != paused) {
+        in->paused = true;
+        if (in->paused && in->dropped_frame)
+            in->request_redraw = true;
+    }
+    pthread_mutex_unlock(&in->lock);
+    vo_control(vo, paused ? VOCTRL_PAUSE : VOCTRL_RESUME, NULL);
+}
+
+int64_t vo_get_drop_count(struct vo *vo)
+{
+    pthread_mutex_lock(&vo->in->lock);
+    int64_t r = vo->in->drop_count;
+    pthread_mutex_unlock(&vo->in->lock);
+    return r;
+}
+
 // Make the VO redraw the OSD at some point in the future.
 void vo_redraw(struct vo *vo)
 {

video/out/vo.h

@@ -85,6 +85,7 @@ enum mp_voctrl {
     VOCTRL_SET_COMMAND_LINE,            // char**
 
     VOCTRL_GET_ICC_PROFILE_PATH,        // char**
+    VOCTRL_GET_DISPLAY_FPS,             // double*
 
     VOCTRL_GET_PREF_DEINT,              // int*
 };
@@ -134,6 +135,8 @@ struct voctrl_screenshot_args {
 
 // VO does handle mp_image_params.rotate in 90 degree steps
 #define VO_CAP_ROTATE90 1
+// VO does framedrop itself (vo_vdpau). Untimed/encoding VOs never drop.
+#define VO_CAP_FRAMEDROP 2
 
 struct vo;
 struct osd_state;
@@ -282,6 +285,8 @@ bool vo_has_frame(struct vo *vo);
 void vo_redraw(struct vo *vo);
 void vo_seek_reset(struct vo *vo);
 void vo_destroy(struct vo *vo);
+void vo_set_paused(struct vo *vo, bool paused);
+int64_t vo_get_drop_count(struct vo *vo);
 
 void vo_set_flip_queue_offset(struct vo *vo, int64_t us);
 void vo_wakeup(struct vo *vo);

video/out/vo_vdpau.c

@@ -1138,6 +1138,7 @@ static int control(struct vo *vo, uint32_t request, void *data)
 const struct vo_driver video_out_vdpau = {
     .description = "VDPAU with X11",
     .name = "vdpau",
+    .caps = VO_CAP_FRAMEDROP,
     .preinit = preinit,
     .query_format = query_format,
     .reconfig = reconfig,

video/out/x11_common.c

@@ -1492,6 +1492,13 @@ int vo_x11_control(struct vo *vo, int *events, int request, void *arg)
     case VOCTRL_UPDATE_WINDOW_TITLE:
         vo_x11_update_window_title(vo);
         return VO_TRUE;
+    case VOCTRL_GET_DISPLAY_FPS: {
+        double fps = vo_x11_vm_get_fps(vo);
+        if (fps <= 0)
+            break;
+        *(double *)arg = fps;
+        return VO_TRUE;
+    }
     }
     return VO_NOTIMPL;
 }