core: add backstep support

Allows stepping back one frame via the frame_back_step inout command,
bound to "," by default.

This uses the precise seeking facility, and a perfect frame index built
on the fly. The index is built during playback and precise seeking, and
contains (as of this commit) the last 100 displayed or skipped frames.
This index is used to find the PTS of the previous frame, which is then
used as target for a precise seek. If no PTS is found, the core attempts
to do a seek before the current frame, and skip decoded frames until the
current frame is reached; this will create a sufficient index and the
normal backstep algorithm can be applied.

This can be rather slow. The worst case for backstepping is about the
same as the worst case for precise seeking if the previous frame can be
deduced from the index. If not, the worst case will be twice as slow.

There's also some minor danger that the index is incorrect in case
framedropping is involved. For framedropping due to --framedrop, this
problem is ignored (use of --framedrop is discouraged anyway). For
framedropping during precise seeking (done to make it faster), we try
to not add frames to the index that are produced when this can happen.
I'm not sure how well that works (or if the logic is sane), and it's
sure to break with some video filters. In the worst case, backstepping
might silently skip frames if you backstep after a user-initiated
precise seek. (Precise seeks to do indexing are not affected.)

Likewise, video filters that somehow change timing of frames and do not
do this in a deterministic way (i.e. if you seek to a position, frames
with different timings are produced than when the position is reached
during normal playback) will make backstepping silently jump to the
wrong frame. Enabling/disabling filters during playback (like for
example deinterlacing) will have similar bad effects.

DOCS/man/en/input.rst

@@ -79,6 +79,17 @@ seek <seconds> [relative|absolute|absolute-percent|- [default-precise|exact|keyf
 frame_step
     Play one frame, then pause.
 
+frame_back_step
+    Go back by one frame, then pause. Note that this can be very slow (it tries
+    to be precise, not fast), and sometimes fails to behave as expected. How
+    well this works depends on whether precise seeking works correctly (e.g.
+    see the ``--hr-seek-demuxer-offset`` option). Video filters or other video
+    postprocessing that modifies timing of frames (e.g. deinterlacing) should
+    usually work, but might make backstepping silently behave incorrectly in
+    corner cases.
+
+    This doesn't work with audio-only playback.
+
 set <property> "<value>"
     Set the given property to the given value.
 

core/command.c

@@ -1832,7 +1832,11 @@ void run_command(MPContext *mpctx, mp_cmd_t *cmd)
     }
 
     case MP_CMD_FRAME_STEP:
-        add_step_frame(mpctx);
+        add_step_frame(mpctx, 1);
+        break;
+
+    case MP_CMD_FRAME_BACK_STEP:
+        add_step_frame(mpctx, -1);
         break;
 
     case MP_CMD_QUIT:

core/input/input.c

@@ -128,6 +128,7 @@ static const mp_cmd_t mp_cmds[] = {
   { MP_CMD_QUIT, "quit", { OARG_INT(0) } },
   { MP_CMD_STOP, "stop", },
   { MP_CMD_FRAME_STEP, "frame_step", },
+  { MP_CMD_FRAME_BACK_STEP, "frame_back_step", },
   { MP_CMD_PLAYLIST_NEXT, "playlist_next", {
       OARG_CHOICE(0, ({"weak", 0},              {"0", 0},
                       {"force", 1},             {"1", 1})),

core/input/input.h

@@ -44,6 +44,7 @@ enum mp_command_type {
     MP_CMD_TV_SET_FREQ,
     MP_CMD_TV_SET_NORM,
     MP_CMD_FRAME_STEP,
+    MP_CMD_FRAME_BACK_STEP,
     MP_CMD_SPEED_MULT,
     MP_CMD_RUN,
     MP_CMD_SUB_ADD,

core/mp_core.h

@@ -117,6 +117,10 @@ struct track {
     struct sub_data *subdata;
 };
 
+enum {
+    MAX_NUM_VO_PTS = 100,
+};
+
 typedef struct MPContext {
     struct MPOpts opts;
     struct m_config *mconfig;
@@ -219,6 +223,15 @@ typedef struct MPContext {
     // Video PTS, or audio PTS if video has ended.
     double playback_pts;
 
+    // History of video frames timestamps that were queued in the VO
+    // This includes even skipped frames during hr-seek
+    double vo_pts_history_pts[MAX_NUM_VO_PTS];
+    // Whether the PTS at vo_pts_history[n] is after a seek reset
+    uint64_t vo_pts_history_seek[MAX_NUM_VO_PTS];
+    uint64_t vo_pts_history_seek_ts;
+    uint64_t backstep_start_seek_ts;
+    bool backstep_active;
+
     float audio_delay;
 
     unsigned int last_heartbeat;
@@ -287,7 +300,7 @@ struct track *mp_add_subtitles(struct MPContext *mpctx, char *filename,
 int reinit_video_chain(struct MPContext *mpctx);
 void pause_player(struct MPContext *mpctx);
 void unpause_player(struct MPContext *mpctx);
-void add_step_frame(struct MPContext *mpctx);
+void add_step_frame(struct MPContext *mpctx, int dir);
 void queue_seek(struct MPContext *mpctx, enum seek_type type, double amount,
                 int exact);
 int seek_chapter(struct MPContext *mpctx, int chapter, double *seek_pts);

core/mplayer.c

@@ -2404,6 +2404,7 @@ int reinit_video_chain(struct MPContext *mpctx)
     sh_video->next_frame_time = 0;
     mpctx->restart_playback = true;
     mpctx->delay = 0;
+    mpctx->vo_pts_history_seek_ts++;
 
     // ========== Init display (sh_video->disp_w*sh_video->disp_h/out_fmt) ============
 
@@ -2419,6 +2420,40 @@ no_video:
     return 0;
 }
 
+static void add_frame_pts(struct MPContext *mpctx, double pts)
+{
+    if (pts == MP_NOPTS_VALUE || mpctx->hrseek_framedrop) {
+        mpctx->vo_pts_history_seek_ts++; // mark discontinuity
+        return;
+    }
+    for (int n = MAX_NUM_VO_PTS - 1; n >= 1; n--) {
+        mpctx->vo_pts_history_seek[n] = mpctx->vo_pts_history_seek[n - 1];
+        mpctx->vo_pts_history_pts[n] = mpctx->vo_pts_history_pts[n - 1];
+    }
+    mpctx->vo_pts_history_seek[0] = mpctx->vo_pts_history_seek_ts;
+    mpctx->vo_pts_history_pts[0] = pts;
+}
+
+static double find_previous_pts(struct MPContext *mpctx, double pts)
+{
+    for (int n = 0; n < MAX_NUM_VO_PTS - 1; n++) {
+        if (pts == mpctx->vo_pts_history_pts[n] &&
+            mpctx->vo_pts_history_seek[n] != 0 &&
+            mpctx->vo_pts_history_seek[n] == mpctx->vo_pts_history_seek[n + 1])
+        {
+            return mpctx->vo_pts_history_pts[n + 1];
+        }
+    }
+    return MP_NOPTS_VALUE;
+}
+
+static double get_last_frame_pts(struct MPContext *mpctx)
+{
+    if (mpctx->vo_pts_history_seek[0] == mpctx->vo_pts_history_seek_ts)
+        return mpctx->vo_pts_history_pts[0];
+    return MP_NOPTS_VALUE;
+}
+
 static bool filter_output_queued_frame(struct MPContext *mpctx)
 {
     struct sh_video *sh_video = mpctx->sh_video;
@@ -2571,6 +2606,7 @@ static double update_video(struct MPContext *mpctx)
         if (pts == MP_NOPTS_VALUE)
             pts = sh_video->last_pts;
     }
+    add_frame_pts(mpctx, pts);
     if (mpctx->hrseek_active && pts < mpctx->hrseek_pts - .005) {
         vo_skip_frame(video_out);
         return 0;
@@ -2658,12 +2694,20 @@ static bool redraw_osd(struct MPContext *mpctx)
     return true;
 }
 
-void add_step_frame(struct MPContext *mpctx)
+void add_step_frame(struct MPContext *mpctx, int dir)
 {
-    mpctx->step_frames++;
-    if (mpctx->video_out && mpctx->sh_video && mpctx->video_out->config_ok)
-        vo_control(mpctx->video_out, VOCTRL_PAUSE, NULL);
-    unpause_player(mpctx);
+    if (dir > 0) {
+        mpctx->step_frames += 1;
+        if (mpctx->video_out && mpctx->sh_video && mpctx->video_out->config_ok)
+            vo_control(mpctx->video_out, VOCTRL_PAUSE, NULL);
+        unpause_player(mpctx);
+    } else if (dir < 0) {
+        if (!mpctx->backstep_active && !mpctx->hrseek_active) {
+            mpctx->backstep_active = true;
+            mpctx->backstep_start_seek_ts = mpctx->vo_pts_history_seek_ts;
+            pause_player(mpctx);
+        }
+    }
 }
 
 static void seek_reset(struct MPContext *mpctx, bool reset_ao, bool reset_ac)
@@ -2701,6 +2745,8 @@ static void seek_reset(struct MPContext *mpctx, bool reset_ao, bool reset_ac)
     mpctx->drop_frame_cnt = 0;
     mpctx->dropped_frames = 0;
     mpctx->playback_pts = MP_NOPTS_VALUE;
+    mpctx->vo_pts_history_seek_ts++;
+    mpctx->backstep_active = false;
 
 #ifdef CONFIG_ENCODING
     encode_lavc_discontinuity(mpctx->encode_lavc_ctx);
@@ -3488,7 +3534,7 @@ static void run_playloop(struct MPContext *mpctx)
                     sleeptime = 0;
                 } else if (mpctx->paused && video_left) {
                     // force redrawing OSD by framestepping
-                    add_step_frame(mpctx);
+                    add_step_frame(mpctx, 1);
                     sleeptime = 0;
                 }
             }
@@ -3525,6 +3571,46 @@ static void run_playloop(struct MPContext *mpctx)
             break;
     }
 
+    if (mpctx->backstep_active) {
+        double current_pts = mpctx->last_vo_pts;
+        mpctx->backstep_active = false;
+        if (mpctx->sh_video && current_pts != MP_NOPTS_VALUE) {
+            double seek_pts = find_previous_pts(mpctx, current_pts);
+            if (seek_pts != MP_NOPTS_VALUE) {
+                queue_seek(mpctx, MPSEEK_ABSOLUTE, seek_pts, 1);
+            } else {
+                double last = get_last_frame_pts(mpctx);
+                if (last != MP_NOPTS_VALUE && last >= current_pts &&
+                    mpctx->backstep_start_seek_ts != mpctx->vo_pts_history_seek_ts)
+                {
+                    mp_msg(MSGT_CPLAYER, MSGL_ERR, "Backstep failed.\n");
+                    queue_seek(mpctx, MPSEEK_ABSOLUTE, current_pts, 1);
+                } else if (!mpctx->hrseek_active) {
+                    mp_msg(MSGT_CPLAYER, MSGL_V, "Start backstep indexing.\n");
+                    // Force it to index the video up until current_pts.
+                    // The whole point is getting frames _before_ that PTS,
+                    // so apply an arbitrary offset. (In theory the offset
+                    // has to be large enough to reach the previous frame.)
+                    seek(mpctx, (struct seek_params){
+                                .type = MPSEEK_ABSOLUTE,
+                                .amount = current_pts - 1.0,
+                                }, false);
+                    // Don't leave hr-seek mode. If all goes right, hr-seek
+                    // mode is cancelled as soon as the frame before
+                    // current_pts is found during hr-seeking.
+                    // Note that current_pts should be part of the index,
+                    // otherwise we can't find the previous frame, so set the
+                    // seek target an arbitrary amount of time after it.
+                    mpctx->hrseek_pts = current_pts + 10.0;
+                    mpctx->hrseek_framedrop = false;
+                    mpctx->backstep_active = true;
+                } else {
+                    mpctx->backstep_active = true;
+                }
+            }
+        }
+    }
+
     // handle -sstep
     if (opts->step_sec > 0 && !mpctx->stop_play && !mpctx->paused &&
         !mpctx->restart_playback)
@@ -4128,6 +4214,7 @@ goto_enable_cache: ;
     mpctx->hrseek_active = false;
     mpctx->hrseek_framedrop = false;
     mpctx->step_frames = 0;
+    mpctx->backstep_active = false;
     mpctx->total_avsync_change = 0;
     mpctx->last_chapter_seek = -2;
     mpctx->playing_msg_shown = false;

etc/input.conf

@@ -60,6 +60,7 @@ q {encode} quit
 ESC quit
 p cycle pause                           # toggle pause/playback mode
 . frame_step                            # advance one frame and pause
+, frame_back_step                       # go back by one frame and pause
 SPACE cycle pause
 > playlist_next                         # skip to next file
 ENTER playlist_next force               # skip to next file or quit