From b3689dd693cf895cc441550c96d16afc05dc7099 Mon Sep 17 00:00:00 2001
From: Jeremy Soller <jackpot51@gmail.com>
Date: Thu, 25 Jan 2024 15:47:09 -0700
Subject: [PATCH] Calculate video drift all the way in application update

---
 src/main.rs   | 26 ++++++++++++++++++++++----
 src/player.rs | 30 ++++++++++--------------------
 2 files changed, 32 insertions(+), 24 deletions(-)

diff --git a/src/main.rs b/src/main.rs
index 7ff2904..a4b96a3 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -20,7 +20,7 @@ use std::{
     path::PathBuf,
     process,
     sync::{mpsc, Arc, Mutex},
-    time::Instant,
+    time::{Duration, Instant},
 };
 
 use config::{AppTheme, Config, CONFIG_VERSION};
@@ -295,7 +295,7 @@ impl Application for App {
             Message::SystemThemeModeChange(_theme_mode) => {
                 return self.update_config();
             }
-            Message::Tick(_time) => {
+            Message::Tick(frame_time) => {
                 let start = Instant::now();
 
                 let video_frame_opt = {
@@ -303,8 +303,11 @@ impl Application for App {
                     //TODO: show frames at desired presentation time instead of clearing
                     let mut video_frame_opt = video_frames.pop_front();
                     while video_frames.len() >= 4 {
-                        if let Some(video_frame) = video_frames.pop_front() {
-                            video_frame_opt = Some(video_frame);
+                        if let Some(extra_frame) = video_frames.pop_front() {
+                            if let Some(old_frame) = video_frame_opt {
+                                log::warn!("skipping video frame {:?}", old_frame.0.pts());
+                            }
+                            video_frame_opt = Some(extra_frame);
                         }
                     }
                     video_frame_opt
@@ -312,6 +315,7 @@ impl Application for App {
                 match video_frame_opt {
                     Some(video_frame) => {
                         let pts = video_frame.0.pts();
+                        let present_time_opt = video_frame.1;
                         self.handle_opt = Some(video_frame.into_handle());
 
                         let duration = start.elapsed();
@@ -320,6 +324,20 @@ impl Application for App {
                             pts,
                             duration
                         );
+
+                        if let Some(present_time) = present_time_opt {
+                            if present_time > frame_time {
+                                let ahead = present_time - frame_time;
+                                if ahead > Duration::from_millis(1) {
+                                    log::debug!("video ahead {:?}", ahead);
+                                }
+                            } else {
+                                let behind = frame_time - present_time;
+                                if behind > Duration::from_millis(1) {
+                                    log::debug!("video behind {:?}", behind);
+                                }
+                            }
+                        }
                     }
                     None => {}
                 }
diff --git a/src/player.rs b/src/player.rs
index 9fc986e..8d03be0 100644
--- a/src/player.rs
+++ b/src/player.rs
@@ -32,7 +32,7 @@ pub enum PlayerMessage {
     SeekRelative(f64),
 }
 
-pub struct VideoFrame(pub Video);
+pub struct VideoFrame(pub Video, pub Option<Instant>);
 
 impl VideoFrame {
     pub fn into_handle(self) -> widget::image::Handle {
@@ -170,8 +170,6 @@ fn ffmpeg_thread<P: AsRef<Path>>(
     let video_width = video_decoder.width();
     let video_height = video_decoder.height();
     let (cpu_frame_tx, cpu_frame_rx) = mpsc::channel::<(Video, Option<Instant>)>();
-    let min_sleep = Duration::from_millis(1);
-    let min_skip = Duration::from_millis(1);
     thread::Builder::new()
         .name("video_scale".to_string())
         .spawn(move || {
@@ -212,27 +210,19 @@ fn ffmpeg_thread<P: AsRef<Path>>(
                 video_scaler.run(&cpu_frame, &mut scaled_frame).unwrap();
                 scaled_frame.set_pts(pts_opt);
 
-                if let Some(pts) = pts_opt {
+                let present_time_opt = if let Some(pts) = pts_opt {
                     let expected_float = pts as f64 * video_time_base;
                     let expected = Duration::from_secs_f64(expected_float);
-                    if let Some(sync_time) = &sync_time_opt {
-                        // Sync with audio
-                        let actual = sync_time.elapsed();
-                        if expected > actual {
-                            let sleep = expected - actual;
-                            if sleep > min_sleep {
-                                log::debug!("video ahead {:?}", sleep);
-                            }
-                        } else {
-                            let skip = actual - expected;
-                            if skip > min_skip {
-                                log::debug!("video behind {:?}", skip);
-                            }
-                        }
+                    if let Some(sync_time) = sync_time_opt {
+                        Some(sync_time + expected)
+                    } else {
+                        None
                     }
-                }
+                } else {
+                    None
+                };
 
-                let video_frame = VideoFrame(scaled_frame);
+                let video_frame = VideoFrame(scaled_frame, present_time_opt);
                 {
                     let mut video_frames = video_frames_lock.lock().unwrap();
                     video_frames.push_back(video_frame);