diff --git a/src/server/connection.rs b/src/server/connection.rs index 24f5bf537..8ac850958 100644 --- a/src/server/connection.rs +++ b/src/server/connection.rs @@ -299,6 +299,12 @@ pub struct Connection { tx_input: std_mpsc::Sender, // handle input messages video_ack_required: bool, + // Diagnostics only, gated by `RUSTDESK_QOS_VERBOSE`: how long the shared + // write path blocked this second. The video send is inline in the message + // loop, so a slow write also delays the delay probe and its reply. + video_send_max_ms: u32, + video_send_sum_ms: u32, + video_send_count: u32, server_audit_conn: String, server_audit_file: String, controlled_context: Option, @@ -504,6 +510,9 @@ impl Connection { show_my_cursor: false, tx_input, video_ack_required: false, + video_send_max_ms: 0, + video_send_sum_ms: 0, + video_send_count: 0, server_audit_conn: "".to_owned(), server_audit_file: "".to_owned(), controlled_context, @@ -950,10 +959,17 @@ impl Connection { video_service::notify_video_frame_fetched(vf.display as usize, id, Some(instant.into())); } } + let send_begin = video_service::qos_diag_verbose().then(Instant::now); if let Err(err) = conn.stream.send(&value as &Message).await { conn.on_close(&err.to_string(), false).await; break; } + if let Some(begin) = send_begin { + let blocked = begin.elapsed().as_millis() as u32; + conn.video_send_max_ms = conn.video_send_max_ms.max(blocked); + conn.video_send_sum_ms = conn.video_send_sum_ms.saturating_add(blocked); + conn.video_send_count += 1; + } }, Some((instant, value)) = rx.recv() => { let latency = instant.elapsed().as_millis() as i64; @@ -1040,6 +1056,21 @@ impl Connection { break; } } + if video_service::qos_diag_verbose() && conn.video_send_count > 0 { + // Joined with `qos_trace` on `t`: a probe that waits behind a + // blocked write is not a slow network. + log::debug!( + "qos_send t={} id={id} frames={} send_max={} send_sum={} queued={}", + hbb_common::get_time(), + conn.video_send_count, + conn.video_send_max_ms, + conn.video_send_sum_ms, + rx_video.len() + ); + conn.video_send_max_ms = 0; + conn.video_send_sum_ms = 0; + conn.video_send_count = 0; + } conn.file_remove_log_control.on_timer().drain(..).map(|x| conn.send_to_cm(x)).count(); #[cfg(feature = "hwcodec")] conn.update_supported_encoding(); diff --git a/src/server/video_qos.rs b/src/server/video_qos.rs index b02f6adf3..a1f2859b9 100644 --- a/src/server/video_qos.rs +++ b/src/server/video_qos.rs @@ -7,24 +7,44 @@ use std::{ /* FPS adjust: -a. new user connected =>set to INIT_FPS -b. TestDelay receive => update user's fps according to network delay - When network delay < DELAY_THRESHOLD_150MS, set minimum fps according to image quality, and increase fps; - When network delay >= DELAY_THRESHOLD_150MS, set minimum fps according to image quality, and decrease fps; -c. second timeout / TestDelay receive => update real fps to the minimum fps from all users +a. new user connected => set to INIT_FPS +b. TestDelay reply => update the user's fps from the excess delay, the reply's delay + above the baseline this connection has shown so far: + startup: two consecutive replies with excess < 50 ms permit doubling toward + the viewer's cap; a higher excess or a brake ends this acceleration; + excess < DELAY_THRESHOLD_150MS: a good reply; grows the fps, and after a + reduction returns to the level held before it after two good replies; + excess >= DELAY_THRESHOLD_150MS: a bad reply; nothing happens until three in a + row confirm congestion, including after each reduction. FPS drops by a + fifth at most; a second of excess cannot wait and halves it immediately. + A recent fast restore also permits halving at 600 ms of excess. + While the bitrate can still be reduced (ABR) it is reduced first and the fps keeps + a floor: bitrate-targeted encoders do not send fewer bytes at fewer frames. +c. probe outstanding for more than two seconds => halve the fps for every further + second, down to MIN_AUTO_FPS and never above the target it found; the late + reply does not reduce again. Automatic reductions respect this floor unless + the viewer requested a lower FPS cap. +d. second timeout / TestDelay reply => real fps is the minimum over all users; + every user starts at INIT_FPS, adapts from its own target and is capped by its + own limit, never by that minimum or by another user's limit ratio adjust: a. user set image quality => update to the maximum ratio of the latest quality b. 3 seconds timeout => update ratio according to network delay When network delay < DELAY_THRESHOLD_150MS, increase ratio, max 150kbps; - When network delay >= DELAY_THRESHOLD_150MS, decrease ratio; - -adjust between FPS and ratio: - When network delay < DELAY_THRESHOLD_150MS, fps is always higher than the minimum fps, and ratio is increasing; - When network delay >= DELAY_THRESHOLD_150MS, fps is always lower than the minimum fps, and ratio is decreasing; + When a user calls for a reduction (two bad replies in a row, or a probe still + out at the second tick past two seconds), decrease ratio by the step that user's + own delay and confirmation call for, the most conservative step over all users; + one slow reply or one short stall does not, and one user's spike is never paired + with another user's confirmation. +c. confirmed congestion => decrease ratio at once, when the 3 seconds cooldown allows delay: - use delay minus RTT as the actual network delay + TestDelay shares the video stream, so it measures the queue in front of it rather + than the path RTT. The baseline starts at the first reply and follows lower + delays immediately. Old minima expire after 20 fresh replies; a higher window + minimum is learned gradually only when the recent floor is no longer rising. + Outstanding-probe checks and their late replies do not age this window. */ // Constants @@ -32,6 +52,7 @@ pub const FPS: u32 = 30; pub const MIN_FPS: u32 = 1; pub const MAX_FPS: u32 = 120; pub const INIT_FPS: u32 = 15; +const MIN_AUTO_FPS: u32 = 5; // Bitrate ratio constants for different quality levels const BR_MAX: f32 = 40.0; // 2000 * 2 / 100 @@ -43,45 +64,196 @@ const HISTORY_DELAY_LEN: usize = 2; const ADJUST_RATIO_INTERVAL: usize = 3; // Adjust quality ratio every 3 seconds const DYNAMIC_SCREEN_THRESHOLD: usize = 2; // Allow increase quality ratio if encode more than 2 times in one second const DELAY_THRESHOLD_150MS: u32 = 150; // 150ms is the threshold for good network condition +const RESTORE_GUARD_SAMPLES: u8 = 5; // A restored level that congests this soon is lowered #[derive(Default, Debug, Clone)] struct UserDelay { - response_delayed: bool, + stall_ticks: u8, // timer ticks the outstanding probe has been out beyond two seconds delay_history: VecDeque, fps: Option, rtt_calculator: RttCalculator, quick_increase_fps_count: usize, increase_fps_count: usize, + consecutive_bad_samples: usize, + fps_bad_samples: u8, // fresh bad replies since the last FPS reduction + good_samples: usize, // since the last reduction, capped at 3 + replies_after_bitrate_reduction: Option, + fps_before_congestion: Option, // level to return to once replies are good again + samples_since_restore: Option, // set by a restore, cleared once it proved stable + stall_reference_fps: Option, // fps when the outstanding probe passed two seconds + startup_good_samples: u8, // u8::MAX permanently ends startup acceleration } impl UserDelay { fn add_delay(&mut self, delay: u32) { - self.rtt_calculator.update(delay); - if self.delay_history.len() > HISTORY_DELAY_LEN { + if self.delay_history.len() >= HISTORY_DELAY_LEN { self.delay_history.pop_front(); } self.delay_history.push_back(delay); } - // Average delay minus RTT - fn avg_delay(&self) -> u32 { - let len = self.delay_history.len(); - if len > 0 { - let avg_delay = self.delay_history.iter().sum::() / len as u32; - - // If RTT is available, subtract it from average delay to get actual network latency - if let Some(rtt) = self.rtt_calculator.get_rtt() { - if avg_delay > rtt { - avg_delay - rtt - } else { - avg_delay - } - } else { - avg_delay - } - } else { - DELAY_THRESHOLD_150MS + fn limit_fps_change( + &mut self, + current_fps: u32, + fps: u32, + delay: u32, + bitrate_first: bool, + braked: bool, + ) -> u32 { + // A spike stays in the average for several samples; confirm congestion with fresh samples. + let delay = delay.saturating_sub(self.rtt_calculator.get_rtt().unwrap_or_default()); + if let Some(samples) = self.samples_since_restore.as_mut() { + *samples = samples.saturating_add(1); } + if delay < DELAY_THRESHOLD_150MS { + self.consecutive_bad_samples = 0; + self.fps_bad_samples = 0; + self.replies_after_bitrate_reduction = None; + self.good_samples = (self.good_samples + 1).min(3); + return self.recover(current_fps, fps); + } + self.consecutive_bad_samples = (self.consecutive_bad_samples + 1).min(3); + self.fps_bad_samples = (self.fps_bad_samples + 1).min(3); + if let Some(replies) = self.replies_after_bitrate_reduction.as_mut() { + *replies = (*replies + 1).min(2); + } + let failed_restore = delay >= 600 + && self + .samples_since_restore + .is_some_and(|samples| samples <= RESTORE_GUARD_SAMPLES); + // A level that congests right after being restored is not the level to return to. + if self + .samples_since_restore + .is_some_and(|samples| samples <= RESTORE_GUARD_SAMPLES) + && (failed_restore || self.consecutive_bad_samples >= 3) + { + self.fps_before_congestion = Some(current_fps - current_fps / 4); + self.samples_since_restore = None; + } + // The timeout brake already reduced for the probe this reply answers. + if fps >= current_fps || braked { + return current_fps; + } + // An extra second of delay cannot wait for another confirmation. + if !failed_restore + && delay < 1000 + && (self.fps_bad_samples < 3 + || (bitrate_first && self.replies_after_bitrate_reduction.unwrap_or_default() < 2)) + { + return current_fps; + } + // A fast restore probes capacity. Roll it back promptly if the queue grows + // again, rather than waiting through another ordinary confirmation window. + let divisor = if delay >= 1000 || failed_restore { + 2 + } else { + 5 + }; + self.on_reduction(current_fps); + fps.max(current_fps.saturating_sub((current_fps / divisor).max(1))) + } + + // Fresh low-delay replies permit recovery even while the average contains a spike: + // a little at first, then back to the level held before congestion. + fn recover(&mut self, current_fps: u32, fps: u32) -> u32 { + let gradual = current_fps + (current_fps / 10).max(1); + let level = self + .fps_before_congestion + .filter(|level| *level > current_fps); + match (self.good_samples, level) { + (2 | 3, Some(level)) => { + self.fps_before_congestion = None; + self.samples_since_restore = Some(0); + fps.max(level) + } + (3, None) => { + self.fps_before_congestion = None; + fps.max(current_fps + (current_fps / 5).max(2)) + } + _ => gradual, + } + } + + fn accelerate_startup( + &mut self, + current_fps: u32, + fps: u32, + cap: u32, + delay: u32, + braked: bool, + ) -> u32 { + if self.startup_good_samples == u8::MAX { + return fps; + } + let excess = delay.saturating_sub(self.rtt_calculator.get_rtt().unwrap_or_default()); + // A low-load sample does not establish capacity: require two clean replies + // per step and abandon startup probing on the first sign of queue growth. + if braked || excess >= 50 || current_fps >= cap { + self.startup_good_samples = u8::MAX; + return fps; + } + self.startup_good_samples += 1; + if self.startup_good_samples < 2 { + return fps; + } + self.startup_good_samples = 0; + let accelerated = fps.max(current_fps.saturating_mul(2)).min(cap); + if accelerated >= cap { + self.startup_good_samples = u8::MAX; + } + accelerated + } + + // The first reduction of an episode remembers the level to return to. + fn on_reduction(&mut self, current_fps: u32) { + self.startup_good_samples = u8::MAX; + self.good_samples = 0; + self.fps_bad_samples = 0; + if self.fps_before_congestion.is_none() { + self.fps_before_congestion = Some(current_fps); + } + } + + // Bitrate is cut on confirmation only: two bad replies in a row, or a probe still + // outstanding at the second tick past two seconds. One slow reply or one short + // stall is jitter, and a static screen would never earn the cut back. + fn needs_bitrate_reduction(&self) -> bool { + self.consecutive_bad_samples >= 2 || self.stall_ticks >= 2 + } + + // The bitrate step this viewer's own evidence calls for, None when it calls for + // none. Severity and confirmation come from the same viewer; the controller + // never pairs one viewer's spike with another viewer's confirmation. + fn ratio_reduction(&self) -> Option { + if !self.needs_bitrate_reduction() { + return None; + } + let excess = self.avg_delay(); + let confirmed = self.consecutive_bad_samples >= 3; + Some(if excess < 200 { + 0.95 + } else if excess < 300 { + 0.9 + } else if excess < 500 { + if confirmed { + 0.7 + } else { + 0.85 + } + } else if confirmed { + 0.5 + } else { + 0.8 + }) + } + + // Average delay above the baseline: what the queue adds on top of the path itself. + fn avg_delay(&self) -> u32 { + if self.delay_history.is_empty() { + return DELAY_THRESHOLD_150MS; + } + let avg_delay = self.delay_history.iter().sum::() / self.delay_history.len() as u32; + avg_delay.saturating_sub(self.rtt_calculator.get_rtt().unwrap_or_default()) } } @@ -93,6 +265,20 @@ struct UserData { quality: Option<(i64, Quality)>, // (time, quality) delay: UserDelay, record: bool, + joined_at: Option, // set by on_connection_open; the start-up guard's clock +} + +impl UserData { + // The frame rate this viewer asked for, from its custom or auto-adjust limit. + fn fps_cap(&self) -> u32 { + let mut fps = self.custom_fps.unwrap_or(FPS); + if let Some(auto_adjust_fps) = self.auto_adjust_fps { + if fps == 0 || auto_adjust_fps < fps { + fps = auto_adjust_fps; + } + } + fps.clamp(MIN_FPS, MAX_FPS) + } } #[derive(Default, Debug, Clone)] @@ -110,7 +296,9 @@ pub struct VideoQoS { bitrate_store: u32, adjust_ratio_instant: Instant, abr_config: bool, - new_user_instant: Instant, + first_reply_adjusts_ratio: bool, // false on Linux, where it can create vaapi twice + #[cfg(test)] + test_now: Option, } impl Default for VideoQoS { @@ -123,11 +311,33 @@ impl Default for VideoQoS { bitrate_store: 0, adjust_ratio_instant: Instant::now(), abr_config: true, - new_user_instant: Instant::now(), + first_reply_adjusts_ratio: !cfg!(target_os = "linux"), + #[cfg(test)] + test_now: None, } } } +// Clock; tests drive a virtual clock so timing is deterministic. +impl VideoQoS { + fn now(&self) -> Instant { + #[cfg(test)] + if let Some(now) = self.test_now { + return now; + } + Instant::now() + } + + fn since(&self, instant: Instant) -> Duration { + self.now().saturating_duration_since(instant) + } + + #[cfg(test)] + fn advance_ms(&mut self, ms: u64) { + self.test_now = Some(self.now() + Duration::from_millis(ms)); + } +} + // Basic functionality impl VideoQoS { // Calculate seconds per frame based on current FPS @@ -184,9 +394,12 @@ impl VideoQoS { impl VideoQoS { // Initialize new user session pub fn on_connection_open(&mut self, id: i32) { - self.users.insert(id, UserData::default()); + let user = UserData { + joined_at: Some(self.now()), + ..Default::default() + }; + self.users.insert(id, user); self.abr_config = Config::get_option("enable-abr") != "N"; - self.new_user_instant = Instant::now(); } // Clean up user session @@ -194,7 +407,11 @@ impl VideoQoS { self.users.remove(&id); if self.users.is_empty() { *self = Default::default(); + return; } + // The stream follows the remaining viewers at once; a departed viewer's + // start-up guard left with its entry. + self.adjust_fps(); } pub fn user_custom_fps(&mut self, id: i32, fps: u32) { @@ -244,8 +461,10 @@ impl VideoQoS { } pub fn user_network_delay(&mut self, id: i32, delay: u32) { - let highest_fps = self.highest_fps(); let target_ratio = self.latest_quality().ratio(); + // Fewer frames only save bytes with encoders that size frames for a fixed rate; + // bitrate-targeted encoders keep the bitrate, so the bitrate has to come down first. + let bitrate_first = self.can_reduce_bitrate(); // For bad network, small fps means quick reaction and high quality let (min_fps, normal_fps) = if target_ratio >= BR_BEST { @@ -260,13 +479,25 @@ impl VideoQoS { let dividend_ms = DELAY_THRESHOLD_150MS * min_fps; let mut adjust_ratio = false; + let mut reduce_bitrate = false; if let Some(user) = self.users.get_mut(&id) { let delay = delay.max(10); + // The reply closes the outstanding probe, braked or not. + user.delay.stall_ticks = 0; + let braked = user.delay.stall_reference_fps.take().is_some(); let old_avg_delay = user.delay.avg_delay(); + if !braked { + user.delay.rtt_calculator.update(delay); + } user.delay.add_delay(delay); let mut avg_delay = user.delay.avg_delay(); avg_delay = avg_delay.max(10); - let mut fps = self.fps; + // Each viewer adapts from its own target, starts at INIT_FPS and is capped + // by its own limit. The stream follows the slowest viewer in adjust_fps; + // neither that minimum nor another viewer's limit feeds back into it. + let user_cap = user.fps_cap(); + let current_fps = user.delay.fps.unwrap_or(INIT_FPS.min(user_cap)); + let mut fps = current_fps; // Adaptive FPS adjustment based on network delay: if avg_delay < 50 { @@ -321,26 +552,84 @@ impl VideoQoS { user.delay.quick_increase_fps_count = 0; } - fps = fps.clamp(MIN_FPS, highest_fps); + if bitrate_first { + // While the bitrate can still come down, the frame rate keeps its floor. + fps = fps.max(min_fps); + } + fps = fps.max(MIN_AUTO_FPS.min(user_cap)); + fps = user + .delay + .limit_fps_change(current_fps, fps, delay, bitrate_first, braked); + fps = user + .delay + .accelerate_startup(current_fps, fps, user_cap, delay, braked); + reduce_bitrate = bitrate_first + && user.delay.needs_bitrate_reduction() + && user.delay.replies_after_bitrate_reduction.is_none(); + fps = fps.clamp(MIN_FPS, user_cap); // first network delay message adjust_ratio = user.delay.fps.is_none(); user.delay.fps = Some(fps); + let base = user.delay.rtt_calculator.get_rtt().unwrap_or_default(); + log::debug!( + "qos_trace t={} id={id} delay={delay} base={base} excess={} avg={avg_delay} bad={} good={} braked={braked} fps={fps} ratio={:.3} reduce_bitrate={reduce_bitrate}", + hbb_common::get_time(), + delay.saturating_sub(base), + user.delay.consecutive_bad_samples, + user.delay.good_samples, + self.ratio, + ); } self.adjust_fps(); - if adjust_ratio && !cfg!(target_os = "linux") { - //Reduce the possibility of vaapi being created twice + // A viewer's first reply is one more trigger of the periodic adjustment and + // keeps its cooldown: a viewer joining right after a cut must not spend the + // other viewers' evidence a second time. + if adjust_ratio + && self.first_reply_adjusts_ratio + && self.since(self.adjust_ratio_instant).as_secs() >= ADJUST_RATIO_INTERVAL as u64 + { + self.adjust_ratio(false); + } + if reduce_bitrate + && self.since(self.adjust_ratio_instant).as_secs() >= ADJUST_RATIO_INTERVAL as u64 + { self.adjust_ratio(false); } } pub fn user_delay_response_elapsed(&mut self, id: i32, elapsed: u128) { - if let Some(user) = self.users.get_mut(&id) { - user.delay.response_delayed = elapsed > 2000; - if user.delay.response_delayed { - user.delay.add_delay(elapsed as u32); - self.adjust_fps(); - } + let Some(user) = self.users.get_mut(&id) else { + return; + }; + if elapsed <= 2000 { + return; } + user.delay.stall_ticks = user.delay.stall_ticks.saturating_add(1); + user.delay.add_delay(elapsed as u32); + // Halve for every second the probe stays out beyond the first: two seconds + // halve, three quarter, and so on down to the floor. + let reference = match user.delay.stall_reference_fps { + Some(reference) => reference, + None => { + let reference = user.delay.fps.unwrap_or(INIT_FPS.min(user.fps_cap())); + user.delay.stall_reference_fps = Some(reference); + user.delay.on_reduction(reference); + reference + } + }; + let divisor = 1u32 << ((elapsed / 1000) as u32).saturating_sub(1).min(5); + let user_cap = user.fps_cap(); + // The floor is a floor, not a lift: a target already below it stays. + let current = user.delay.fps.unwrap_or(reference); + let fps = (reference / divisor) + .clamp(MIN_AUTO_FPS.min(user_cap), user_cap) + .min(current); + user.delay.fps = Some(fps); + log::debug!( + "qos_trace t={} id={id} timeout={elapsed} fps={fps}", + hbb_common::get_time() + ); + self.adjust_fps(); } } @@ -362,14 +651,11 @@ impl VideoQoS { self.adjust_fps(); let abr_enabled = self.in_vbr_state(); if abr_enabled { - if self.adjust_ratio_instant.elapsed().as_secs() >= ADJUST_RATIO_INTERVAL as u64 { + if self.since(self.adjust_ratio_instant).as_secs() >= ADJUST_RATIO_INTERVAL as u64 { let dynamic_screen = self .displays .iter() .any(|d| d.1.send_counter >= ADJUST_RATIO_INTERVAL * DYNAMIC_SCREEN_THRESHOLD); - self.displays.iter_mut().for_each(|d| { - d.1.send_counter = 0; - }); self.adjust_ratio(dynamic_screen); } } else { @@ -379,25 +665,12 @@ impl VideoQoS { #[inline] fn highest_fps(&self) -> u32 { - let user_fps = |u: &UserData| { - let mut fps = u.custom_fps.unwrap_or(FPS); - if let Some(auto_adjust_fps) = u.auto_adjust_fps { - if fps == 0 || auto_adjust_fps < fps { - fps = auto_adjust_fps; - } - } - fps - }; - - let fps = self - .users - .iter() - .map(|(_, u)| user_fps(u)) - .filter(|u| *u >= MIN_FPS) + self.users + .values() + .map(|u| u.fps_cap()) .min() - .unwrap_or(FPS); - - fps.clamp(MIN_FPS, MAX_FPS) + .unwrap_or(FPS) + .clamp(MIN_FPS, MAX_FPS) } // Get latest quality settings from all users @@ -412,40 +685,16 @@ impl VideoQoS { .1 } - // Adjust quality ratio based on network delay and screen changes - fn adjust_ratio(&mut self, dynamic_screen: bool) { - if !self.in_vbr_state() { - return; - } - // Get maximum delay from all users - let max_delay = self.users.iter().map(|u| u.1.delay.avg_delay()).max(); - let Some(max_delay) = max_delay else { - return; - }; - - let target_quality = self.latest_quality(); - let target_ratio = self.latest_quality().ratio(); - let current_ratio = self.ratio; + // Lowest ratio the latest quality allows: keeps about 1Mbps at high resolutions. + fn min_ratio(&self) -> f32 { let current_bitrate = self.bitrate(); - - // Calculate minimum ratio for high resolution (1Mbps baseline) let ratio_1mbps = if current_bitrate > 0 { - Some((current_ratio * 1000.0 / current_bitrate as f32).max(BR_MIN_HIGH_RESOLUTION)) + Some((self.ratio * 1000.0 / current_bitrate as f32).max(BR_MIN_HIGH_RESOLUTION)) } else { None }; - - // Calculate ratio for adding 150kbps bandwidth - let ratio_add_150kbps = if current_bitrate > 0 { - Some((current_bitrate + 150) as f32 * current_ratio / current_bitrate as f32) - } else { - None - }; - - // Set minimum ratio based on quality mode - let min = match target_quality { + match self.latest_quality() { Quality::Best => { - // For Best quality, ensure minimum 1Mbps for high resolution let mut min = BR_BEST / 2.5; if let Some(ratio_1mbps) = ratio_1mbps { if min > ratio_1mbps { @@ -463,15 +712,67 @@ impl VideoQoS { } min.max(BR_MIN_HIGH_RESOLUTION) } - Quality::Low => BR_MIN_HIGH_RESOLUTION, - Quality::Custom(_) => BR_MIN_HIGH_RESOLUTION, + Quality::Low | Quality::Custom(_) => BR_MIN_HIGH_RESOLUTION, + } + } + + // Whether congestion can still be answered with a lower bitrate. Within two + // percent of the floor another step is not worth waiting a cooldown for. + fn can_reduce_bitrate(&self) -> bool { + self.in_vbr_state() && !self.displays.is_empty() && self.ratio > self.min_ratio() * 1.02 + } + + // Every ratio adjustment starts a new window for the dynamic screen counters. + fn reset_send_counters(&mut self) { + self.displays.values_mut().for_each(|d| d.send_counter = 0); + } + + // Adjust quality ratio based on network delay and screen changes + fn adjust_ratio(&mut self, dynamic_screen: bool) { + if !self.in_vbr_state() { + return; + } + // Get maximum delay from all users + let max_delay = self.users.iter().map(|u| u.1.delay.avg_delay()).max(); + let Some(max_delay) = max_delay else { + return; }; + // Each viewer judges its own delay; the stream takes the most conservative + // step any viewer asks for. + let reduction = self + .users + .values() + .filter_map(|u| u.delay.ratio_reduction()) + .reduce(f32::min); + if reduction.is_none() && max_delay >= DELAY_THRESHOLD_150MS { + // Elevated but unconfirmed: no change, and no cooldown either, so a + // confirmation on the next reply is acted on at once. + self.reset_send_counters(); + return; + } + + let target_ratio = self.latest_quality().ratio(); + let current_ratio = self.ratio; + let current_bitrate = self.bitrate(); + + // Calculate ratio for adding 150kbps bandwidth + let ratio_add_150kbps = if current_bitrate > 0 { + Some((current_bitrate + 150) as f32 * current_ratio / current_bitrate as f32) + } else { + None + }; + + let min = self.min_ratio(); let max = target_ratio * MAX_BR_MULTIPLE; let mut v = current_ratio; - // Adjust ratio based on network delay thresholds - if max_delay < 50 { + // Three bad replies in a row confirm congestion; with a bitrate-targeted + // encoder the bitrate is then the only thing that drains the queue, so it + // comes down hard. Increases need every viewer below the threshold. + if let Some(factor) = reduction { + v = current_ratio * factor; + } else if max_delay < 50 { if dynamic_screen { v = current_ratio * 1.15; } @@ -479,18 +780,8 @@ impl VideoQoS { if dynamic_screen { v = current_ratio * 1.1; } - } else if max_delay < DELAY_THRESHOLD_150MS { - if dynamic_screen { - v = current_ratio * 1.05; - } - } else if max_delay < 200 { - v = current_ratio * 0.95; - } else if max_delay < 300 { - v = current_ratio * 0.9; - } else if max_delay < 500 { - v = current_ratio * 0.85; - } else { - v = current_ratio * 0.8; + } else if dynamic_screen { + v = current_ratio * 1.05; } // Limit quality increase rate for better stability @@ -503,8 +794,24 @@ impl VideoQoS { } } + if reduction.is_some() { + for user in self.users.values_mut() { + if user.delay.needs_bitrate_reduction() + && user.delay.replies_after_bitrate_reduction.is_none() + { + // One outstanding probe may have started before the bitrate change. + user.delay.replies_after_bitrate_reduction = + Some(if v.clamp(min, max) < current_ratio { + 0 + } else { + 2 + }); + } + } + } self.ratio = v.clamp(min, max); - self.adjust_ratio_instant = Instant::now(); + self.reset_send_counters(); + self.adjust_ratio_instant = self.now(); } // Adjust fps based on network delay and user response time @@ -518,17 +825,13 @@ impl VideoQoS { .min() .unwrap_or(INIT_FPS); - if self.users.iter().any(|u| u.1.delay.response_delayed) { - if fps > MIN_FPS + 1 { - fps = MIN_FPS + 1; - } - } - - // For new connections (within 1 second), cap fps to INIT_FPS to ensure stability - if self.new_user_instant.elapsed().as_secs() < 1 { - if fps > INIT_FPS { - fps = INIT_FPS; - } + // Every viewer inside its first second keeps the stream at INIT_FPS to + // ensure stability; each viewer carries its own start-up clock. + if self.users.values().any(|u| { + u.joined_at + .is_some_and(|joined| self.since(joined).as_secs() < 1) + }) { + fps = fps.min(INIT_FPS); } // Ensure fps stays within valid range @@ -538,58 +841,183 @@ impl VideoQoS { #[derive(Default, Debug, Clone)] struct RttCalculator { - min_rtt: Option, // Historical minimum RTT ever observed - window_min_rtt: Option, // Minimum RTT within last 60 samples - smoothed_rtt: Option, // Smoothed RTT estimation - samples: VecDeque, // Last 60 RTT samples + baseline: Option, + samples: VecDeque, } impl RttCalculator { - const WINDOW_SAMPLES: usize = 60; // Keep last 60 samples - const MIN_SAMPLES: usize = 10; // Require at least 10 samples - const ALPHA: f32 = 0.5; // Smoothing factor for weighted average + const WINDOW_SAMPLES: usize = 20; + const MAX_INCREASE_MS: u32 = 50; - /// Update RTT estimates with a new sample pub fn update(&mut self, delay: u32) { - // 1. Update historical minimum RTT - match self.min_rtt { - Some(min_rtt) if delay < min_rtt => self.min_rtt = Some(delay), - None => self.min_rtt = Some(delay), - _ => {} - } - - // 2. Update sample window if self.samples.len() >= Self::WINDOW_SAMPLES { self.samples.pop_front(); } self.samples.push_back(delay); + let baseline = self.baseline.unwrap_or(delay).min(delay); + self.baseline = Some(baseline); - // 3. Calculate minimum RTT within the window - self.window_min_rtt = self.samples.iter().min().copied(); - - // 4. Calculate smoothed RTT - // Use weighted average if we have enough samples - if self.samples.len() >= Self::WINDOW_SAMPLES { - if let (Some(min), Some(window_min)) = (self.min_rtt, self.window_min_rtt) { - // Weighted average of historical minimum and window minimum - let new_srtt = - ((1.0 - Self::ALPHA) * min as f32 + Self::ALPHA * window_min as f32) as u32; - self.smoothed_rtt = Some(new_srtt); - } + if self.samples.len() < Self::WINDOW_SAMPLES { + return; + } + let half = Self::WINDOW_SAMPLES / 2; + let older_min = self.samples.iter().take(half).min().copied(); + let recent_min = self.samples.iter().skip(half).min().copied(); + let (Some(older_min), Some(recent_min)) = (older_min, recent_min) else { + return; + }; + // A rising floor can be a growing queue. Allow 10 ms of probe granularity, + // but wait for it to settle before forgetting the old baseline. + if recent_min > older_min.saturating_add(10) { + return; + } + let rise = older_min.min(recent_min).saturating_sub(baseline); + if rise > 0 { + self.baseline = Some(baseline + (rise / 2).clamp(1, Self::MAX_INCREASE_MS)); } } - /// Get current RTT estimate - /// Returns None if no valid estimation is available pub fn get_rtt(&self) -> Option { - if let Some(rtt) = self.smoothed_rtt { - return Some(rtt); - } - if self.samples.len() >= Self::MIN_SAMPLES { - if let Some(rtt) = self.min_rtt { - return Some(rtt); - } - } - None + self.baseline } } + +#[cfg(test)] +mod tests { + use super::*; + + fn stable_qos() -> VideoQoS { + let mut qos = VideoQoS::default(); + qos.advance_ms(2000); + qos.users.insert(1, UserData::default()); + for _ in 0..12 { + qos.user_network_delay(1, 10); + } + assert_eq!(qos.fps(), FPS); + qos + } + + #[test] + fn isolated_delay_spike_does_not_lower_fps() { + let mut qos = stable_qos(); + for delay in [800, 10, 10, 10] { + qos.user_network_delay(1, delay); + assert_eq!(qos.fps(), FPS); + } + } + + #[test] + fn occasional_spikes_do_not_accumulate_congestion() { + let mut qos = stable_qos(); + for delay in [800, 10, 10].repeat(20) { + qos.user_network_delay(1, delay); + assert_eq!(qos.fps(), FPS); + } + } + + #[test] + fn sustained_delay_reduces_fps_gradually() { + let mut qos = stable_qos(); + for expected_fps in [30, 30, 24, 24, 24, 20] { + qos.user_network_delay(1, 800); + assert_eq!(qos.fps(), expected_fps); + } + } + + #[test] + fn delay_history_keeps_two_samples() { + let mut delay = UserDelay::default(); + for sample in [1, 2, 3] { + delay.add_delay(sample); + } + assert_eq!(delay.delay_history.len(), HISTORY_DELAY_LEN); + } + + #[test] + fn response_timeout_halves_fps_for_each_second_outstanding() { + let mut qos = stable_qos(); + for (elapsed, expected) in [(2001, 15), (3001, 7), (4001, 5), (5001, 5), (6001, 5)] { + qos.user_delay_response_elapsed(1, elapsed); + assert_eq!(qos.fps(), expected, "{elapsed} ms outstanding"); + } + } + + #[test] + fn severe_delay_does_not_wait_for_another_reply() { + let mut qos = stable_qos(); + qos.user_network_delay(1, 1200); + assert_eq!(qos.fps(), 15); + } + + #[test] + fn response_timeout_recovers_in_two_good_replies() { + let mut qos = stable_qos(); + qos.user_delay_response_elapsed(1, 3000); + assert_eq!(qos.fps(), 7); + qos.user_network_delay(1, 3200); + assert_eq!( + qos.fps(), + 7, + "the late reply belongs to the stall that was braked" + ); + qos.user_delay_response_elapsed(1, 0); + qos.user_network_delay(1, 10); + assert_eq!( + qos.fps(), + 8, + "one good reply must not restore the full frame rate" + ); + qos.user_network_delay(1, 10); + assert_eq!( + qos.fps(), + FPS, + "the second good reply restores the frame rate" + ); + qos.user_network_delay(1, 10); + assert_eq!(qos.fps(), FPS, "the third keeps the restored frame rate"); + } + + #[test] + fn restore_aims_lower_after_a_restore_that_congested() { + let mut qos = stable_qos(); + for _ in 0..2 { + qos.user_network_delay(1, 1200); + } + assert_eq!(qos.fps(), 8); + qos.user_network_delay(1, 10); + qos.user_network_delay(1, 10); + assert_eq!(qos.fps(), FPS); + // The restored level congests at once, so the next restore aims lower. + for _ in 0..3 { + qos.user_network_delay(1, 400); + } + assert!(qos.fps() < FPS); + qos.user_network_delay(1, 10); + qos.user_network_delay(1, 10); + assert!( + qos.fps() < FPS, + "no return to the level that failed: {}", + qos.fps() + ); + qos.user_network_delay(1, 10); + assert_eq!(qos.fps(), FPS, "ordinary recovery can still reach the cap"); + } + + #[test] + fn custom_fps_limit_applies_during_delay_spike() { + let mut qos = stable_qos(); + qos.user_custom_fps(1, 12); + qos.user_network_delay(1, 800); + assert_eq!(qos.fps(), 12); + } + + mod adaptation; + mod baseline; + mod invariants; + mod jitter; + mod recovery; + mod robustness; + mod sim; + mod smoke; + mod startup; +} diff --git a/src/server/video_qos/tests/adaptation.rs b/src/server/video_qos/tests/adaptation.rs new file mode 100644 index 000000000..0ecbaea3d --- /dev/null +++ b/src/server/video_qos/tests/adaptation.rs @@ -0,0 +1,353 @@ +//! Regression tests for sustained capacity drops and path-delay/content changes. +//! Capacity drops use the closed-loop model; delay and activity fixtures are open-loop. +use super::*; + +fn percentile(values: &[f64], p: f64) -> f64 { + assert!(!values.is_empty()); + let mut sorted = values.to_vec(); + sorted.sort_by(|a, b| a.partial_cmp(b).unwrap()); + sorted[((sorted.len() - 1) as f64 * p).round() as usize] +} + +// Count a fall and subsequent rise of at least `amplitude`, ignoring smaller +// reversals. A one-way reduction or recovery is not an oscillation. +fn round_trips(values: &[f64], amplitude: f64) -> usize { + let mut peak = values[0]; + let mut trough = peak; + let mut falling = false; + let mut count = 0; + for &value in &values[1..] { + if falling { + trough = trough.min(value); + if value - trough >= amplitude { + count += 1; + peak = value; + falling = false; + } + } else { + peak = peak.max(value); + if peak - value >= amplitude { + trough = value; + falling = true; + } + } + } + count +} + +#[test] +fn oscillation_metric_distinguishes_recovery_and_small_jitter() { + assert_eq!(round_trips(&[30.0, 29.0, 30.0, 28.0, 30.0], 7.5), 0); + assert_eq!(round_trips(&[30.0, 20.0, 10.0], 7.5), 0); + assert_eq!(round_trips(&[10.0, 20.0, 30.0], 7.5), 0); + assert_eq!(round_trips(&[30.0, 10.0, 30.0, 20.0, 30.0], 7.5), 2); + assert_eq!(round_trips(&[0.49, 0.17, 0.49], 0.67 * 0.25), 1); +} + +struct Oscillation { + mean_fps: f64, + mean_ratio: f64, + fps_span: f64, + ratio_span: f64, + fps_cycles_per_min: f64, + ratio_cycles_per_min: f64, + queue_p95_ms: f64, +} + +fn permanent_drop(seeds: std::ops::RangeInclusive) { + use super::sim::{self, Summary}; + + const STEADY_START_MS: u32 = 120_000; + const END_MS: u32 = 600_000; + let cases = [ + ("bandwidth_halved_30", 3000.0), + ("bandwidth_halved_fixed_rate_30", 2000.0), + ("bandwidth_halved_fixed_rate_no_abr_30", 3000.0), + ]; + println!("Permanent drop: 8 -> 2.5 Mbps at 60 s; duration 600 s; seeds {seeds:?}."); + println!("Oscillation/queue window: 120-600 s. Spans are p95-p5; round trips require 25% of the requested FPS/ratio in each direction. Delivered FPS/frame age cover 15-600 s."); + println!("| scenario | capacity wobble | steady mean FPS (median) | mean ratio (median) | FPS span (p90) | ratio span (p90) | FPS cycles/min (p90) | ratio cycles/min (p90) | queue p95 (p90) | delivered FPS (median) | frame age p95 (p90) |"); + println!("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|"); + let mut cases_run = 0; + for mut sc in sim::scenarios() { + let Some((_, queue_bound_ms)) = cases.iter().find(|(name, _)| *name == sc.name) else { + continue; + }; + cases_run += 1; + sc.seconds = END_MS / 1000; + sc.link.capacity_kbps = vec![(0, 8000.0), (60_000, 2500.0)]; + let original_wobble = sc.link.wobble; + for wobble in [0.0, original_wobble] { + sc.link.wobble = wobble; + let mut reports = Vec::new(); + let mut oscillations = Vec::new(); + for seed in seeds.clone() { + sc.seed = seed; + let report = sim::run(&sc); + assert!( + report + .trace + .iter() + .filter(|(t, ..)| (30_000..60_000).contains(t)) + .all(|(_, fps, _, _)| *fps == sc.limit), + "healthy pre-drop phase: {} seed {seed}", + sc.name + ); + let steady: Vec<_> = report + .trace + .iter() + .filter(|(t, ..)| *t >= STEADY_START_MS) + .collect(); + let fps: Vec<_> = steady.iter().map(|(_, fps, _, _)| *fps as f64).collect(); + let ratios: Vec<_> = steady + .iter() + .map(|(_, _, _, ratio)| *ratio as f64) + .collect(); + let queues: Vec<_> = steady + .iter() + .map(|(_, _, queue, _)| *queue as f64) + .collect(); + let minutes = (END_MS - STEADY_START_MS) as f64 / 60_000.0; + oscillations.push(Oscillation { + mean_fps: fps.iter().sum::() / fps.len() as f64, + mean_ratio: ratios.iter().sum::() / ratios.len() as f64, + fps_span: percentile(&fps, 0.95) - percentile(&fps, 0.05), + ratio_span: percentile(&ratios, 0.95) - percentile(&ratios, 0.05), + fps_cycles_per_min: round_trips(&fps, sc.limit as f64 * 0.25) as f64 / minutes, + ratio_cycles_per_min: round_trips(&ratios, sc.quality.ratio() as f64 * 0.25) + as f64 + / minutes, + queue_p95_ms: percentile(&queues, 0.95), + }); + reports.push(report); + } + let metric = |field: fn(&Oscillation) -> f64, p| { + percentile(&oscillations.iter().map(field).collect::>(), p) + }; + let summary = Summary::of(&reports); + let queue_p95 = metric(|o| o.queue_p95_ms, 0.9); + println!("| {} | {:.0}% | {:.1} | {:.3} | {:.1} | {:.3} | {:.2} | {:.2} | {:.0} ms | {:.1} | {} ms |", + sc.name, wobble * 100.0, metric(|o| o.mean_fps, 0.5), + metric(|o| o.mean_ratio, 0.5), + metric(|o| o.fps_span, 0.9), metric(|o| o.ratio_span, 0.9), + metric(|o| o.fps_cycles_per_min, 0.9), metric(|o| o.ratio_cycles_per_min, 0.9), + queue_p95, summary.delivered_median, summary.frame_age_p95_p90); + // Queue and frame-age limits reuse the transient-drop budgets; + // oscillation statistics are diagnostic. + assert!( + queue_p95 < *queue_bound_ms, + "{}: steady queue {queue_p95} ms", + sc.name + ); + assert!( + (summary.frame_age_p95_p90 as f64) < *queue_bound_ms, + "{summary:?}" + ); + assert!( + summary.delivered_median >= sc.limit as f64 * 0.4, + "{summary:?}" + ); + } + } + assert_eq!(cases_run, cases.len()); +} + +#[test] +fn permanent_capacity_drop_600s() { + permanent_drop(super::sim::SEEDS); +} + +#[test] +#[ignore = "extended permanent-drop coverage over 100 held-out seeds"] +fn permanent_capacity_drop_held_out_seeds() { + permanent_drop(21..=120); +} + +#[test] +fn low_capacity_preserves_auto_floor_and_recovers() { + let mut sc = super::sim::scenarios() + .into_iter() + .find(|s| s.name == "bandwidth_halved_fixed_rate_no_abr_30") + .unwrap(); + sc.link.capacity_kbps = vec![(0, 8000.0), (60_000, 700.0), (120_000, 8000.0)]; + for seed in super::sim::SEEDS { + sc.seed = seed; + let report = super::sim::run(&sc); + assert!( + report.trace.iter().all(|(_, fps, ..)| *fps >= 5), + "seed {seed}: automatic reductions went below 5 FPS" + ); + let congested: Vec<_> = report + .trace + .iter() + .filter(|(t, ..)| (80_000..120_000).contains(t)) + .collect(); + let min_fps = congested.iter().map(|(_, fps, ..)| *fps).min().unwrap(); + let min_queue = congested + .iter() + .map(|(_, _, queue, _)| *queue) + .min() + .unwrap(); + assert_eq!( + min_fps, 5, + "seed {seed}: severe congestion must reach the floor" + ); + // At 5 FPS this model sends about 670 kbps, leaving little room to drain + // existing backlog at 700 kbps. Require drainage after capacity returns. + assert!( + report.recovery_ms.is_some_and(|ms| ms <= 20_000), + "seed {seed}: recovery took {:?}", + report.recovery_ms + ); + println!("700 kbps fixed-rate, seed {seed}: min FPS={min_fps}, min queue={min_queue} ms, queue p95={} ms, frame age p95={} ms, recovery={:?}", report.queue_p95_ms, report.frame_age_p95_ms, report.recovery_ms); + } +} + +const DISPLAY: &str = "adaptation"; + +fn session(abr: bool) -> VideoQoS { + let mut qos = super::smoke::session(FPS, Quality::Balanced); + qos.abr_config = abr; + qos.new_display(DISPLAY.to_owned()); + qos.set_support_changing_quality(DISPLAY, true); + sync_bitrate(&mut qos); + qos +} + +fn sync_bitrate(qos: &mut VideoQoS) { + let bitrate = (6000.0 * qos.ratio()) as u32; + qos.store_bitrate(bitrate); +} + +fn second(qos: &mut VideoQoS, delay: u32, dynamic: bool) { + let encoded = if dynamic { qos.fps() as usize } else { 0 }; + qos.advance_ms(1000); + sync_bitrate(qos); + qos.user_network_delay(1, delay); + sync_bitrate(qos); + qos.update_display_data(DISPLAY, encoded); + sync_bitrate(qos); +} + +fn baseline_steps() -> Vec { + let mut unmet = Vec::new(); + println!("Baseline step: 90 s at 10 ms, 180 s at new RTT, 90 s at 10 ms; one fresh reply per second. Relearning requires FPS=30 and excess<150 ms throughout the final 60 s at the new RTT."); + println!("| ABR | new RTT | cold-start final FPS | learned baseline | final excess | final FPS | final ratio | relearned | returned-path FPS |"); + println!("|---|---:|---:|---:|---:|---:|---:|---|---:|"); + for abr in [false, true] { + for rtt in [310, 410] { + let mut cold = session(abr); + for _ in 0..90 { + second(&mut cold, rtt, true); + assert!(cold.fps() >= INIT_FPS, "stable cold-start RTT {rtt}"); + } + assert_eq!(cold.fps(), FPS); + let mut qos = session(abr); + for _ in 0..90 { + second(&mut qos, 10, true); + } + assert_eq!(qos.fps(), FPS); + let mut relearned = true; + for s in 0..180 { + second(&mut qos, rtt, true); + if s >= 120 { + let base = qos.users[&1].delay.rtt_calculator.get_rtt().unwrap(); + relearned &= qos.fps() == FPS && rtt.saturating_sub(base) < 150; + } + } + let base = qos.users[&1].delay.rtt_calculator.get_rtt().unwrap(); + let high_fps = qos.fps(); + let high_ratio = qos.ratio(); + for _ in 0..90 { + second(&mut qos, 10, true); + } + assert_eq!( + qos.fps(), + FPS, + "return to the original path: ABR={abr} RTT={rtt}" + ); + assert!(qos.ratio() >= BR_BALANCED * 0.95); + println!("| {abr} | {rtt} ms | {} | {base} ms | {} ms | {high_fps} | {high_ratio:.3} | {relearned} | {} |", + cold.fps(), rtt.saturating_sub(base), qos.fps()); + if !relearned { + unmet.push(format!( + "ABR={abr}, RTT 10 -> {rtt} ms: base={base}, FPS={high_fps}" + )); + } + } + } + unmet +} + +#[test] +fn baseline_step_relearns_higher_rtt() { + let unmet = baseline_steps(); + assert!( + unmet.is_empty(), + "higher baseline was not relearned: {unmet:?}" + ); +} + +#[test] +fn static_to_dynamic_ratio_recovery() { + println!("Static recovery: 90 s healthy video, 12 s confirmed 800 ms delay, 60 s healthy static screen, then 90 s video. Bitrate is modeled as ratio * 6000 kbps."); + println!("| restart profile | ratio after cut | ratio after static | time to 90% | time to 95% | final ratio | final FPS | modeled bitrate |"); + println!("|---|---:|---:|---|---|---:|---:|---:|"); + for (profile, restart_delay, growing_queue) in [ + ("healthy 10 ms", 10, false), + ("stable path 800 ms", 800, false), + ("growing queue 800 + 10 ms/s", 800, true), + ] { + let mut qos = session(true); + for _ in 0..90 { + second(&mut qos, 10, true); + } + let target = qos.latest_quality().ratio(); + assert_eq!(qos.ratio(), target); + for _ in 0..12 { + second(&mut qos, 800, true); + } + let after_cut = qos.ratio(); + assert!( + after_cut < target * 0.5, + "fixture must confirm congestion and cut bitrate" + ); + for _ in 0..60 { + second(&mut qos, 10, false); + } + let after_static = qos.ratio(); + let mut t90 = (after_static >= target * 0.90).then_some(0); + let mut t95 = (after_static >= target * 0.95).then_some(0); + for s in 1..=90 { + let delay = restart_delay + if growing_queue { s * 10 } else { 0 }; + second(&mut qos, delay, true); + let ratio = qos.ratio(); + if ratio >= target * 0.90 { + t90.get_or_insert(s); + } + if ratio >= target * 0.95 { + t95.get_or_insert(s); + } + if growing_queue { + assert!( + ratio <= after_static * 1.02, + "activity must not restore quality into congestion" + ); + } + } + let seconds = |time: Option| { + time.map(|s| format!("{s} s")) + .unwrap_or_else(|| "never".to_owned()) + }; + let ratio = qos.ratio(); + println!("| {profile} | {after_cut:.3} | {after_static:.3} | {} | {} | {ratio:.3} | {} | {} kbps |", + seconds(t90), seconds(t95), qos.fps(), qos.bitrate()); + if !growing_queue { + assert!( + t95.is_some(), + "{profile}: video did not regain 95% quality within 90 s" + ); + assert_eq!(qos.fps(), FPS); + } + } +} diff --git a/src/server/video_qos/tests/baseline.rs b/src/server/video_qos/tests/baseline.rs new file mode 100644 index 000000000..26e8837a1 --- /dev/null +++ b/src/server/video_qos/tests/baseline.rs @@ -0,0 +1,110 @@ +use super::*; + +fn learned_baseline(delay: u32) -> RttCalculator { + let mut rtt = RttCalculator::default(); + for _ in 0..90 { + rtt.update(delay); + } + rtt +} + +#[test] +fn old_minimum_expires_after_a_stable_path_change() { + for (old, new) in [(10, 310), (10, 500), (159, 500)] { + let mut rtt = learned_baseline(old); + for i in 0..40 { + let before = rtt.get_rtt().unwrap(); + rtt.update(new + i % 5 * 10); + let after = rtt.get_rtt().unwrap(); + assert!(after <= before + 50, "limit the cost of relearning"); + if i < 10 { + assert_eq!(after, old, "a short burst must not replace the baseline"); + } + } + assert_eq!(rtt.get_rtt(), Some(new), "{old} -> {new}"); + rtt.update(old); + assert_eq!(rtt.get_rtt(), Some(old), "a lower delay is direct evidence"); + } +} + +#[test] +fn rising_delay_is_not_learned_as_a_new_baseline() { + for step in [2, 10, 50] { + let mut rtt = learned_baseline(10); + for i in 0..120 { + rtt.update(200 + i * step); + assert_eq!(rtt.get_rtt(), Some(10), "rising by {step} ms per reply"); + } + } +} + +#[test] +fn intermittent_spikes_do_not_raise_the_baseline() { + let mut rtt = learned_baseline(159); + for delay in [159, 900, 500, 159, 350].repeat(30) { + rtt.update(delay); + assert_eq!(rtt.get_rtt(), Some(159)); + } +} + +#[test] +fn pending_probes_and_late_replies_do_not_age_the_baseline() { + let mut qos = stable_qos(); + for elapsed in (2001..122_001).step_by(1000) { + qos.user_delay_response_elapsed(1, elapsed); + assert_eq!(qos.users[&1].delay.rtt_calculator.get_rtt(), Some(10)); + } + qos.user_network_delay(1, 122_000); + assert_eq!(qos.users[&1].delay.rtt_calculator.get_rtt(), Some(10)); + for _ in 0..30 { + qos.user_delay_response_elapsed(1, 2500); + qos.user_network_delay(1, 2600); + } + assert_eq!(qos.users[&1].delay.rtt_calculator.get_rtt(), Some(10)); + assert_eq!(qos.fps(), 5); +} + +#[test] +fn stable_path_change_recovers_without_reconnecting() { + println!("| ABR | path delay | seconds to 30 FPS | final baseline | final FPS |"); + println!("|---|---:|---:|---:|---:|"); + for abr in [false, true] { + for delay in [310, 410, 500] { + let mut qos = super::smoke::session(FPS, Quality::Balanced); + qos.abr_config = abr; + qos.new_display("baseline".to_owned()); + qos.set_support_changing_quality("baseline", true); + let second = |qos: &mut VideoQoS, delay| { + qos.advance_ms(1000); + let bitrate = (6000.0 * qos.ratio()) as u32; + qos.store_bitrate(bitrate); + qos.user_network_delay(1, delay); + let bitrate = (6000.0 * qos.ratio()) as u32; + qos.store_bitrate(bitrate); + qos.update_display_data("baseline", qos.fps() as usize); + }; + for _ in 0..90 { + second(&mut qos, 10); + } + let mut first_recovered = None; + for s in 1..=90 { + second(&mut qos, delay); + if s >= 10 && qos.fps() == FPS && first_recovered.is_none() { + first_recovered = Some(s); + } + if s >= 40 { + assert_eq!(qos.fps(), FPS, "stay recovered: ABR={abr}, delay={delay}"); + } + } + let base = qos.users[&1].delay.rtt_calculator.get_rtt().unwrap(); + println!( + "| {abr} | 10 -> {delay} ms | {first_recovered:?} | {base} | {} |", + qos.fps() + ); + assert!(first_recovered.is_some_and(|s| s <= 30)); + assert_eq!(base, delay); + second(&mut qos, 10); + assert_eq!(qos.users[&1].delay.rtt_calculator.get_rtt(), Some(10)); + } + } +} diff --git a/src/server/video_qos/tests/invariants.rs b/src/server/video_qos/tests/invariants.rs new file mode 100644 index 000000000..d5da5acb2 --- /dev/null +++ b/src/server/video_qos/tests/invariants.rs @@ -0,0 +1,498 @@ +//! The controller's invariants as properties over random sessions. A scenario +//! test pins one trajectory; these hold whatever the trajectory: +//! +//! 1. viewer isolation: a viewer's private target is a function of its own +//! replies, timeouts and limit, never of another viewer's (with ABR on, the +//! shared bitrate state is the one designed input: the frame rate keeps a +//! floor while the bitrate can still come down); +//! 2. bad evidence never raises anything: a bad reply or a timeout tick keeps or +//! lowers that viewer's target and the bitrate ratio; +//! 3. lifecycle: a join adds a constraint and a leave removes it, and neither +//! touches any other viewer's state; +//! 4. evidence ownership: a bitrate cut is asked for by a viewer's own evidence, +//! by the step that viewer's own evidence calls for, and a newcomer's first +//! reply does not spend that evidence again; +//! 5. caps: a reply leaves the target within `[MIN_FPS, cap]`, and the stream is +//! the aggregation of the targets, the caps and the start-up guards; +//! 6. pairing: the late reply of a braked probe does not brake again. +use super::*; + +/// xorshift64*, so the tests need no external crate and stay reproducible. +struct Rng(u64); + +impl Rng { + fn new(seed: u64) -> Self { + Rng((seed ^ 0x9E37_79B9_7F4A_7C15).max(1)) + } + + fn next(&mut self) -> u64 { + let mut x = self.0; + x ^= x >> 12; + x ^= x << 25; + x ^= x >> 27; + self.0 = x; + x.wrapping_mul(0x2545_F491_4F6C_DD1D) + } + + fn below(&mut self, n: u64) -> u64 { + self.next() % n + } + + fn chance(&mut self, pct: u64) -> bool { + self.below(100) < pct + } +} + +#[derive(Debug, Clone, Copy)] +enum Step { + Reply { id: i32, delay: u32 }, + Timeout { id: i32, elapsed: u128 }, + Wait(u64), + Tick(usize), + Cap { id: i32, fps: u32 }, +} + +const SEEDS: u64 = 150; +const STEPS: usize = 300; + +/// Random events for a set of viewers on one link. A probe that is out stays +/// out until a reply: the connection reports a growing elapsed time every second +/// and the reply that ends the stall carries at least that delay. +struct Driver { + rng: Rng, + ids: Vec, + base_rtt: u32, + outstanding: HashMap, +} + +impl Driver { + fn new(seed: u64, ids: Vec) -> Self { + let mut rng = Rng::new(seed); + let base_rtt = 10 + rng.below(300) as u32; + Driver { + rng, + ids, + base_rtt, + outstanding: HashMap::new(), + } + } + + fn step(&mut self) -> Step { + let id = self.ids[self.rng.below(self.ids.len() as u64) as usize]; + let roll = self.rng.below(100); + match roll { + 0..=64 => { + let mut delay = if roll < 45 { + self.base_rtt + self.rng.below(140) as u32 + } else { + self.base_rtt + DELAY_THRESHOLD_150MS + self.rng.below(1500) as u32 + }; + if let Some(elapsed) = self.outstanding.remove(&id) { + delay = delay.max(elapsed as u32 + self.rng.below(500) as u32); + } + Step::Reply { id, delay } + } + 65..=74 => { + let elapsed = match self.outstanding.get(&id) { + Some(elapsed) => elapsed + 1000, + None => 2001 + self.rng.below(1000) as u128, + }; + self.outstanding.insert(id, elapsed); + Step::Timeout { id, elapsed } + } + 75..=89 => Step::Wait(self.rng.below(1500)), + 90..=96 => Step::Tick(self.rng.below(31) as usize), + _ => Step::Cap { + id, + fps: 1 + self.rng.below(60) as u32, + }, + } + } +} + +/// What `on_connection_open` inserts, without touching the config store. +fn open(qos: &mut VideoQoS, id: i32) { + qos.users.insert( + id, + UserData { + joined_at: Some(qos.now()), + ..Default::default() + }, + ); +} + +fn session(abr: bool) -> VideoQoS { + let mut qos = VideoQoS::default(); + qos.advance_ms(2000); + qos.abr_config = abr; + qos.first_reply_adjusts_ratio = true; + qos.new_display("test".to_owned()); + qos.set_support_changing_quality("test", true); + qos.store_bitrate(4000); + qos +} + +/// The video loop reports the encoder's bitrate as soon as it applies a ratio. +fn sync_bitrate(qos: &mut VideoQoS) { + let target = qos.latest_quality().ratio(); + let ratio = qos.ratio(); + qos.store_bitrate((4000.0 * ratio / target) as u32); +} + +fn apply(qos: &mut VideoQoS, step: Step) { + match step { + Step::Reply { id, delay } => qos.user_network_delay(id, delay), + Step::Timeout { id, elapsed } => qos.user_delay_response_elapsed(id, elapsed), + Step::Wait(ms) => qos.advance_ms(ms), + Step::Tick(encoded) => qos.update_display_data("test", encoded), + Step::Cap { id, fps } => qos.user_custom_fps(id, fps), + } + sync_bitrate(qos); +} + +/// The viewer's private target, as the controller reads it before a reply. +fn target(qos: &VideoQoS, id: i32) -> u32 { + let user = &qos.users[&id]; + user.delay.fps.unwrap_or(INIT_FPS.min(user.fps_cap())) +} + +fn baseline(qos: &VideoQoS, id: i32) -> Option { + qos.users[&id].delay.rtt_calculator.get_rtt() +} + +/// Everything the controller keeps about a viewer, for change detection. +fn snapshot(qos: &VideoQoS, id: i32) -> String { + format!("{:?}", qos.users[&id]) +} + +/// The aggregation `adjust_fps` is meant to compute: the slowest viewer's target, +/// INIT_FPS for a viewer without a reply or inside its first second, within the +/// lowest cap. +fn expected_stream(qos: &VideoQoS) -> u32 { + let mut fps = qos + .users + .values() + .map(|u| u.delay.fps.unwrap_or(INIT_FPS)) + .min() + .unwrap_or(INIT_FPS); + if qos + .users + .values() + .any(|u| u.joined_at.is_some_and(|j| qos.since(j).as_secs() < 1)) + { + fps = fps.min(INIT_FPS); + } + let cap = qos + .users + .values() + .map(|u| u.fps_cap()) + .min() + .unwrap_or(FPS); + fps.clamp(MIN_FPS, cap) +} + +// Invariant 2: bad evidence never raises a target or the ratio. +#[test] +fn bad_evidence_never_raises_a_target_or_the_ratio() { + for seed in 0..SEEDS { + for abr in [false, true] { + let mut qos = session(abr); + let ids: Vec = (1..=1 + (seed % 3) as i32).collect(); + for id in &ids { + open(&mut qos, *id); + } + let mut driver = Driver::new(seed, ids); + for step_no in 0..STEPS { + let step = driver.step(); + let ratio_before = qos.ratio(); + let before = match step { + Step::Reply { id, .. } | Step::Timeout { id, .. } => Some((id, target(&qos, id))), + _ => None, + }; + apply(&mut qos, step); + // Bad by the baseline the controller used for this reply: the reply + // itself may have relearned it. + let bad = match step { + Step::Reply { id, delay } => baseline(&qos, id) + .is_some_and(|base| delay >= base + DELAY_THRESHOLD_150MS), + Step::Timeout { .. } => true, + _ => false, + }; + if let (true, Some((id, before))) = (bad, before) { + let after = target(&qos, id); + assert!( + after <= before, + "seed {seed} abr {abr} step {step_no} {step:?}: target {before} -> {after}" + ); + assert!( + qos.ratio() <= ratio_before, + "seed {seed} abr {abr} step {step_no} {step:?}: ratio {ratio_before} -> {}", + qos.ratio() + ); + } + } + } + } +} + +// Invariant 2 and 6: a timeout tick keeps or lowers the target, whatever it is. +#[test] +fn a_timeout_keeps_or_lowers_every_target() { + for reference in MIN_FPS..=MAX_FPS { + for elapsed in [2001, 2999, 3000, 3001, 4500, 6001, 9000, 30_000] { + let mut qos = session(false); + open(&mut qos, 1); + qos.user_custom_fps(1, MAX_FPS); + qos.users.get_mut(&1).unwrap().delay.fps = Some(reference); + qos.adjust_fps(); + let stream = qos.fps(); + qos.user_delay_response_elapsed(1, elapsed); + assert!( + target(&qos, 1) <= reference, + "{elapsed} ms outstanding at {reference} fps: target {}", + target(&qos, 1) + ); + assert!( + qos.fps() <= stream, + "{elapsed} ms outstanding at {reference} fps: stream {stream} -> {}", + qos.fps() + ); + } + } +} + +// Invariant 6: the late reply of a braked probe does not brake again. +#[test] +fn a_late_reply_after_a_brake_does_not_brake_again() { + for reference in (MIN_FPS + 1..=MAX_FPS).step_by(3) { + for elapsed in [2001u32, 3001, 4500, 6001, 9000] { + for abr in [false, true] { + let mut qos = session(abr); + open(&mut qos, 1); + qos.user_custom_fps(1, MAX_FPS); + for _ in 0..3 { + qos.user_network_delay(1, 20); + } + qos.users.get_mut(&1).unwrap().delay.fps = Some(reference); + qos.user_delay_response_elapsed(1, elapsed as u128); + let braked = target(&qos, 1); + qos.user_network_delay(1, elapsed + 100); + assert_eq!( + target(&qos, 1), + braked, + "abr {abr}, {elapsed} ms outstanding at {reference} fps" + ); + } + } + } +} + +/// Viewer 1's target after each of its own events, alone or with company whose +/// events are interleaved: a second viewer with its own replies, timeouts, waits +/// and limits, and a third that joins and leaves along the way. The display +/// timer is left out: on a dynamic screen it raises the ratio off its floor, +/// and the property holds the bitrate state fixed. +fn viewer_one_targets(seed: u64, company: bool, abr: bool, at_floor: bool) -> Vec { + let mut own = Driver::new(seed, vec![1]); + let mut others = Driver::new(seed ^ 0xC0FF_EE, vec![2]); + let mut qos = session(abr); + if at_floor { + qos.ratio = qos.min_ratio(); + sync_bitrate(&mut qos); + assert!(!qos.can_reduce_bitrate()); + } + open(&mut qos, 1); + if company { + open(&mut qos, 2); + } + let no_tick = |step: Step| match step { + Step::Tick(_) => Step::Wait(1000), + step => step, + }; + let mut targets = Vec::new(); + for step_no in 0..STEPS { + if company { + for _ in 0..others.rng.below(3) { + let step = no_tick(others.step()); + apply(&mut qos, step); + } + if step_no == STEPS / 3 { + open(&mut qos, 3); + others.ids.push(3); + } + if step_no == 2 * STEPS / 3 { + qos.on_connection_close(3); + others.ids.pop(); + } + } + let step = no_tick(own.step()); + apply(&mut qos, step); + targets.push(target(&qos, 1)); + } + targets +} + +// Invariant 1: another viewer's replies, timeouts, limits, joins and leaves do +// not change a viewer's private target. With ABR off the bitrate is fixed; with +// ABR on the shared bitrate state is a designed input, so the property is checked +// at the bitrate floor, where it can no longer change. +#[test] +fn a_viewers_target_is_independent_of_other_viewers() { + for seed in 0..SEEDS { + for (abr, at_floor) in [(false, false), (true, true)] { + let alone = viewer_one_targets(seed, false, abr, at_floor); + let with_company = viewer_one_targets(seed, true, abr, at_floor); + assert_eq!( + alone, with_company, + "seed {seed} abr {abr}: viewer 1's targets differ with company" + ); + } + } +} + +// Invariant 3: a join adds a constraint, a leave removes it, and neither touches +// another viewer's state. +#[test] +fn joins_and_leaves_only_change_the_aggregation() { + for seed in 0..SEEDS { + for abr in [false, true] { + let mut qos = session(abr); + open(&mut qos, 1); + open(&mut qos, 2); + let mut driver = Driver::new(seed, vec![1, 2]); + let mut next_id = 3; + let mut present: Vec = Vec::new(); + for step_no in 0..STEPS { + let step = driver.step(); + apply(&mut qos, step); + if driver.rng.chance(10) { + let others: Vec = qos.users.keys().copied().collect(); + let before: Vec = others.iter().map(|id| snapshot(&qos, *id)).collect(); + qos.adjust_fps(); + let stream = qos.fps(); + open(&mut qos, next_id); + present.push(next_id); + driver.ids.push(next_id); + next_id += 1; + qos.adjust_fps(); + assert!( + qos.fps() <= stream, + "seed {seed} abr {abr} step {step_no}: a join raised the stream {stream} -> {}", + qos.fps() + ); + assert_eq!(qos.fps(), expected_stream(&qos)); + let after: Vec = others.iter().map(|id| snapshot(&qos, *id)).collect(); + assert_eq!( + before, after, + "seed {seed} abr {abr} step {step_no}: a join changed a viewer" + ); + } else if !present.is_empty() && driver.rng.chance(10) { + let leaving = present.remove(driver.rng.below(present.len() as u64) as usize); + driver.ids.retain(|id| *id != leaving); + driver.outstanding.remove(&leaving); + let others: Vec = qos.users.keys().copied().filter(|id| *id != leaving).collect(); + let before: Vec = others.iter().map(|id| snapshot(&qos, *id)).collect(); + qos.on_connection_close(leaving); + let after: Vec = others.iter().map(|id| snapshot(&qos, *id)).collect(); + assert_eq!( + before, after, + "seed {seed} abr {abr} step {step_no}: a leave changed a viewer" + ); + assert_eq!( + qos.fps(), + expected_stream(&qos), + "seed {seed} abr {abr} step {step_no}: the stream after a leave" + ); + } + } + } + } +} + +// Invariant 4: the ratio comes down only when a viewer's own evidence asks for +// it, by that viewer's own step, and a newcomer's first reply does not spend the +// evidence again. +#[test] +fn a_bitrate_cut_is_owned_by_a_viewers_evidence() { + let mut cuts = 0; + for seed in 0..SEEDS { + let mut qos = session(true); + let ids: Vec = (1..=1 + (seed % 3) as i32).collect(); + for id in &ids { + open(&mut qos, *id); + } + let mut driver = Driver::new(seed, ids); + for step_no in 0..STEPS { + let step = driver.step(); + let before = qos.ratio(); + apply(&mut qos, step); + let after = qos.ratio(); + if after >= before { + continue; + } + cuts += 1; + let asked: Vec = qos + .users + .values() + .filter_map(|u| u.delay.ratio_reduction()) + .collect(); + assert!( + !asked.is_empty(), + "seed {seed} step {step_no} {step:?}: a cut nobody asked for" + ); + let deepest = asked.iter().copied().fold(1.0_f32, f32::min); + assert!( + after >= before * deepest * 0.999, + "seed {seed} step {step_no} {step:?}: cut {before} -> {after}, deepest step asked {deepest}" + ); + // A newcomer replying inside the cooldown finds the evidence spent. + open(&mut qos, 99); + qos.advance_ms(driver.rng.below(2900)); + qos.user_network_delay(99, driver.base_rtt); + assert_eq!( + qos.ratio(), + after, + "seed {seed} step {step_no}: a newcomer's first reply spent the evidence again" + ); + qos.on_connection_close(99); + } + } + assert!(cuts > SEEDS as usize, "only {cuts} cuts across {SEEDS} sessions"); +} + +// Invariant 5: a reply leaves the target within its cap, and the stream is the +// aggregation of targets, caps and start-up guards after every decision. +#[test] +fn targets_stay_within_caps_and_the_stream_is_their_aggregation() { + for seed in 0..SEEDS { + for abr in [false, true] { + let mut qos = session(abr); + let ids: Vec = (1..=1 + (seed % 3) as i32).collect(); + for id in &ids { + open(&mut qos, *id); + } + let mut driver = Driver::new(seed, ids); + for step_no in 0..STEPS { + let step = driver.step(); + apply(&mut qos, step); + match step { + Step::Reply { id, .. } => { + let cap = qos.users[&id].fps_cap(); + let t = target(&qos, id); + assert!( + (MIN_FPS..=cap).contains(&t), + "seed {seed} abr {abr} step {step_no} {step:?}: target {t} outside [{MIN_FPS}, {cap}]" + ); + } + Step::Wait(_) | Step::Cap { .. } => continue, + _ => {} + } + assert_eq!( + qos.fps(), + expected_stream(&qos), + "seed {seed} abr {abr} step {step_no} {step:?}: the stream is not the aggregation" + ); + } + } + } +} diff --git a/src/server/video_qos/tests/jitter.rs b/src/server/video_qos/tests/jitter.rs new file mode 100644 index 000000000..320d75c54 --- /dev/null +++ b/src/server/video_qos/tests/jitter.rs @@ -0,0 +1,493 @@ +use super::*; + +fn abr_session() -> VideoQoS { + let mut qos = stable_qos(); + qos.new_display("test".to_owned()); + qos.set_support_changing_quality("test", true); + qos.store_bitrate(4000); + // Linux skips the first-reply adjustment; exercise it on every platform. + qos.first_reply_adjusts_ratio = true; + qos.advance_ms(4000); + qos +} + +#[test] +fn bitrate_reduction_precedes_ordinary_fps_reduction() { + let mut qos = abr_session(); + let ratio = qos.ratio(); + qos.user_network_delay(1, 400); + assert_eq!(qos.fps(), FPS); + assert_eq!(qos.ratio(), ratio); + qos.user_network_delay(1, 400); + assert_eq!(qos.fps(), FPS); + assert!(qos.ratio() < ratio); + qos.user_network_delay(1, 400); + assert_eq!(qos.fps(), FPS); + qos.user_network_delay(1, 400); + assert!(qos.fps() < FPS); +} + +#[test] +fn bitrate_cooldown_defers_ordinary_fps_reduction() { + let mut qos = abr_session(); + qos.adjust_ratio_instant = qos.now(); + let ratio = qos.ratio(); + for _ in 0..3 { + qos.user_network_delay(1, 400); + assert_eq!(qos.fps(), FPS); + assert_eq!(qos.ratio(), ratio); + } + qos.advance_ms(4000); + qos.user_network_delay(1, 400); + assert_eq!(qos.fps(), FPS); + assert!(qos.ratio() < ratio); + qos.user_network_delay(1, 400); + assert_eq!(qos.fps(), FPS); + qos.user_network_delay(1, 400); + assert!(qos.fps() < FPS); +} + +#[test] +fn unavailable_abr_or_minimum_bitrate_does_not_prevent_fps_reduction() { + for mode in ["disabled", "unsupported", "minimum"] { + let mut qos = abr_session(); + match mode { + "disabled" => qos.abr_config = false, + "unsupported" => qos.set_support_changing_quality("test", false), + "minimum" => qos.ratio = BR_MIN_HIGH_RESOLUTION, + _ => unreachable!(), + } + for _ in 0..3 { + qos.user_network_delay(1, 400); + } + assert!(qos.fps() < FPS, "{mode}"); + } +} + +#[test] +fn severe_delay_and_timeout_bypass_bitrate_cooldown() { + let mut qos = abr_session(); + qos.adjust_ratio_instant = qos.now(); + qos.user_network_delay(1, 1200); + assert_eq!(qos.fps(), 15); + qos.user_delay_response_elapsed(1, 2500); + assert_eq!(qos.fps(), 7); +} + +#[test] +fn minimum_bitrate_during_cooldown_does_not_block_fps_reduction() { + // ABR on, ratio at its floor, a good reply cleared the post-reduction counter, + // and the adjustment cooldown has just restarted: bitrate cannot help here. + let mut qos = abr_session(); + qos.ratio = BR_MIN_HIGH_RESOLUTION; + qos.user_network_delay(1, 10); + qos.adjust_ratio_instant = qos.now(); + for _ in 0..3 { + qos.user_network_delay(1, 400); + } + assert!(qos.fps() < FPS); +} + +fn abr_session_from_scratch() -> VideoQoS { + let mut qos = VideoQoS::default(); + qos.advance_ms(2000); + qos.users.insert(1, UserData::default()); + qos.new_display("test".to_owned()); + qos.set_support_changing_quality("test", true); + qos.store_bitrate(4000); + qos +} + +/// The video loop reports the encoder's bitrate as soon as it applies a new ratio. +fn sync_bitrate(qos: &mut VideoQoS) { + let target = qos.latest_quality().ratio(); + let ratio = qos.ratio(); + qos.store_bitrate((4000.0 * ratio / target) as u32); +} + +/// One second of wall clock, one probe reply, one display update: what a +/// connection does every second. +fn second(qos: &mut VideoQoS, delay: u32, encoded: usize) { + qos.advance_ms(1000); + sync_bitrate(qos); + qos.user_network_delay(1, delay); + sync_bitrate(qos); + qos.update_display_data("test", encoded); + sync_bitrate(qos); +} + +#[test] +fn stable_high_rtt_restores_bitrate() { + for rtt in [180, 300] { + let mut qos = abr_session_from_scratch(); + let target = qos.latest_quality().ratio(); + for _ in 0..120 { + second(&mut qos, rtt, 30); + } + assert_eq!(qos.fps(), FPS, "rtt {rtt}"); + assert!( + qos.ratio() >= target * 0.99, + "rtt {rtt}: ratio {}", + qos.ratio() + ); + } +} + +#[test] +fn congestion_bitrate_reduction_resets_dynamic_screen_window() { + // A static screen encodes about one frame per second. While the congestion path + // adjusts the ratio at every cooldown, the periodic branch never runs, so the + // encode counter must not keep accumulating across the whole episode. + let mut qos = abr_session(); + for _ in 0..10 { + qos.advance_ms(4000); + qos.user_network_delay(1, 10); + qos.user_network_delay(1, 400); + qos.user_network_delay(1, 400); + qos.update_display_data("test", 1); + } + for _ in 0..12 { + qos.advance_ms(1000); + qos.user_network_delay(1, 10); + } + let ratio = qos.ratio(); + qos.advance_ms(4000); + qos.update_display_data("test", 1); + assert!( + qos.ratio() <= ratio, + "a static screen must not look dynamic after congestion" + ); +} + +#[test] +fn a_single_stall_does_not_cut_bitrate() { + // One probe out for 2.5 s, then its late reply: jitter, not congestion. + let mut qos = abr_session(); + let ratio = qos.ratio(); + qos.user_delay_response_elapsed(1, 2500); + qos.advance_ms(1000); + qos.update_display_data("test", 30); + assert_eq!(qos.ratio(), ratio, "the timeout tick alone"); + qos.user_network_delay(1, 2600); + assert_eq!(qos.ratio(), ratio, "the late reply alone"); + qos.user_network_delay(1, 400); + assert!(qos.ratio() < ratio, "a second bad reply confirms"); +} + +#[test] +fn a_stall_beyond_three_seconds_cuts_bitrate() { + let mut qos = abr_session(); + let ratio = qos.ratio(); + qos.user_delay_response_elapsed(1, 2001); + qos.update_display_data("test", 30); + assert_eq!(qos.ratio(), ratio); + qos.advance_ms(1000); + qos.user_delay_response_elapsed(1, 3001); + qos.update_display_data("test", 30); + assert!(qos.ratio() < ratio, "still out at the next tick"); +} + +#[test] +fn stable_high_rtt_does_not_dip_at_start() { + for rtt in [180, 300] { + let mut qos = super::smoke::session(30, Quality::Balanced); + for _ in 0..20 { + qos.advance_ms(1000); + qos.user_network_delay(1, rtt); + assert!(qos.fps() >= INIT_FPS, "rtt {rtt}: {}", qos.fps()); + } + assert_eq!(qos.fps(), FPS, "rtt {rtt}"); + } +} + +#[test] +fn confirmed_severe_congestion_halves_bitrate() { + let mut qos = abr_session(); + let ratio = qos.ratio(); + qos.user_network_delay(1, 800); + qos.user_network_delay(1, 800); // two bad replies: an ordinary step + let after_first = qos.ratio(); + assert!( + after_first < ratio && after_first > ratio * 0.75, + "{after_first}" + ); + qos.user_network_delay(1, 800); // three: confirmed + qos.advance_ms(4000); + qos.update_display_data("test", 30); + assert!(qos.ratio() <= after_first * 0.55, "{}", qos.ratio()); +} + +#[test] +fn fps_holds_its_floor_while_bitrate_can_still_drop() { + // With a bitrate-targeted encoder fewer frames do not mean fewer bytes, so the + // bitrate comes down first and the frame rate keeps its floor meanwhile. + let mut qos = abr_session(); + let mut reached_floor = false; + // Keep the queue growing, rather than presenting a stable new path delay. + let mut delay = 400; + for _ in 0..60 { + qos.advance_ms(3000); + second(&mut qos, delay, 30); + delay += 10; + if qos.ratio() > 0.17 { + assert!( + qos.fps() >= 10, + "fps {} at ratio {}", + qos.fps(), + qos.ratio() + ); + } else { + reached_floor = true; + break; + } + } + assert!( + reached_floor, + "bitrate must reach its floor: {}", + qos.ratio() + ); + for _ in 0..24 { + qos.user_network_delay(1, delay); + delay += 10; + } + assert!( + qos.fps() < 10, + "an exhausted bitrate frees the frame rate: {}", + qos.fps() + ); +} + +#[test] +fn bitrate_timer_does_not_punish_unconfirmed_spikes_or_stale_averages() { + let mut qos = abr_session(); + let ratio = qos.ratio(); + for delay in [800, 10, 350, 10, 350, 10].repeat(10) { + qos.user_network_delay(1, delay); + qos.adjust_ratio(false); + assert_eq!(qos.fps(), FPS); + assert_eq!(qos.ratio(), ratio); + } +} + +#[test] +fn viewers_confirm_congestion_independently() { + let mut qos = stable_qos(); + qos.users.insert(2, UserData::default()); + for _ in 0..30 { + qos.user_network_delay(2, 10); + qos.user_network_delay(1, 10); + } + for id in [1, 2, 1, 2] { + qos.user_network_delay(id, 400); + assert_eq!(qos.fps(), FPS); + } + qos.user_network_delay(2, 10); + qos.user_network_delay(1, 400); + assert!(qos.fps() < FPS); + qos.user_custom_fps(2, 12); + qos.user_network_delay(1, 10); + assert_eq!(qos.fps(), 12); +} + +#[test] +fn a_congested_viewer_does_not_lower_another_viewers_target() { + let mut qos = stable_qos(); + qos.users.insert(2, UserData::default()); + for _ in 0..30 { + qos.user_network_delay(2, 10); + qos.user_network_delay(1, 10); + } + assert_eq!(qos.fps(), FPS); + // Viewer 1 congests; the stream follows the slowest viewer. + for _ in 0..2 { + qos.user_network_delay(1, 1200); + } + assert_eq!(qos.fps(), 8); + // Viewer 2 is fine and keeps its own target rather than inheriting viewer 1's. + qos.user_network_delay(2, 10); + assert_eq!(qos.users[&2].delay.fps, Some(FPS)); + // Once viewer 1 restores, the stream is back at once. + for _ in 0..3 { + qos.user_network_delay(1, 10); + } + assert_eq!(qos.fps(), FPS); +} + +#[test] +fn pending_probe_checks_do_not_count_as_fresh_bad_replies() { + let mut qos = stable_qos(); + qos.user_network_delay(1, 400); + for elapsed in [1000, 1500, 1900] { + qos.user_delay_response_elapsed(1, elapsed); + assert_eq!(qos.fps(), FPS); + } + qos.user_network_delay(1, 400); + assert_eq!(qos.fps(), FPS); + qos.user_network_delay(1, 10); + qos.user_network_delay(1, 400); + qos.user_network_delay(1, 400); + assert_eq!(qos.fps(), FPS); +} + +#[test] +fn recovery_continues_with_intermittent_jitter() { + let mut qos = stable_qos(); + for _ in 0..3 { + qos.user_network_delay(1, 1200); + } + assert_eq!(qos.fps(), 5); + for delay in [10, 350].repeat(30) { + qos.user_network_delay(1, delay); + } + assert_eq!(qos.fps(), FPS); +} + +#[test] +fn custom_limit_of_one_viewer_does_not_lower_another_viewers_target() { + let mut qos = stable_qos(); + qos.users.insert(2, UserData::default()); + for _ in 0..30 { + qos.user_network_delay(2, 10); + qos.user_network_delay(1, 10); + } + qos.user_custom_fps(2, 12); + qos.user_network_delay(1, 10); + assert_eq!(qos.fps(), 12, "the stream follows the lowest limit"); + assert_eq!( + qos.users[&1].delay.fps, + Some(FPS), + "viewer 1's own target is not a function of viewer 2's limit" + ); + qos.on_connection_close(2); + assert_eq!( + qos.fps(), + FPS, + "the stream is back the moment the limit is gone" + ); +} + +#[test] +fn new_viewers_first_reply_does_not_bypass_bitrate_cooldown() { + let mut qos = abr_session(); + for _ in 0..3 { + qos.user_network_delay(1, 800); + } + // Viewer 1 is confirmed and its evidence was spent on a cut a moment ago. + let ratio = qos.ratio(); + assert!(ratio < Quality::Balanced.ratio()); + qos.users.insert(2, UserData::default()); + qos.user_network_delay(2, 10); + assert_eq!( + qos.ratio(), + ratio, + "viewer 2's first reply must not spend viewer 1's evidence again inside the cooldown" + ); +} + +#[test] +fn new_viewer_does_not_inherit_another_viewers_congested_fps() { + let mut qos = stable_qos(); + for _ in 0..2 { + qos.user_network_delay(1, 1200); + } + assert_eq!(qos.fps(), 8); + qos.users.insert(2, UserData::default()); + qos.user_network_delay(2, 10); + assert!( + qos.users[&2].delay.fps >= Some(INIT_FPS), + "a new viewer starts from INIT_FPS, not from the congested stream: {:?}", + qos.users[&2].delay.fps + ); +} + +#[test] +fn unconfirmed_severe_viewer_does_not_amplify_another_viewers_confirmed_mild_congestion() { + let mut qos = abr_session(); + qos.users.insert(2, UserData::default()); + for _ in 0..30 { + qos.user_network_delay(2, 10); + qos.user_network_delay(1, 10); + } + // The first reply of a new viewer adjusts the ratio and restarts the cooldown. + qos.advance_ms(4000); + let ratio = qos.ratio(); + // Viewer 2: mild congestion, confirmed over three replies. Viewer 1: one + // severe spike, never confirmed. Each viewer on its own calls for at most a + // five percent step; together they must not turn into a halving. + qos.user_network_delay(2, 200); + qos.user_network_delay(1, 1200); + qos.user_network_delay(2, 200); + let after_two = qos.ratio(); + assert!(after_two < ratio, "viewer 2's second bad reply cuts"); + assert!( + after_two >= ratio * 0.94, + "viewer 2's own mild excess is a five percent step, not {after_two}" + ); + qos.user_network_delay(2, 200); + qos.advance_ms(4000); + qos.update_display_data("test", 30); + assert!( + qos.ratio() >= after_two * 0.94, + "viewer 1's severity must not be paired with viewer 2's confirmation: {}", + qos.ratio() + ); +} + +/// What `on_connection_open` inserts, without touching the config store. +fn newcomer(qos: &VideoQoS) -> UserData { + UserData { + joined_at: Some(qos.now()), + ..Default::default() + } +} + +#[test] +fn closing_a_just_opened_viewer_does_not_throttle_existing_viewers() { + let mut qos = stable_qos(); + assert_eq!(qos.fps(), FPS); + qos.users.insert(2, newcomer(&qos)); + assert_eq!( + qos.fps(), + FPS, + "nothing changes until the stream is re-aggregated" + ); + qos.on_connection_close(2); + assert_eq!( + qos.fps(), + FPS, + "the guard leaves with the viewer that brought it" + ); +} + +#[test] +fn a_new_viewer_caps_the_stream_at_init_fps_for_a_second() { + let mut qos = stable_qos(); + qos.users.insert(2, newcomer(&qos)); + qos.user_network_delay(1, 10); + assert_eq!(qos.fps(), INIT_FPS); + qos.advance_ms(1000); + qos.user_network_delay(2, 10); + qos.user_network_delay(1, 10); + assert!( + qos.fps() > INIT_FPS, + "after a second the stream follows the viewers' own targets: {}", + qos.fps() + ); +} + +#[test] +fn closing_the_latest_newcomer_keeps_an_earlier_newcomers_guard() { + let mut qos = stable_qos(); + qos.users.insert(2, newcomer(&qos)); + qos.user_network_delay(2, 10); + assert_eq!(qos.fps(), INIT_FPS); + qos.advance_ms(100); + qos.users.insert(3, newcomer(&qos)); + qos.advance_ms(100); + qos.on_connection_close(3); + assert_eq!( + qos.fps(), + INIT_FPS, + "viewer 2 is still inside its own start-up window" + ); +} diff --git a/src/server/video_qos/tests/recovery.rs b/src/server/video_qos/tests/recovery.rs new file mode 100644 index 000000000..79dccf9e9 --- /dev/null +++ b/src/server/video_qos/tests/recovery.rs @@ -0,0 +1,120 @@ +use super::*; + +#[test] +fn ordinary_congestion_waits_between_bounded_cuts() { + for delay in [400, 800] { + let mut qos = stable_qos(); + for _ in 0..2 { + qos.user_network_delay(1, delay); + assert_eq!(qos.fps(), FPS); + } + qos.user_network_delay(1, delay); + let first_cut = qos.fps(); + assert!((24..FPS).contains(&first_cut), "delay={delay}: {first_cut}"); + for _ in 0..2 { + qos.user_network_delay(1, delay); + assert_eq!(qos.fps(), first_cut, "wait for new evidence after a cut"); + } + qos.user_network_delay(1, delay); + assert!(qos.fps() < first_cut); + assert!(qos.fps() >= first_cut - first_cut / 5); + } +} + +#[test] +fn automatic_floor_preserves_lower_custom_limits() { + for limit in [1, 3, 5, 30, 60, 120] { + for abr in [false, true] { + for timeout in [false, true] { + let mut qos = super::smoke::session(limit, Quality::Balanced); + qos.abr_config = abr; + qos.new_display("test".to_owned()); + qos.set_support_changing_quality("test", true); + for _ in 0..90 { + qos.user_network_delay(1, 10); + } + assert_eq!(qos.fps(), limit); + qos.ratio = BR_MIN_HIGH_RESOLUTION; + qos.store_bitrate(600); + let floor = 5.min(limit); + if timeout { + for elapsed in [2001, 3001, 4001, 5001, 10_000, 30_000] { + qos.user_delay_response_elapsed(1, elapsed); + assert!( + (floor..=limit).contains(&qos.fps()), + "timeout={elapsed}, limit={limit}, ABR={abr}" + ); + } + } else { + for _ in 0..8 { + qos.user_network_delay(1, 1500); + assert!( + (floor..=limit).contains(&qos.fps()), + "limit={limit}, ABR={abr}" + ); + } + } + assert_eq!( + qos.fps(), + floor, + "timeout={timeout}, limit={limit}, ABR={abr}" + ); + if timeout { + qos.user_network_delay(1, 30_100); + assert_eq!(qos.fps(), floor, "a late reply must preserve the floor"); + } + for _ in 0..2 { + qos.user_network_delay(1, 10); + } + assert_eq!(qos.fps(), limit, "recover: timeout={timeout}, ABR={abr}"); + } + } + } +} + +#[test] +fn two_good_replies_restore_after_a_severe_stall() { + let mut qos = stable_qos(); + qos.user_delay_response_elapsed(1, 5001); + assert_eq!(qos.fps(), 5); + qos.user_network_delay(1, 5100); + assert_eq!(qos.fps(), 5, "the late reply must not brake twice"); + qos.user_network_delay(1, 10); + assert!( + (5..FPS).contains(&qos.fps()), + "one good reply is not enough" + ); + qos.user_network_delay(1, 10); + assert_eq!(qos.fps(), FPS); +} + +#[test] +fn jitter_during_recovery_does_not_discard_the_restore_target() { + let mut qos = stable_qos(); + for _ in 0..3 { + qos.user_network_delay(1, 1200); + } + for delay in [10, 350, 10] { + qos.user_network_delay(1, delay); + } + assert_eq!(qos.fps(), FPS); +} + +#[test] +fn a_failed_fast_restore_rolls_back_before_the_queue_grows() { + let mut qos = stable_qos(); + qos.user_delay_response_elapsed(1, 5001); + for _ in 0..2 { + qos.user_network_delay(1, 10); + } + assert_eq!(qos.fps(), FPS); + qos.user_network_delay(1, 800); + assert_eq!(qos.fps(), FPS / 2); + for _ in 0..2 { + qos.user_network_delay(1, 10); + } + assert!( + qos.fps() < FPS, + "a failed restore must lower the next probe" + ); +} diff --git a/src/server/video_qos/tests/robustness.rs b/src/server/video_qos/tests/robustness.rs new file mode 100644 index 000000000..c3237977e --- /dev/null +++ b/src/server/video_qos/tests/robustness.rs @@ -0,0 +1,180 @@ +//! Guards against tuning the controller to the simulator: the CI bounds applied +//! to seeds that never took part in setting them, and a sweep of the scenario +//! parameters. Both are `#[ignore]`d: they take a few seconds and are meant for +//! anyone changing a controller constant or a bound. +//! +//! ```text +//! cargo test --lib video_qos::tests::robustness -- --ignored --nocapture +//! ``` +use super::sim::{bound_violations, run, scenarios, Report, Scenario, Summary}; +use super::*; + +fn summarise(sc: &Scenario, seeds: impl Iterator) -> Summary { + let reports: Vec = seeds + .map(|seed| run(&Scenario { seed, ..sc.clone() })) + .collect(); + Summary::of(&reports) +} + +/// Seeds 21 to 120 in five blocks of twenty, each block held to the CI bounds. +#[test] +#[ignore] +fn held_out_seeds() { + println!("| scenario | blocks violating | which | median target | worst p10 | below limit/2 (p90) | queue p95 (p90) |"); + println!("|---|---:|---|---:|---:|---:|---:|"); + let mut failures = Vec::new(); + for sc in scenarios() { + let mut failing_blocks = 0; + let mut which: Vec<&str> = Vec::new(); + let mut reports: Vec = Vec::new(); + for block in 0..5u64 { + let first = 21 + block * 20; + let block_reports: Vec = (first..first + 20) + .map(|seed| run(&Scenario { seed, ..sc.clone() })) + .collect(); + let s = Summary::of(&block_reports); + reports.extend(block_reports); + let violations = bound_violations(&s); + if !violations.is_empty() { + failing_blocks += 1; + for v in violations { + if !which.contains(&v) { + which.push(v); + } + } + } + } + let all = Summary::of(&reports); + println!( + "| {} | {}/5 | {} | {:.1} | {} | {:.1}% | {} ms |", + sc.name, + failing_blocks, + which.join("; "), + all.mean_target_median, + all.p10_target_worst, + all.below_half_p90, + all.queue_p95_p90 + ); + if failing_blocks > 0 { + failures.push(format!( + "{} ({} of 5 blocks: {})", + sc.name, + failing_blocks, + which.join("; ") + )); + } + } + assert!( + failures.is_empty(), + "bounds fail on held-out seeds, so they were fitted to the CI seeds: {failures:?}" + ); +} + +/// One scenario parameter at a time, halved and doubled, twenty seeds each. No +/// bounds: the point is to see that nothing falls off a cliff, and to compare with +/// master by running the same test there. +#[test] +#[ignore] +fn sensitivity() { + let base = scenarios(); + let pick = |name: &str| base.iter().find(|s| s.name == name).unwrap().clone(); + let home = pick("home_wifi_30"); + let halved = pick("bandwidth_halved_30"); + let with_link = |sc: &Scenario, edit: &dyn Fn(&mut super::sim::Link)| { + let mut link = sc.link.clone(); + edit(&mut link); + Scenario { link, ..sc.clone() } + }; + let variants: Vec<(&str, Scenario)> = vec![ + ("home base", home.clone()), + ( + "home stalls half as long", + with_link(&home, &|l| l.stall_ms = (150.0, 1250.0)), + ), + ( + "home stalls twice as long", + with_link(&home, &|l| l.stall_ms = (600.0, 5000.0)), + ), + ( + "home stalls twice as often", + with_link(&home, &|l| l.stall_mean_interval_s = 10.0), + ), + ( + "home stalls half as often", + with_link(&home, &|l| l.stall_mean_interval_s = 40.0), + ), + ( + "home loss doubled", + with_link(&home, &|l| l.loss_per_s = 0.4), + ), + ( + "home capacity halved", + with_link(&home, &|l| l.capacity_kbps = vec![(0, 10_000.0)]), + ), + ( + "home capacity 6 Mbps", + with_link(&home, &|l| l.capacity_kbps = vec![(0, 6_000.0)]), + ), + ( + "home jitter heavier", + with_link(&home, &|l| { + l.jitter_sigma = 1.5; + l.jitter_median_ms = 30.0; + }), + ), + ( + "home base rtt 80 ms", + with_link(&home, &|l| l.base_rtt_ms = 80.0), + ), + ( + "home 60 fps limit", + Scenario { + limit: 60, + ..home.clone() + }, + ), + ("bandwidth halved base", halved.clone()), + ( + "bandwidth to 4 Mbps", + with_link(&halved, &|l| { + l.capacity_kbps = vec![(0, 8_000.0), (60_000, 4_000.0), (120_000, 8_000.0)] + }), + ), + ( + "bandwidth to 1.5 Mbps", + with_link(&halved, &|l| { + l.capacity_kbps = vec![(0, 8_000.0), (60_000, 1_500.0), (120_000, 8_000.0)] + }), + ), + ( + "bandwidth halved, never restored, 300 s", + Scenario { + seconds: 300, + ..with_link(&halved, &|l| { + l.capacity_kbps = vec![(0, 8_000.0), (60_000, 2_500.0)] + }) + }, + ), + ]; + println!("| variant | median target | worst p10 | below limit/2 (p90) | queue p95 (p90) | delivered | sustained recovery (worst) |"); + println!("|---|---:|---:|---:|---:|---:|---:|"); + for (label, sc) in &variants { + let s = summarise(sc, super::sim::SEEDS); + println!( + "| {} | {:.1} | {} | {:.1}% | {} ms | {:.1} | {} |", + label, + s.mean_target_median, + s.p10_target_worst, + s.below_half_p90, + s.queue_p95_p90, + s.delivered_median, + if s.has_restore { + s.recovery_worst_ms + .map(|ms| format!("{:.1}s", ms as f64 / 1000.0)) + .unwrap_or_else(|| "never".to_owned()) + } else { + "-".to_owned() + } + ); + } +} diff --git a/src/server/video_qos/tests/sim.rs b/src/server/video_qos/tests/sim.rs new file mode 100644 index 000000000..1e8f9f81f --- /dev/null +++ b/src/server/video_qos/tests/sim.rs @@ -0,0 +1,988 @@ +//! Closed-loop network simulation for the QoS controller. +//! +//! The controller is driven the way `Connection` drives it: one TestDelay probe per +//! second, a single probe outstanding, `user_delay_response_elapsed` on every timer +//! tick, `update_display_data` once per second. Video frames and probes share one +//! FIFO, which stands for the downstream shared path (stream, transport, link): the +//! probe measures the bytes that were handed to that path in front of it. It is not +//! the server's `tx_video` channel, which the probe does not pass through, and the +//! model does not stall the timer while a send is blocked, as the real loop does. +//! +//! Three independent random streams keep an A/B comparison paired: the network +//! trace (capacity wobble, stalls, loss events) is generated before the run from the +//! network stream alone, probe jitter is a per-second table from its own stream, +//! and scene changes follow the wall clock, so two controllers with the same seed +//! face the same link, the same jitter and the same content timeline whatever they +//! decide. Only the frame size noise depends on how many frames were produced. +//! +//! The encoder model conserves its bitrate budget: a scene change costs three +//! frames' worth of data and the surplus is repaid by the following frames, so the +//! long-term offered load does not depend on the frame rate under CBR. +//! +//! It still is a model, not a network: it does not reproduce a real transport's +//! congestion control or a real encoder. Its job is to show how the controller +//! reacts to the *kind* of behaviour a home Wi-Fi, a stable relay or a saturated +//! uplink produce, deterministically and over many seeds. +//! +//! Against overfitting: the CI run uses seeds 1 to 20; `robustness.rs` applies the +//! same bounds to seeds 21 to 120 and sweeps the scenario parameters. Scenario +//! parameters are educated guesses until a recorded `qos_trace` calibrates them. +use super::*; + +/// xorshift64* generator, so the tests need no external crate and stay reproducible. +struct Rng(u64); + +impl Rng { + fn new(seed: u64) -> Self { + Rng((seed ^ 0x9E37_79B9_7F4A_7C15).max(1)) + } + + fn next_u64(&mut self) -> u64 { + let mut x = self.0; + x ^= x >> 12; + x ^= x << 25; + x ^= x >> 27; + self.0 = x; + x.wrapping_mul(0x2545_F491_4F6C_DD1D) + } + + /// Uniform in `[0, 1)`. + fn uniform(&mut self) -> f64 { + (self.next_u64() >> 11) as f64 / (1u64 << 53) as f64 + } + + fn range(&mut self, lo: f64, hi: f64) -> f64 { + lo + (hi - lo) * self.uniform() + } + + fn normal(&mut self) -> f64 { + let u1 = (1.0 - self.uniform()).max(1e-12); + let u2 = self.uniform(); + (-2.0 * u1.ln()).sqrt() * (2.0 * std::f64::consts::PI * u2).cos() + } + + fn log_normal(&mut self, median: f64, sigma: f64) -> f64 { + median * (sigma * self.normal()).exp() + } + + fn exponential(&mut self, mean: f64) -> f64 { + -mean * (1.0 - self.uniform()).max(1e-12).ln() + } +} + +#[derive(Clone)] +pub struct Link { + /// Step schedule `(from_ms, kbps)`, sorted by time. + pub capacity_kbps: Vec<(u32, f64)>, + /// Slow random walk of the capacity, as a fraction of the nominal value. + pub wobble: f64, + pub base_rtt_ms: f64, + /// Log-normal jitter added to every probe round trip. + pub jitter_median_ms: f64, + pub jitter_sigma: f64, + /// Loss events per second. A reliable stream turns a loss into a 200-400 ms + /// retransmission stall followed by a second at half rate. + pub loss_per_s: f64, + /// Mean interval between link stalls in seconds, `0` for none. + pub stall_mean_interval_s: f64, + /// Uniform stall duration range in milliseconds. + pub stall_ms: (f64, f64), +} + +impl Link { + fn capacity_at(&self, now_ms: u32) -> f64 { + self.capacity_kbps + .iter() + .rev() + .find(|(from, _)| *from <= now_ms) + .map(|(_, kbps)| *kbps) + .unwrap_or(self.capacity_kbps[0].1) + } + + /// Time at which the capacity was last restored to its initial value, if it ever dropped. + fn restore_ms(&self) -> Option { + let initial = self.capacity_kbps[0].1; + let mut dropped = false; + for (from, kbps) in &self.capacity_kbps { + if *kbps < initial { + dropped = true; + } else if dropped && *kbps >= initial { + return Some(*from); + } + } + None + } +} + +/// Everything the link does during a run, decided before the run starts. +struct LinkTrace { + capacity_kbps: Vec, // per tick, wobble and retransmission backoff applied + stalled: Vec, // per tick +} + +fn mark(flags: &mut [bool], from_ms: f64, to_ms: f64) { + let from = (from_ms / TICK_MS as f64).max(0.0) as usize; + let to = ((to_ms / TICK_MS as f64).ceil() as usize).min(flags.len()); + for flag in flags.iter_mut().take(to).skip(from) { + *flag = true; + } +} + +fn link_trace(link: &Link, ticks: usize, rng: &mut Rng) -> LinkTrace { + let mut capacity_kbps = vec![0.0; ticks]; + let mut stalled = vec![false; ticks]; + let mut backoff = vec![false; ticks]; + let mut wobble = 0.0_f64; + for (i, capacity) in capacity_kbps.iter_mut().enumerate() { + let now = i as u32 * TICK_MS; + if now % 100 == 0 { + wobble = (wobble + rng.normal() * 0.03).clamp(-link.wobble, link.wobble); + } + *capacity = link.capacity_at(now) * (1.0 + wobble); + } + if link.stall_mean_interval_s > 0.0 { + let mut start = rng.exponential(link.stall_mean_interval_s) * 1000.0; + while start < (ticks as f64) * TICK_MS as f64 { + let len = rng.range(link.stall_ms.0, link.stall_ms.1); + mark(&mut stalled, start, start + len); + start += rng.exponential(link.stall_mean_interval_s) * 1000.0; + } + } + if link.loss_per_s > 0.0 { + let per_tick = link.loss_per_s * TICK_MS as f64 / 1000.0; + for i in 0..ticks { + if rng.uniform() < per_tick { + let start = (i as u32 * TICK_MS) as f64; + let len = rng.range(200.0, 400.0); + mark(&mut stalled, start, start + len); + mark(&mut backoff, start + len, start + len + 1000.0); + } + } + } + for (capacity, backoff) in capacity_kbps.iter_mut().zip(&backoff) { + if *backoff { + *capacity *= 0.5; + } + } + LinkTrace { + capacity_kbps, + stalled, + } +} + +#[derive(Clone, Copy, PartialEq)] +pub enum Content { + /// Every frame changes: a video call or a movie. + Video, + /// Mostly static: a couple of changed frames per second. + Office, +} + +/// How the encoder turns a bitrate into frame sizes. +#[derive(Clone, Copy, PartialEq)] +pub enum EncoderModel { + /// VP8, VP9 and AV1 run CBR against millisecond timestamps: fewer frames per + /// second means bigger frames, the bitrate stays. Only the ratio moves bytes. + Cbr, + /// Hardware encoders configured for a fixed 30 fps rate-control assumption: + /// every frame carries a thirtieth of the bitrate, so fewer frames mean fewer + /// bytes. Actual hardware behaviour is backend dependent (Android's MediaCodec + /// path runs VBR). + FixedRate, +} + +#[derive(Clone)] +pub struct Scenario { + pub name: &'static str, + pub seconds: u32, + pub limit: u32, + pub quality: Quality, + pub abr: bool, + pub content: Content, + pub encoder: EncoderModel, + pub link: Link, + pub seed: u64, +} + +/// Bitrate at ratio 1.0; balanced quality (0.67) then encodes at about 4 Mbps. +const BASE_KBPS: f64 = 6000.0; +/// The frame rate hardware encoders are configured for. +const ENCODER_CONFIGURED_FPS: f64 = 30.0; +/// Log-normal spread of frame sizes around their budget. +const FRAME_SIZE_SIGMA: f64 = 0.35; +const TICK_MS: u32 = 10; +/// Samples taken before this instant belong to the cold start, not the steady state. +const WARM_UP_MS: u32 = 15_000; +/// A recovery counts once target and queue have held for this long. +const SUSTAINED_MS: u32 = 5_000; +/// Seeds every scenario is run with. +pub const SEEDS: std::ops::RangeInclusive = 1..=20; + +/// Frame sizes with a conserved bitrate budget. +struct Encoder { + model: EncoderModel, + content: Content, + rng: Rng, + next_scene_ms: u32, + debt_bits: f64, +} + +/// A scene change every five seconds of video. +const SCENE_INTERVAL_MS: u32 = 5_000; + +impl Encoder { + fn frame_bits(&mut self, now_ms: u32, bitrate_kbps: f64, produce_rate: f64) -> f64 { + let target = match (self.content, self.model) { + // A changed region of a static screen is small whatever the rate control does. + (Content::Office, _) => bitrate_kbps * 1000.0 / ENCODER_CONFIGURED_FPS * 0.3, + (Content::Video, EncoderModel::Cbr) => bitrate_kbps * 1000.0 / produce_rate, + (Content::Video, EncoderModel::FixedRate) => { + bitrate_kbps * 1000.0 / ENCODER_CONFIGURED_FPS + } + }; + // Mean one: the spread must not change the offered load. + let noise = self.rng.log_normal(1.0, FRAME_SIZE_SIGMA) + * (-FRAME_SIZE_SIGMA * FRAME_SIZE_SIGMA / 2.0).exp(); + let mut bits = target * noise; + // Scene changes follow the wall clock, not the frame count, so every + // controller meets the same content timeline. + let scene_change = self.content == Content::Video && now_ms >= self.next_scene_ms; + if scene_change { + self.next_scene_ms += SCENE_INTERVAL_MS; + // A scene change costs a few frames' worth of data; rate control claws + // it back from the frames that follow. + bits *= 3.0; + self.debt_bits += bits - target; + } else if self.debt_bits > 0.0 { + let repay = self.debt_bits.min(target * 0.5).min(bits * 0.5); + bits -= repay; + self.debt_bits -= repay; + } + bits + } +} + +struct Packet { + bits: f64, + enqueued_ms: u32, + probe_sent_ms: Option, +} + +#[derive(Debug, Clone)] +pub struct Report { + pub name: String, + pub seed: u64, + pub limit: u32, + /// Controller target, sampled every 100 ms after the warm-up. + pub mean_target_fps: f64, + pub p10_target_fps: u32, + pub min_target_fps: u32, + /// Share of the measured time the target spent below half of the limit. + pub below_half_pct: f64, + /// Frames the encoder produced per second. + pub produced_fps: f64, + /// Frames that left the shared path per second. + pub delivered_fps: f64, + /// Time a delivered frame spent in the shared path, 95th percentile. + pub frame_age_p95_ms: u32, + pub queue_p95_ms: u32, + pub max_delay_ms: u32, + /// Whether the link drops and restores its capacity at all. + pub has_restore: bool, + /// Time from the capacity restore until target at the limit and queue below + /// 200 ms held for `SUSTAINED_MS`. + pub recovery_ms: Option, + /// Lowest target during the first `WARM_UP_MS`. + pub cold_start_min_fps: u32, + /// First time the target reached 90% of the limit. + pub time_to_90pct_ms: Option, + pub final_fps: u32, + pub final_ratio: f32, + pub trace: Vec<(u32, u32, u32, f32)>, // (time_ms, target fps, queue_ms, ratio) +} + +fn percentile_u32(values: &[u32], p: f64) -> u32 { + if values.is_empty() { + return 0; + } + let mut sorted = values.to_vec(); + sorted.sort_unstable(); + sorted[(((sorted.len() - 1) as f64) * p).round() as usize] +} + +fn percentile_f64(values: &[f64], p: f64) -> f64 { + if values.is_empty() { + return 0.0; + } + let mut sorted = values.to_vec(); + sorted.sort_by(|a, b| a.partial_cmp(b).unwrap()); + sorted[(((sorted.len() - 1) as f64) * p).round() as usize] +} + +pub fn run(sc: &Scenario) -> Report { + let mut network_rng = Rng::new(sc.seed); + let mut probe_rng = Rng::new(sc.seed ^ 0x5052_4F42_45); + let mut encoder = Encoder { + model: sc.encoder, + content: sc.content, + rng: Rng::new(sc.seed ^ 0x454E_434F_4445), + next_scene_ms: SCENE_INTERVAL_MS, + debt_bits: 0.0, + }; + let total_ms = sc.seconds * 1000; + let ticks = (total_ms / TICK_MS) as usize; + let link = link_trace(&sc.link, ticks, &mut network_rng); + // Probe jitter indexed by the probe's send second, so the number of probes a + // controller manages to send does not change the jitter the next one meets. + let probe_jitter_ms: Vec = (0..=sc.seconds) + .map(|_| probe_rng.log_normal(sc.link.jitter_median_ms, sc.link.jitter_sigma)) + .collect(); + + let mut qos = super::smoke::session(sc.limit, sc.quality); + qos.abr_config = sc.abr; + if sc.abr { + qos.new_display("sim".to_owned()); + qos.set_support_changing_quality("sim", true); + } + + let mut queue: VecDeque = VecDeque::new(); + let mut queued_bits = 0.0_f64; + let mut encode_phase = 0.0_f64; + let mut encoded_this_second = 0_usize; + let mut probe_sent: Option = None; + let mut replies: Vec<(u32, u32)> = Vec::new(); // (arrive_ms, delay_ms) + let restore_ms = sc.link.restore_ms(); + + let mut fps_samples = Vec::new(); + let mut queue_samples = Vec::new(); + let mut produced = 0_u64; + let mut delivered = 0_u64; + let mut frame_ages = Vec::new(); + let mut trace = Vec::new(); + let mut max_delay = 0_u32; + let mut recovery_ms = None; + let mut good_since: Option = None; + let mut cold_start_min_fps = u32::MAX; + let mut time_to_90pct_ms = None; + + for tick in 0..ticks { + let now = tick as u32 * TICK_MS; + qos.advance_ms(TICK_MS as u64); + let capacity_kbps = link.capacity_kbps[tick]; + + // Encoder: frames at the controller's rate, sized by the controller's ratio. + // The video loop reports the bitrate as soon as it applies a new ratio. + let fps = qos.fps(); + let ratio = qos.ratio(); + let bitrate_kbps = BASE_KBPS * ratio as f64; + qos.store_bitrate(bitrate_kbps as u32); + let produce_rate = match sc.content { + Content::Video => fps as f64, + Content::Office => (fps as f64).min(2.0), + }; + encode_phase += produce_rate * TICK_MS as f64 / 1000.0; + while encode_phase >= 1.0 { + encode_phase -= 1.0; + encoded_this_second += 1; + if now >= WARM_UP_MS { + produced += 1; + } + let bits = encoder.frame_bits(now, bitrate_kbps, produce_rate); + queue.push_back(Packet { + bits, + enqueued_ms: now, + probe_sent_ms: None, + }); + queued_bits += bits; + } + + // Shared path drain: probes are tiny and leave as soon as they reach the head. + if !link.stalled[tick] { + let mut budget = capacity_kbps * TICK_MS as f64; + while budget > 0.0 { + let Some(head) = queue.front_mut() else { break }; + if let Some(sent) = head.probe_sent_ms { + let round_trip = sc.link.base_rtt_ms + probe_jitter_ms[(sent / 1000) as usize]; + let arrive = now + round_trip as u32; + replies.push((arrive, arrive - sent)); + queue.pop_front(); + continue; + } + let take = budget.min(head.bits); + head.bits -= take; + queued_bits -= take; + budget -= take; + if head.bits <= 1e-9 { + if now >= WARM_UP_MS { + delivered += 1; + frame_ages.push(now - head.enqueued_ms); + } + queue.pop_front(); + } + } + } + + // Probe replies reach the controller. + replies.sort_by_key(|r| r.0); + while replies.first().is_some_and(|r| r.0 <= now) { + let (_, delay) = replies.remove(0); + max_delay = max_delay.max(delay); + probe_sent = None; + qos.user_network_delay(1, delay); + } + + // The connection's one second timer. + if now % 1000 == 0 { + if probe_sent.is_none() { + probe_sent = Some(now); + queue.push_back(Packet { + bits: 0.0, + enqueued_ms: now, + probe_sent_ms: Some(now), + }); + } + qos.user_delay_response_elapsed(1, (now - probe_sent.unwrap()) as u128); + if sc.abr { + qos.update_display_data("sim", encoded_this_second); + } + encoded_this_second = 0; + } + + if now % 100 == 0 { + let queue_ms = (queued_bits / capacity_kbps.max(1.0)) as u32; + let fps = qos.fps(); + trace.push((now, fps, queue_ms, qos.ratio())); + if now < WARM_UP_MS { + cold_start_min_fps = cold_start_min_fps.min(fps); + } else { + fps_samples.push(fps); + queue_samples.push(queue_ms); + } + if time_to_90pct_ms.is_none() && fps * 10 >= sc.limit * 9 { + time_to_90pct_ms = Some(now); + } + if let Some(restore) = restore_ms { + if now >= restore && recovery_ms.is_none() { + if fps >= sc.limit && queue_ms < 200 { + let since = *good_since.get_or_insert(now); + if now - since >= SUSTAINED_MS { + recovery_ms = Some(since - restore); + } + } else { + good_since = None; + } + } + } + } + } + + let measured_s = (total_ms - WARM_UP_MS) as f64 / 1000.0; + let below_half = fps_samples.iter().filter(|f| **f * 2 < sc.limit).count(); + Report { + name: sc.name.to_owned(), + seed: sc.seed, + limit: sc.limit, + mean_target_fps: fps_samples.iter().map(|f| *f as f64).sum::() + / fps_samples.len().max(1) as f64, + p10_target_fps: percentile_u32(&fps_samples, 0.10), + min_target_fps: fps_samples.iter().copied().min().unwrap_or(0), + below_half_pct: 100.0 * below_half as f64 / fps_samples.len().max(1) as f64, + produced_fps: produced as f64 / measured_s, + delivered_fps: delivered as f64 / measured_s, + frame_age_p95_ms: percentile_u32(&frame_ages, 0.95), + queue_p95_ms: percentile_u32(&queue_samples, 0.95), + max_delay_ms: max_delay, + has_restore: restore_ms.is_some(), + recovery_ms, + cold_start_min_fps, + time_to_90pct_ms, + final_fps: qos.fps(), + final_ratio: qos.ratio(), + trace, + } +} + +/// One scenario over all seeds, summarised by the statistics the assertions use. +#[derive(Debug)] +pub struct Summary { + pub name: String, + pub limit: u32, + pub mean_target_median: f64, + pub p10_target_worst: u32, + pub below_half_p90: f64, + pub queue_p95_p90: u32, + pub delivered_median: f64, + pub frame_age_p95_p90: u32, + pub has_restore: bool, + /// Slowest sustained recovery, `None` when any seed never recovered. + pub recovery_worst_ms: Option, + pub cold_start_min_median: u32, + /// Slowest time to 90% of the limit, `None` when any seed never got there. + pub time_to_90pct_worst_ms: Option, +} + +impl Summary { + pub fn of(reports: &[Report]) -> Self { + let f = |g: fn(&Report) -> f64| reports.iter().map(g).collect::>(); + let u = |g: fn(&Report) -> u32| reports.iter().map(g).collect::>(); + let all = |g: fn(&Report) -> Option| { + reports + .iter() + .map(g) + .try_fold(0, |worst, ms| ms.map(|ms| worst.max(ms))) + }; + Summary { + name: reports[0].name.clone(), + limit: reports[0].limit, + mean_target_median: percentile_f64(&f(|r| r.mean_target_fps), 0.5), + p10_target_worst: percentile_u32(&u(|r| r.p10_target_fps), 0.0), + below_half_p90: percentile_f64(&f(|r| r.below_half_pct), 0.9), + queue_p95_p90: percentile_u32(&u(|r| r.queue_p95_ms), 0.9), + delivered_median: percentile_f64(&f(|r| r.delivered_fps), 0.5), + frame_age_p95_p90: percentile_u32(&u(|r| r.frame_age_p95_ms), 0.9), + has_restore: reports[0].has_restore, + recovery_worst_ms: all(|r| r.recovery_ms), + cold_start_min_median: percentile_u32(&u(|r| r.cold_start_min_fps), 0.5), + time_to_90pct_worst_ms: all(|r| r.time_to_90pct_ms), + } + } + + pub const HEADER: &'static str = "| scenario | limit | target fps (median of means) | worst p10 | below limit/2 (p90) | queue p95 (p90) | delivered fps (median) | frame age p95 (p90) | sustained recovery (worst) | cold-start min (median) | time to 90% (worst) |\n|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|"; + + pub fn row(&self) -> String { + let secs = |ms: Option| { + ms.map(|ms| format!("{:.1}s", ms as f64 / 1000.0)) + .unwrap_or_else(|| "never".to_owned()) + }; + format!( + "| {} | {} | {:.1} | {} | {:.1}% | {} ms | {:.1} | {} ms | {} | {} | {} |", + self.name, + self.limit, + self.mean_target_median, + self.p10_target_worst, + self.below_half_p90, + self.queue_p95_p90, + self.delivered_median, + self.frame_age_p95_p90, + if self.has_restore { + secs(self.recovery_worst_ms) + } else { + "-".to_owned() + }, + self.cold_start_min_median, + secs(self.time_to_90pct_worst_ms), + ) + } +} + +fn clean_link(capacity_kbps: f64) -> Link { + Link { + capacity_kbps: vec![(0, capacity_kbps)], + wobble: 0.05, + base_rtt_ms: 15.0, + jitter_median_ms: 3.0, + jitter_sigma: 0.5, + loss_per_s: 0.0, + stall_mean_interval_s: 0.0, + stall_ms: (0.0, 0.0), + } +} + +/// Weak-signal home Wi-Fi with ample average capacity: heavy-tailed jitter, +/// retransmissions, and a link stall of up to 2.5 s every twenty seconds or so. +/// Deliberately nasty; it isolates "capacity is fine, timing is not". +fn home_wifi_link() -> Link { + Link { + capacity_kbps: vec![(0, 20_000.0)], + wobble: 0.5, + base_rtt_ms: 8.0, + jitter_median_ms: 15.0, + jitter_sigma: 1.0, + loss_per_s: 0.2, + stall_mean_interval_s: 20.0, + stall_ms: (300.0, 2500.0), + } +} + +fn intercontinental_link() -> Link { + Link { + capacity_kbps: vec![(0, 20_000.0)], + wobble: 0.1, + base_rtt_ms: 250.0, + jitter_median_ms: 5.0, + jitter_sigma: 0.5, + loss_per_s: 0.05, + stall_mean_interval_s: 0.0, + stall_ms: (0.0, 0.0), + } +} + +/// 8 Mbps for a minute, 2.5 Mbps for the next, 8 Mbps again. +fn halved_link() -> Link { + Link { + capacity_kbps: vec![(0, 8_000.0), (60_000, 2_500.0), (120_000, 8_000.0)], + ..clean_link(8_000.0) + } +} + +fn mobile_link() -> Link { + Link { + capacity_kbps: vec![(0, 6_000.0)], + wobble: 0.4, + base_rtt_ms: 40.0, + jitter_median_ms: 30.0, + jitter_sigma: 0.8, + loss_per_s: 0.02, + stall_mean_interval_s: 0.0, + stall_ms: (0.0, 0.0), + } +} + +pub fn scenarios() -> Vec { + let base = |name, limit, link, abr, encoder| Scenario { + name, + seconds: 180, + limit, + quality: Quality::Balanced, + abr, + content: Content::Video, + encoder, + link, + seed: 1, + }; + use EncoderModel::*; + vec![ + base("home_wifi_30", 30, home_wifi_link(), true, Cbr), + base("home_wifi_60", 60, home_wifi_link(), true, Cbr), + base( + "home_wifi_fixed_rate_30", + 30, + home_wifi_link(), + true, + FixedRate, + ), + base("home_wifi_no_abr_30", 30, home_wifi_link(), false, Cbr), + Scenario { + content: Content::Office, + ..base("office_home_wifi_30", 30, home_wifi_link(), true, Cbr) + }, + base("city_relay_30", 30, clean_link(50_000.0), true, Cbr), + base("city_relay_60", 60, clean_link(50_000.0), true, Cbr), + base( + "intercontinental_30", + 30, + intercontinental_link(), + true, + Cbr, + ), + base("bandwidth_halved_30", 30, halved_link(), true, Cbr), + base( + "bandwidth_halved_fixed_rate_30", + 30, + halved_link(), + true, + FixedRate, + ), + base( + "bandwidth_halved_fixed_rate_no_abr_30", + 30, + halved_link(), + false, + FixedRate, + ), + base("bandwidth_halved_no_abr_30", 30, halved_link(), false, Cbr), + base("mobile_bufferbloat_30", 30, mobile_link(), true, Cbr), + ] +} + +/// Runs every scenario over `SEEDS` and returns the per-scenario summaries. +pub fn run_all() -> Vec<(Summary, Vec)> { + scenarios() + .iter() + .map(|sc| { + let reports: Vec = SEEDS + .map(|seed| run(&Scenario { seed, ..sc.clone() })) + .collect(); + (Summary::of(&reports), reports) + }) + .collect() +} + +fn write_traces(results: &[(Summary, Vec)]) { + use std::fmt::Write; + if let Ok(path) = std::env::var("RUSTDESK_QOS_SIM_CSV") { + let mut csv = String::from("scenario,seed,time_ms,target_fps,queue_ms,ratio\n"); + for (_, reports) in results { + for report in reports { + for (t, fps, queue, ratio) in &report.trace { + writeln!( + csv, + "{},{},{t},{fps},{queue},{ratio:.3}", + report.name, report.seed + ) + .unwrap(); + } + } + } + std::fs::write(path, csv).unwrap(); + } +} + +#[test] +fn sim_scenarios() { + let results = run_all(); + println!("{}", Summary::HEADER); + for (summary, _) in &results { + println!("{}", summary.row()); + } + if std::env::var("RUSTDESK_QOS_SIM_VERBOSE").is_ok() { + for (_, reports) in &results { + for r in reports { + println!( + "{} seed {}: target mean {:.1} p10 {} min {} below-half {:.1}% delivered {:.1} age p95 {} queue p95 {} max probe {} recovery {:?} cold-start min {} t90 {:?}", + r.name, r.seed, r.mean_target_fps, r.p10_target_fps, r.min_target_fps, + r.below_half_pct, r.delivered_fps, r.frame_age_p95_ms, r.queue_p95_ms, + r.max_delay_ms, r.recovery_ms, r.cold_start_min_fps, r.time_to_90pct_ms + ); + } + } + } + write_traces(&results); + for (summary, _) in &results { + let violations = bound_violations(summary); + assert!( + violations.is_empty(), + "{}: {violations:?}\n{summary:?}", + summary.name + ); + } +} + +/// The bounds every scenario summary has to meet, shared by the CI run over `SEEDS` +/// and by the held-out run in `robustness.rs`. They state what the product needs, +/// not what one seed produced. If a new seed or a new scenario violates a bound, +/// change the design or loosen the bound with a written reason; never tune a +/// controller constant until the bound passes. +pub fn bound_violations(s: &Summary) -> Vec<&'static str> { + let name = s.name.as_str(); + let limit = s.limit as f64; + let mut v = Vec::new(); + let mut check = |ok: bool, what: &'static str| { + if !ok { + v.push(what); + } + }; + if name.starts_with("home_wifi") || name.starts_with("office_home_wifi") { + // A jittery but healthy link must stay fast: the whole point of the change. + // The target rarely leaves the limit, never collapses, and stalls of up to + // 2.5 s leave about a second of queue at worst. + check( + s.mean_target_median >= 0.85 * limit, + "median target below 85%", + ); + check(s.p10_target_worst * 3 >= s.limit, "worst p10 below a third"); + check(s.below_half_p90 <= 10.0, "below half the limit over 10%"); + check(s.queue_p95_p90 < 1000, "queue p95 p90 over 1 s"); + if name != "office_home_wifi_30" { + check(s.delivered_median >= 0.8 * limit, "delivered below 80%"); + } + // Frame age is the time a delivered frame spent in the shared path: what a + // viewer waits for on top of the round trip. A jittery high-capacity link + // contains isolated stalls of up to 2.5 s, and the bound is on the p90 of + // per-seed p95 frame age: isolated stalls are tolerated, but they must not + // turn into a sustained multi-second backlog. A regression bound, not a + // latency target; set from the scenario, not from a run. + check(s.frame_age_p95_p90 < 1500, "frame age p95 p90 over 1.5 s"); + } else if name.starts_with("city_relay") { + // A clean link is where the developers test; every seed sits at the limit, + // and a fresh connection reaches 90% of it within ten seconds. + check(s.p10_target_worst == s.limit, "left the limit"); + check(s.queue_p95_p90 < 50, "queue on a clean link"); + check(s.frame_age_p95_p90 < 100, "frame age on a clean link"); + check( + s.time_to_90pct_worst_ms.is_some_and(|ms| ms <= 10_000), + "cold start over 10 s", + ); + } else if name == "intercontinental_30" { + // High but stable RTT is not congestion, not even during the cold start. + check( + s.mean_target_median >= 0.9 * limit, + "median target below 90%", + ); + check( + s.cold_start_min_median >= INIT_FPS, + "cold start below INIT_FPS", + ); + // Frame age excludes the round trip, so a high RTT earns no allowance. + check(s.frame_age_p95_p90 < 150, "frame age over 150 ms"); + check( + s.time_to_90pct_worst_ms.is_some_and(|ms| ms <= 10_000), + "cold start over 10 s", + ); + } else if let Some((queue_p95_bound_ms, below_half_bound_pct)) = match name { + // Real congestion must be detected, drained and recovered from. With a CBR + // encoder only the bitrate drains the queue, and three probe replies at one + // second cadence plus a three second ratio cooldown are needed before a + // confirmed cut, so a few seconds of queue are inherent there. Without + // ABR nothing drains a CBR queue at all, so that combination is reported + // but not asserted. + "bandwidth_halved_30" => Some((3000, 40.0)), + "bandwidth_halved_fixed_rate_30" => Some((2000, 10.0)), + "bandwidth_halved_fixed_rate_no_abr_30" => Some((3000, 30.0)), + _ => None, + } { + check(s.queue_p95_p90 < queue_p95_bound_ms, "queue p95 p90 bound"); + check( + s.frame_age_p95_p90 < queue_p95_bound_ms, + "frame age p95 p90 bound", + ); + check(s.below_half_p90 <= below_half_bound_pct, "below half bound"); + check( + s.recovery_worst_ms.is_some_and(|ms| ms <= 20_000), + "sustained recovery over 20 s", + ); + } else if name == "mobile_bufferbloat_30" { + check(s.queue_p95_p90 < 2000, "queue p95 p90 over 2 s"); + check(s.frame_age_p95_p90 < 2000, "frame age p95 p90 over 2 s"); + check(s.below_half_p90 <= 15.0, "below half the limit over 15%"); + } + v +} + +/// Replays `qos_trace` lines through a fresh controller and returns +/// `(time_ms, id, recorded_fps, replayed_fps)` per line. Open loop, FPS only: +/// the recorded delays do not react to the replayed decisions, the session runs +/// with ABR off, and connections are replayed as separate viewers of one 30 fps +/// balanced session. Time advances by the wall-clock delta between consecutive +/// lines whatever their connection, or by one second per line when a trace +/// predates the `t=` field. +pub fn replay(text: &str) -> Vec<(u64, i32, u64, u32)> { + // A present but malformed value is a corrupt trace, not a missing field. + let field = |line: &str, key: &str| -> Option { + line.split_whitespace() + .find_map(|kv| kv.strip_prefix(key).and_then(|v| v.strip_prefix('='))) + .map(|v| { + v.parse() + .unwrap_or_else(|e| panic!("bad {key}={v:?} in {line:?}: {e}")) + }) + }; + let mut qos = super::smoke::session(30, Quality::Balanced); + qos.users.clear(); + let mut last_t: Option = None; + let mut now = 0_u64; + let mut trace = Vec::new(); + for line in text.lines().filter(|l| l.contains("qos_trace")) { + let id = field(line, "id").unwrap_or(1) as i32; + qos.users.entry(id).or_default(); + let t = field(line, "t"); + let step = match (t, last_t) { + (Some(t), Some(prev)) => t.saturating_sub(prev).clamp(1, 10_000), + _ => 1000, + }; + if t.is_some() { + last_t = t; + } + now += step; + qos.advance_ms(step); + if let Some(elapsed) = field(line, "timeout") { + qos.user_delay_response_elapsed(id, elapsed as u128); + } else if let Some(delay) = field(line, "delay") { + qos.user_delay_response_elapsed(id, 0); + qos.user_network_delay(id, delay as u32); + } + let recorded = field(line, "fps").unwrap_or(0); + trace.push((now, id, recorded, qos.fps())); + } + trace +} + +/// Replays the log named by `RUSTDESK_QOS_TRACE` and prints the result. +#[test] +fn replay_recorded_trace() { + let Ok(path) = std::env::var("RUSTDESK_QOS_TRACE") else { + return; + }; + let trace = replay(&std::fs::read_to_string(&path).unwrap()); + println!("time_ms,id,recorded_fps,replayed_fps"); + for (t, id, recorded, replayed) in &trace { + println!("{t},{id},{recorded},{replayed}"); + } + let mean = trace.iter().map(|t| t.3 as f64).sum::() / trace.len().max(1) as f64; + println!( + "replayed mean target fps: {mean:.1} over {} lines", + trace.len() + ); +} + +#[test] +fn replay_time_axis_is_shared_across_connections() { + // Two viewers each log once a second for twenty seconds: twenty seconds of + // wall clock, not forty. + let text: String = (0..20) + .flat_map(|i| { + [ + format!("qos_trace t={} id=1 delay=10 fps=30\n", 100_000 + i * 1000), + format!("qos_trace t={} id=2 delay=10 fps=30\n", 100_001 + i * 1000), + ] + }) + .collect(); + let trace = replay(&text); + let elapsed = trace.last().unwrap().0 - trace.first().unwrap().0; + assert!( + (19_000..=19_100).contains(&elapsed), + "replayed {elapsed} ms for 19 s of wall clock" + ); +} + +#[test] +fn replay_recorded_trace_is_independent_of_connection_id() { + let replay = |id: i32| { + let text: String = (0..20) + .map(|i| { + format!( + "qos_trace t={} id={id} delay=10 fps=30\n", + 100_000 + i * 1000 + ) + }) + .collect(); + let nonce = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let path = std::env::temp_dir().join(format!( + "rustdesk-qos-replay-{}-{nonce}-{id}.log", + std::process::id() + )); + std::fs::write(&path, text).unwrap(); + // Exercise the real replay entry point without changing other tests' environment. + let test = format!( + "{}::replay_recorded_trace", + module_path!().split_once("::").unwrap().1 + ); + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", &test, "--nocapture", "--test-threads=1"]) + .env("RUSTDESK_QOS_TRACE", &path) + .output(); + std::fs::remove_file(&path).unwrap(); + let output = output.unwrap(); + assert!(output.status.success(), "replay failed: {output:?}"); + let stdout = String::from_utf8(output.stdout).unwrap(); + let id = id.to_string(); + let fps: Vec = stdout + .lines() + .filter_map(|line| { + let fields: Vec<_> = line.split(',').collect(); + if fields.len() == 4 && fields[1] == id { + Some(fields[3].parse().unwrap()) + } else { + None + } + }) + .collect(); + assert_eq!(fps.len(), 20, "missing replay samples: {stdout}"); + fps + }; + let expected = replay(1); + assert_eq!(expected.last(), Some(&30)); + assert_eq!(replay(1652), expected); +} diff --git a/src/server/video_qos/tests/smoke.rs b/src/server/video_qos/tests/smoke.rs new file mode 100644 index 000000000..86a93a7de --- /dev/null +++ b/src/server/video_qos/tests/smoke.rs @@ -0,0 +1,369 @@ +use super::*; + +pub(super) fn session(fps: u32, quality: Quality) -> VideoQoS { + let mut qos = VideoQoS { + fps: INIT_FPS.min(fps), + abr_config: false, + ..Default::default() + }; + qos.advance_ms(2000); + qos.users.insert( + 1, + UserData { + custom_fps: Some(fps), + quality: Some((0, quality)), + ..Default::default() + }, + ); + qos +} + +fn profiles() -> Vec<(&'static str, Vec, bool)> { + vec![ + ("stable_10", vec![10; 120], false), + ("stable_80", vec![80; 120], false), + ("stable_180", vec![180; 120], false), + ("stable_300", vec![300; 120], false), + ( + "lan_jitter", + (0..120).map(|i| 10 + (i * 37 % 70)).collect(), + true, + ), + ( + "isolated_spikes", + (0..120) + .map(|i| if i % 15 == 0 { 800 } else { 10 }) + .collect(), + true, + ), + ("alternating_10_350", [10, 350].repeat(60), true), + ( + "two_sample_bursts", + (0..120) + .map(|i| if i % 12 < 2 { 700 } else { 10 }) + .collect(), + true, + ), + ( + "threshold_jitter", + [140, 180, 150, 190, 130, 170].repeat(20), + true, + ), + ( + "congestion_200_recovery", + [vec![200; 20], vec![10; 60]].concat(), + true, + ), + ( + "congestion_800_recovery", + [vec![800; 20], vec![10; 60]].concat(), + true, + ), + ( + "congestion_1500_recovery", + [vec![1500; 20], vec![10; 60]].concat(), + true, + ), + ( + "rising_then_falling", + (0..80).map(|i| 10 + i.min(79 - i) * 20).collect(), + true, + ), + ] +} + +#[test] +fn smoke_latency_profiles() { + use std::fmt::Write; + + let mut csv = String::from("profile,limit,quality,sample,delay_ms,fps\n"); + for (quality_name, quality) in [ + ("balanced", Quality::Balanced), + ("best", Quality::Best), + ("low", Quality::Low), + ] { + for limit in [1, 5, 15, 30, 60, 120] { + for (name, delays, warm_up) in profiles() { + let mut qos = session(limit, quality); + if warm_up { + for _ in 0..90 { + qos.user_network_delay(1, 10); + } + assert_eq!(qos.fps(), limit); + } + let mut trace = Vec::new(); + for (i, delay) in delays.into_iter().enumerate() { + qos.user_network_delay(1, delay); + let fps = qos.fps(); + assert!((MIN_FPS..=limit).contains(&fps), "{name}: {fps}"); + trace.push(fps); + writeln!(csv, "{name},{limit},{quality_name},{i},{delay},{fps}").unwrap(); + } + if limit == 30 && quality_name == "balanced" { + println!( + "{name}: first20={:?}, last={}", + &trace[..20], + trace.last().unwrap() + ); + } + if name.starts_with("stable_") { + assert_eq!(trace.last(), Some(&limit), "{name}, {quality_name}"); + } + if matches!( + name, + "lan_jitter" + | "isolated_spikes" + | "alternating_10_350" + | "two_sample_bursts" + | "threshold_jitter" + ) { + assert!( + trace.iter().all(|fps| *fps == limit), + "{name}, {quality_name}" + ); + } + if matches!(name, "congestion_800_recovery" | "congestion_1500_recovery") { + assert!( + trace.iter().all(|fps| *fps >= limit.min(5)), + "automatic reductions must preserve the floor: {trace:?}" + ); + if limit >= 15 { + assert!( + trace[20] < limit, + "a single good reply must not restore the full frame rate" + ); + } + assert_eq!( + trace[21], limit, + "two fresh good replies must restore the frame rate" + ); + if name == "congestion_1500_recovery" { + assert_eq!( + trace[5], + limit.min(5), + "severe congestion must brake promptly" + ); + } + } + if name == "congestion_200_recovery" && limit >= 15 { + assert!( + trace[..20].iter().min() < Some(&limit), + "moderate sustained congestion must reduce the frame rate: {trace:?}" + ); + } + if name.ends_with("_recovery") { + assert_eq!(trace.last(), Some(&limit), "{name}, {quality_name}"); + } + } + } + } + if let Ok(path) = std::env::var("RUSTDESK_QOS_SMOKE_CSV") { + std::fs::write(path, csv).unwrap(); + } +} + +#[test] +fn smoke_bandwidth_drop_and_recovery() { + use std::fmt::Write; + + let mut qos = session(30, Quality::Balanced); + for _ in 0..90 { + qos.user_network_delay(1, 10); + } + // Fixed-size frames, FIFO link and one outstanding TestDelay, with a 10 ms base RTT. + // Encoding and ABR are intentionally absent so that FPS alone controls offered load. + let mut queue = 0.0_f64; + let mut probe: Option<(u32, f64, Option)> = None; + let mut last_delay = 10; + let mut csv = String::from("time_ms,capacity_fps,queue_ms,delay_ms,fps\n"); + let mut max_queue_ms = 0; + let mut drained = false; + let mut recovered_at = None; + for now in (0..120_000).step_by(10) { + qos.advance_ms(10); + let capacity = if (10_000..70_000).contains(&now) { + 15.0 + } else { + 40.0 + }; + queue = (queue + (qos.fps() as f64 - capacity) * 0.01).max(0.0); + if let Some((_, remaining, reply_at)) = probe.as_mut() { + *remaining -= capacity * 0.01; + if *remaining <= 0.0 && reply_at.is_none() { + *reply_at = Some(now + 10); + } + } + if let Some((sent, _, Some(reply_at))) = probe { + if now >= reply_at { + last_delay = now - sent; + qos.user_network_delay(1, last_delay); + probe = None; + } + } + if now % 1000 == 0 { + if probe.is_none() { + probe = Some((now, queue, None)); + } + qos.user_delay_response_elapsed(1, (now - probe.unwrap().0) as u128); + let queue_ms = (queue / capacity * 1000.0) as u32; + max_queue_ms = max_queue_ms.max(queue_ms); + if (20_000..40_000).contains(&now) && queue_ms < 100 { + drained = true; + } + if now >= 70_000 && qos.fps() == 30 && recovered_at.is_none() { + recovered_at = Some(now - 70_000); + } + writeln!( + csv, + "{now},{capacity},{queue_ms},{last_delay},{}", + qos.fps() + ) + .unwrap(); + } + } + println!( + "bandwidth 40 -> 15 -> 40 fps: max_queue_ms={max_queue_ms}, recovery_ms={recovered_at:?}, final_fps={}", + qos.fps() + ); + if let Ok(path) = std::env::var("RUSTDESK_QOS_SMOKE_CSV") { + std::fs::write(std::path::Path::new(&path).with_extension("queue.csv"), csv).unwrap(); + } + assert!(drained, "congestion must drain after the capacity drop"); + assert!( + max_queue_ms < 2500, + "queue must not grow while awaiting confirmation" + ); + assert!( + recovered_at.is_some_and(|ms| ms <= 15_000) && qos.fps() == 30, + "FPS must recover when capacity returns" + ); +} + +#[test] +fn smoke_capacity_with_short_stalls() { + for limit in [30, 60] { + // Probes run at one second cadence; sweep the stall phase so no alignment hides. + for phase in (0..1000).step_by(50) { + let mut qos = session(limit, Quality::Balanced); + for _ in 0..90 { + qos.user_network_delay(1, 10); + } + let capacity = limit as f64 * 2.0; + let mut queue = 0.0_f64; + let mut probe: Option<(u32, f64, Option)> = None; + let mut max_delay = 0; + for now in (0..120_000).step_by(10) { + qos.advance_ms(10); + // A 700 ms pause affects video and probes on the same FIFO link. + let available = if (now + phase) % 6000 < 700 { + 0.0 + } else { + capacity + }; + queue = (queue + (qos.fps() as f64 - available) * 0.01).max(0.0); + if let Some((_, remaining, reply_at)) = probe.as_mut() { + *remaining -= available * 0.01; + if available > 0.0 && *remaining <= 0.0 && reply_at.is_none() { + *reply_at = Some(now + 10); + } + } + if let Some((sent, _, Some(reply_at))) = probe { + if now >= reply_at { + max_delay = max_delay.max(now - sent); + qos.user_network_delay(1, now - sent); + probe = None; + } + } + if now % 1000 == 0 { + if probe.is_none() { + probe = Some((now, queue, None)); + } + qos.user_delay_response_elapsed(1, (now - probe.unwrap().0) as u128); + } + assert_eq!(qos.fps(), limit, "limit={limit}, phase={phase}, time={now}"); + } + println!("healthy FIFO link: limit={limit}, phase={phase}, max_delay_ms={max_delay}, final_fps={}", qos.fps()); + } + } +} + +#[test] +fn smoke_abr_bandwidth_drop_and_recovery() { + for reduced_capacity in [27, 24, 15] { + let mut qos = session(30, Quality::Balanced); + for _ in 0..90 { + qos.user_network_delay(1, 10); + } + qos.abr_config = true; + qos.new_display("test".to_owned()); + qos.set_support_changing_quality("test", true); + qos.store_bitrate(4000); + let initial_ratio = qos.ratio(); + let mut queue = 0.0_f64; + let mut probe: Option<(u32, f64, Option)> = None; + let mut first_ratio_drop = None; + let mut first_fps_drop = None; + let mut first_fps_drop_delay = None; + let mut last_delay = 10; + let mut max_queue_ms = 0; + let mut drained = false; + let mut recovered_at = None; + for now in (0..120_000).step_by(10) { + qos.advance_ms(10); + let capacity = if (10_000..70_000).contains(&now) { + reduced_capacity as f64 + } else { + 40.0 + }; + // Frame size scales with the requested ratio; video and probes share one FIFO. + let frame_size = (qos.ratio() / initial_ratio) as f64; + queue = (queue + (qos.fps() as f64 * frame_size - capacity) * 0.01).max(0.0); + qos.store_bitrate((4000.0 * frame_size) as u32); + if let Some((_, remaining, reply_at)) = probe.as_mut() { + *remaining -= capacity * 0.01; + if *remaining <= 0.0 && reply_at.is_none() { + *reply_at = Some(now + 10); + } + } + if let Some((sent, _, Some(reply_at))) = probe { + if now >= reply_at { + last_delay = now - sent; + qos.user_network_delay(1, last_delay); + probe = None; + } + } + if now % 1000 == 0 { + if probe.is_none() { + probe = Some((now, queue, None)); + } + qos.user_delay_response_elapsed(1, (now - probe.unwrap().0) as u128); + qos.update_display_data("test", qos.fps() as usize); + let queue_ms = (queue / capacity * 1000.0) as u32; + max_queue_ms = max_queue_ms.max(queue_ms); + if (20_000..40_000).contains(&now) && queue_ms < 100 { + drained = true; + } + } + if qos.ratio() < initial_ratio && first_ratio_drop.is_none() { + first_ratio_drop = Some(now); + } + if qos.fps() < 30 && first_fps_drop.is_none() { + first_fps_drop = Some(now); + first_fps_drop_delay = Some(last_delay); + } + if now >= 70_000 && qos.fps() == 30 && recovered_at.is_none() { + recovered_at = Some(now - 70_000); + } + } + println!("ABR bandwidth 40 -> {reduced_capacity} -> 40: max_queue_ms={max_queue_ms}, first_ratio_drop_ms={first_ratio_drop:?}, first_fps_drop_ms={first_fps_drop:?}, first_fps_drop_delay_ms={first_fps_drop_delay:?}, recovery_ms={recovered_at:?}, final_fps={}, final_ratio={:.3}", qos.fps(), qos.ratio()); + assert!(drained && max_queue_ms < 2500); + assert!(recovered_at.is_some_and(|ms| ms <= 15_000)); + assert_eq!(qos.fps(), 30); + assert!(qos.ratio() >= initial_ratio * 0.8); + if reduced_capacity >= 24 { + assert!(first_ratio_drop.is_some_and(|ratio_time| { + first_fps_drop.map_or(true, |fps_time| ratio_time < fps_time) + })); + } + } +} diff --git a/src/server/video_qos/tests/startup.rs b/src/server/video_qos/tests/startup.rs new file mode 100644 index 000000000..6efea7d4d --- /dev/null +++ b/src/server/video_qos/tests/startup.rs @@ -0,0 +1,234 @@ +use super::*; + +fn session(cap: u32, abr: bool) -> VideoQoS { + let mut qos = super::smoke::session(cap, Quality::Balanced); + let joined = qos.now(); + qos.users.get_mut(&1).unwrap().joined_at = Some(joined); + qos.abr_config = abr; + qos.new_display("startup".to_owned()); + qos.set_support_changing_quality("startup", true); + qos +} + +fn reply(qos: &mut VideoQoS, delay: u32) { + qos.user_network_delay(1, delay); + let ratio = qos.ratio(); + qos.store_bitrate((6000.0 * ratio) as u32); + qos.update_display_data("startup", qos.fps() as usize); + qos.advance_ms(1000); +} + +#[test] +fn clean_startup_reaches_the_cap_without_waiting_for_slow_growth() { + println!("| cap | base RTT ms | ABR | replies to cap | FPS after each reply |"); + println!("|---|---|---|---|---|"); + for (cap, budget) in [(3, 1), (15, 1), (30, 2), (60, 4), (120, 6)] { + for base in [10, 150, 300, 600] { + for abr in [false, true] { + let mut qos = session(cap, abr); + qos.advance_ms(base as u64); + let mut trace = Vec::new(); + for n in 1..=budget { + reply(&mut qos, base); + let fps = qos.fps(); + assert!(fps <= cap); + if n == 1 { + assert!(fps <= INIT_FPS.min(cap), "keep the first-second guard"); + } + trace.push(fps); + } + println!("| {cap} | {base} | {abr} | {budget} | {trace:?} |"); + assert_eq!(qos.fps(), cap, "base={base} ABR={abr}: {trace:?}"); + assert_eq!(qos.ratio(), Quality::Balanced.ratio()); + } + } + } +} + +#[test] +fn an_unclean_startup_reply_disables_faster_growth() { + for excess in [50, 149, 150, 400] { + let mut qos = session(120, false); + reply(&mut qos, 10); + reply(&mut qos, 10 + excess); + for _ in 0..15 { + let before = qos.fps(); + reply(&mut qos, 10); + assert!( + qos.fps() <= before + (before / 5).max(6), + "excess={excess}: startup acceleration restarted: {before} -> {}", + qos.fps() + ); + } + } +} + +#[test] +fn a_failed_startup_probe_rolls_back_and_does_not_restart() { + let mut qos = session(120, false); + for _ in 0..2 { + reply(&mut qos, 10); + } + let probe = qos.fps(); + assert!(probe >= 30, "fixture must reach the accelerated level"); + reply(&mut qos, 1200); + assert!(qos.fps() <= probe / 2, "rollback must remain prompt"); + for _ in 0..2 { + reply(&mut qos, 10); + } + assert_eq!(qos.fps(), probe, "keep the existing two-reply recovery"); + for _ in 0..8 { + let before = qos.fps(); + reply(&mut qos, 10); + assert!(qos.fps() <= before + (before / 5).max(6)); + } +} + +#[test] +fn a_startup_spike_still_requires_congestion_confirmation() { + let mut qos = session(120, false); + for _ in 0..2 { + reply(&mut qos, 10); + } + let probe = qos.fps(); + for _ in 0..2 { + reply(&mut qos, 800); + assert_eq!(qos.fps(), probe, "startup is not a failed fast restore"); + } + reply(&mut qos, 800); + assert!(qos.fps() < probe, "three fresh bad replies must reduce FPS"); + assert!(qos.fps() >= probe - probe / 5); +} + +#[test] +fn a_startup_timeout_keeps_legacy_recovery() { + let mut qos = session(120, false); + qos.advance_ms(3000); + qos.user_delay_response_elapsed(1, 3001); + assert_eq!(qos.fps(), 5); + reply(&mut qos, 3100); + assert_eq!(qos.fps(), 5, "late reply must not undo the brake"); + for _ in 0..2 { + reply(&mut qos, 10); + } + assert_eq!(qos.fps(), INIT_FPS); + for _ in 0..8 { + let before = qos.fps(); + reply(&mut qos, 10); + assert!(qos.fps() <= before + (before / 5).max(6)); + } +} + +#[test] +fn raising_the_cap_does_not_restart_startup_acceleration() { + let mut qos = session(30, false); + for _ in 0..6 { + reply(&mut qos, 10); + } + assert_eq!(qos.fps(), 30); + qos.user_custom_fps(1, 120); + for _ in 0..6 { + let before = qos.fps(); + reply(&mut qos, 10); + assert!(qos.fps() <= before + (before / 5).max(6)); + } +} + +#[test] +fn startup_remains_per_viewer_and_preserves_the_join_guard() { + let mut qos = session(120, false); + for _ in 0..20 { + reply(&mut qos, 10); + } + qos.users.insert( + 2, + UserData { + joined_at: Some(qos.now()), + custom_fps: Some(30), + ..Default::default() + }, + ); + qos.user_network_delay(2, 10); + assert_eq!(qos.fps(), INIT_FPS); + assert_eq!(qos.users[&1].delay.fps, Some(120)); + qos.advance_ms(1000); + qos.user_network_delay(2, 10); + assert_eq!(qos.users[&2].delay.fps, Some(30)); + assert_eq!(qos.fps(), 30); + qos.on_connection_close(2); + assert_eq!(qos.fps(), 120); +} + +#[test] +fn closed_loop_startup_preserves_quality_and_bounds_queueing() { + use super::sim::{self, EncoderModel}; + for cap in [30, 60, 120] { + for base in [10, 150, 300, 600] { + for encoder in [EncoderModel::Cbr, EncoderModel::FixedRate] { + for seed in 1..=5 { + let mut sc = sim::scenarios() + .into_iter() + .find(|sc| sc.name == "city_relay_30") + .unwrap(); + sc.seconds = 30; + sc.limit = cap; + sc.link.base_rtt_ms = base as f64; + sc.encoder = encoder; + sc.seed = seed; + let report = sim::run(&sc); + let budget = match cap { + 30 => 2000, + 60 => 4000, + _ => 6000, + } + base; + assert!( + report.time_to_90pct_ms.is_some_and(|t| t <= budget), + "cap={cap} base={base}: {report:?}" + ); + assert_eq!(report.final_fps, cap); + assert_eq!(report.final_ratio, sc.quality.ratio()); + assert!(report.trace.iter().all(|(_, _, queue, _)| *queue < 150)); + assert!(report.frame_age_p95_ms < 150); + } + } + } + } +} + +#[test] +fn constrained_startup_does_not_leave_a_large_queue() { + use super::sim::{self, EncoderModel}; + for cap in [30, 60, 120] { + for encoder in [EncoderModel::Cbr, EncoderModel::FixedRate] { + for seed in sim::SEEDS { + let mut sc = sim::scenarios() + .into_iter() + .find(|sc| sc.name == "bandwidth_halved_30") + .unwrap(); + sc.limit = cap; + sc.encoder = encoder; + sc.seed = seed; + sc.link.capacity_kbps = vec![(0, 2500.0)]; + let report = sim::run(&sc); + let startup_queue = report + .trace + .iter() + .filter(|(t, ..)| *t < 15_000) + .map(|(_, _, queue, _)| *queue) + .max() + .unwrap(); + println!("startup limited cap={cap} seed={seed} frame_budget={} peak_queue_ms={startup_queue} queue_p95_ms={} age_p95_ms={}", encoder == EncoderModel::FixedRate, report.queue_p95_ms, report.frame_age_p95_ms); + assert!( + startup_queue < 4000, + "cap={cap} seed={seed}: {startup_queue}" + ); + assert!(report.queue_p95_ms < 3000); + assert!(report.frame_age_p95_ms < 3000); + assert!(report + .trace + .iter() + .all(|(_, fps, _, _)| (5..=cap).contains(fps))); + } + } + } +} diff --git a/src/server/video_service.rs b/src/server/video_service.rs index e91ecc7d4..195441b38 100644 --- a/src/server/video_service.rs +++ b/src/server/video_service.rs @@ -655,6 +655,12 @@ fn run(vs: VideoService) -> ResultType<()> { let capture_width = c.width; let capture_height = c.height; let (mut second_instant, mut send_counter) = (Instant::now(), 0); + // Diagnostics only. `send_counter` counts capture rounds, which is not the + // number of frames that reached a connection: the encoder's own rate control + // drops frames when the bitrate cannot carry them. `wait_max_ms` is how long + // a round waited for the previous frame to be picked up, so a blocked write + // shows up here as capture stalling rather than as a slow network. + let (mut sent_counter, mut wait_max_ms) = (0usize, 0u32); while sp.ok() { #[cfg(windows)] @@ -665,6 +671,8 @@ fn run(vs: VideoService) -> ResultType<()> { &mut spf, client_record, &mut send_counter, + &mut sent_counter, + &mut wait_max_ms, &mut second_instant, &sp.name(), )?; @@ -785,6 +793,9 @@ fn run(vs: VideoService) -> ResultType<()> { capture_width, capture_height, )?; + if !send_conn_ids.is_empty() { + sent_counter += 1; + } frame_controller.set_send(now, send_conn_ids); send_counter += 1; } @@ -844,6 +855,9 @@ fn run(vs: VideoService) -> ResultType<()> { capture_width, capture_height, )?; + if !send_conn_ids.is_empty() { + sent_counter += 1; + } frame_controller.set_send(now, send_conn_ids); send_counter += 1; } @@ -885,6 +899,7 @@ fn run(vs: VideoService) -> ResultType<()> { break; } } + wait_max_ms = wait_max_ms.max(wait_begin.elapsed().as_millis() as u32); DISPLAY_CONN_IDS.lock().unwrap().remove(&display_idx); let elapsed = now.elapsed(); @@ -1315,12 +1330,22 @@ pub fn make_display_changed_msg( Some(msg_out) } +/// Per-second pipeline diagnostics, off unless `RUSTDESK_QOS_VERBOSE` is set. +/// The default log level is `debug`, so an unconditional line here would land in +/// every user's log file once a second forever. Nothing enables it implicitly. +pub(crate) fn qos_diag_verbose() -> bool { + static VERBOSE: std::sync::OnceLock = std::sync::OnceLock::new(); + *VERBOSE.get_or_init(|| std::env::var("RUSTDESK_QOS_VERBOSE").is_ok()) +} + fn check_qos( encoder: &mut Encoder, ratio: &mut f32, spf: &mut Duration, client_record: bool, send_counter: &mut usize, + sent_counter: &mut usize, + wait_max_ms: &mut u32, second_instant: &mut Instant, name: &str, ) -> ResultType<()> { @@ -1346,7 +1371,21 @@ fn check_qos( if second_instant.elapsed() > Duration::from_secs(1) { *second_instant = Instant::now(); video_qos.update_display_data(&name, *send_counter); + // Diagnostics only, joined with `qos_trace` on `t`: the controller's target + // is not the rate the encoder produced, and neither is the rate the send + // path accepted. + if qos_diag_verbose() { + log::debug!( + "qos_video t={} display={name} captured={} sent={} wait_max={}", + hbb_common::get_time(), + *send_counter, + *sent_counter, + *wait_max_ms + ); + } *send_counter = 0; + *sent_counter = 0; + *wait_max_ms = 0; } drop(video_qos); Ok(())