diff --git a/crates/rendering/src/layers/captions.rs b/crates/rendering/src/layers/captions.rs index 3240e93606d..98898806062 100644 --- a/crates/rendering/src/layers/captions.rs +++ b/crates/rendering/src/layers/captions.rs @@ -1,3 +1,6 @@ +use std::borrow::Cow; +use std::ops::Range; + use bytemuck::{Pod, Zeroable}; use cap_project::{CaptionWord, XY}; use glyphon::cosmic_text::LayoutRunIter; @@ -261,6 +264,83 @@ fn calculate_word_highlight( 0.0 } +fn caption_rgba(rgb: [f32; 3], alpha: f32) -> Color { + Color::rgba( + (rgb[0] * 255.0) as u8, + (rgb[1] * 255.0) as u8, + (rgb[2] * 255.0) as u8, + (alpha * 255.0) as u8, + ) +} + +struct WordHighlight { + base_rgb: [f32; 3], + highlight_rgb: [f32; 3], + base_alpha: f32, + highlight_alpha: f32, + transition_duration: f32, +} + +fn word_highlight_spans( + text: &str, + words: &[CaptionWord], + uppercase: bool, + current_time: f32, + highlight: &WordHighlight, +) -> Vec<(Range, Color)> { + let base = caption_rgba(highlight.base_rgb, highlight.base_alpha); + let mut spans = Vec::with_capacity(words.len() * 2 + 1); + let mut last_end = 0usize; + + for (idx, word) in words.iter().enumerate() { + let needle = if uppercase { + Cow::Owned(word.text.to_uppercase()) + } else { + Cow::Borrowed(word.text.as_str()) + }; + let Some(start_pos) = text.get(last_end..).and_then(|s| s.find(needle.as_ref())) else { + continue; + }; + let abs_start = last_end + start_pos; + if abs_start > last_end { + spans.push((last_end..abs_start, base)); + } + + let word_highlight = calculate_word_highlight( + current_time, + word, + idx, + words, + highlight.transition_duration, + ); + let blended_rgb: [f32; 3] = std::array::from_fn(|channel| { + highlight.base_rgb[channel] + + (highlight.highlight_rgb[channel] - highlight.base_rgb[channel]) * word_highlight + }); + let blended_alpha = highlight.base_alpha + + (highlight.highlight_alpha - highlight.base_alpha) * word_highlight; + + let word_end = abs_start + needle.len(); + spans.push(( + abs_start..word_end, + caption_rgba(blended_rgb, blended_alpha), + )); + last_end = word_end; + } + + if last_end < text.len() { + spans.push((last_end..text.len(), base)); + } + + spans +} + +#[derive(PartialEq)] +enum CaptionTextColors { + Uniform(Color), + Spans(Vec<(Range, Color)>), +} + #[derive(PartialEq)] struct ShapedCaptionText { text: String, @@ -268,7 +348,56 @@ struct ShapedCaptionText { wrap_width: u32, family: Family<'static>, weight: Weight, - color: Color, + colors: CaptionTextColors, +} + +fn shape_caption_text(font_system: &mut FontSystem, shaped: &ShapedCaptionText) -> Buffer { + let font_size = f32::from_bits(shaped.font_size); + let mut buffer = Buffer::new(font_system, Metrics::new(font_size, font_size * 1.2)); + buffer.set_size(font_system, Some(f32::from_bits(shaped.wrap_width)), None); + buffer.set_wrap(font_system, glyphon::Wrap::None); + let attrs = Attrs::new().family(shaped.family).weight(shaped.weight); + match &shaped.colors { + CaptionTextColors::Uniform(color) => buffer.set_text( + font_system, + &shaped.text, + &attrs.color(*color), + Shaping::Advanced, + ), + CaptionTextColors::Spans(spans) => buffer.set_rich_text( + font_system, + spans + .iter() + .map(|(range, color)| (&shaped.text[range.clone()], attrs.clone().color(*color))), + &attrs, + Shaping::Advanced, + None, + ), + } + buffer +} + +struct CaptionText { + buffer: Buffer, + shaped: Option, +} + +impl CaptionText { + fn new() -> Self { + Self { + buffer: Buffer::new_empty(Metrics::new(24.0, 24.0 * 1.2)), + shaped: None, + } + } + + fn update(&mut self, font_system: &mut FontSystem, shaped: ShapedCaptionText) -> bool { + if self.shaped.as_ref() == Some(&shaped) { + return false; + } + self.buffer = shape_caption_text(font_system, &shaped); + self.shaped = Some(shaped); + true + } } pub struct CaptionsLayer { @@ -277,8 +406,7 @@ pub struct CaptionsLayer { swash_cache: SwashCache, text_atlas: TextAtlas, text_renderer: TextRenderer, - text_buffer: Buffer, - shaped: Option, + text: CaptionText, current_text: Option, current_segment_start: f32, current_segment_end: f32, @@ -319,9 +447,6 @@ impl CaptionsLayer { ); glyph_phase.finish("returned"); - let metrics = Metrics::new(24.0, 24.0 * 1.2); - let text_buffer = Buffer::new_empty(metrics); - let background_uniforms = CaptionBackgroundUniforms { rect: [0.0; 4], color: [0.0; 4], @@ -422,8 +547,7 @@ impl CaptionsLayer { swash_cache, text_atlas, text_renderer, - text_buffer, - shaped: None, + text: CaptionText::new(), current_text: None, current_segment_start: 0.0, current_segment_end: 0.0, @@ -618,13 +742,6 @@ impl CaptionsLayer { let font_size = caption_data.settings.size as f32 * (height as f32 / 1080.0); let wrap_width = (width as f32 - margin * 2.0).max(font_size); - let new_buffer = |font_system: &mut FontSystem| { - let mut buffer = Buffer::new(font_system, Metrics::new(font_size, font_size * 1.2)); - buffer.set_size(font_system, Some(wrap_width), None); - buffer.set_wrap(font_system, glyphon::Wrap::None); - buffer - }; - let font_family = match caption_data.settings.font.as_str() { "System Serif" => Family::Serif, "System Monospace" => Family::Monospace, @@ -642,127 +759,39 @@ impl CaptionsLayer { let base_alpha = (fade_opacity * BASE_TEXT_OPACITY).clamp(0.0, 1.0); let highlight_alpha = fade_opacity.clamp(0.0, 1.0); - if use_color_highlight { - let mut rich_text: Vec<(&str, Attrs)> = Vec::new(); - let full_text = caption_text.as_str(); - let mut last_end = 0usize; - - for (idx, word) in caption_words.iter().enumerate() { - let needle = if uppercase { - word.text.to_uppercase() - } else { - word.text.clone() - }; - if let Some(start_pos) = full_text.get(last_end..).and_then(|s| s.find(&needle)) { - let abs_start = last_end + start_pos; - - if abs_start > last_end { - let space = &full_text[last_end..abs_start]; - rich_text.push(( - space, - Attrs::new() - .family(font_family) - .weight(weight) - .color(Color::rgba( - (base_color[0] * 255.0) as u8, - (base_color[1] * 255.0) as u8, - (base_color[2] * 255.0) as u8, - (base_alpha * 255.0) as u8, - )), - )); - } - - let word_highlight = calculate_word_highlight( - current_time as f32, - word, - idx, - caption_words, - word_transition_duration, - ); - - let blended_color = [ - base_color[0] + (highlight_color_rgb[0] - base_color[0]) * word_highlight, - base_color[1] + (highlight_color_rgb[1] - base_color[1]) * word_highlight, - base_color[2] + (highlight_color_rgb[2] - base_color[2]) * word_highlight, - ]; - - let blended_alpha = - base_alpha + (highlight_alpha - base_alpha) * word_highlight; - - let word_end = abs_start + needle.len(); - rich_text.push(( - &full_text[abs_start..word_end], - Attrs::new() - .family(font_family) - .weight(weight) - .color(Color::rgba( - (blended_color[0] * 255.0) as u8, - (blended_color[1] * 255.0) as u8, - (blended_color[2] * 255.0) as u8, - (blended_alpha * 255.0) as u8, - )), - )); - last_end = word_end; - } - } - - if last_end < full_text.len() { - rich_text.push(( - &full_text[last_end..], - Attrs::new() - .family(font_family) - .weight(weight) - .color(Color::rgba( - (base_color[0] * 255.0) as u8, - (base_color[1] * 255.0) as u8, - (base_color[2] * 255.0) as u8, - (base_alpha * 255.0) as u8, - )), - )); - } - - let mut buffer = new_buffer(&mut self.font_system); - buffer.set_rich_text( - &mut self.font_system, - rich_text, - &Attrs::new().family(font_family).weight(weight), - Shaping::Advanced, - None, - ); - self.text_buffer = buffer; - self.shaped = None; + let colors = if use_color_highlight { + CaptionTextColors::Spans(word_highlight_spans( + &caption_text, + caption_words, + uppercase, + current_time as f32, + &WordHighlight { + base_rgb: base_color, + highlight_rgb: highlight_color_rgb, + base_alpha, + highlight_alpha, + transition_duration: word_transition_duration, + }, + )) } else { - let color = Color::rgba( - (base_color[0] * 255.0) as u8, - (base_color[1] * 255.0) as u8, - (base_color[2] * 255.0) as u8, - (highlight_alpha * 255.0) as u8, - ); - let shaped = ShapedCaptionText { + CaptionTextColors::Uniform(caption_rgba(base_color, highlight_alpha)) + }; + self.text.update( + &mut self.font_system, + ShapedCaptionText { text: caption_text, font_size: font_size.to_bits(), wrap_width: wrap_width.to_bits(), family: font_family, weight, - color, - }; - if self.shaped.as_ref() != Some(&shaped) { - let mut buffer = new_buffer(&mut self.font_system); - buffer.set_text( - &mut self.font_system, - &shaped.text, - &Attrs::new().family(font_family).weight(weight).color(color), - Shaping::Advanced, - ); - self.text_buffer = buffer; - self.shaped = Some(shaped); - } - } + colors, + }, + ); let mut layout_width: f32 = 0.0; let mut layout_height: f32 = 0.0; let mut highlight_extent: Option<(f32, f32, f32, f32)> = None; - for run in LayoutRunIter::new(&self.text_buffer) { + for run in LayoutRunIter::new(&self.text.buffer) { layout_width = layout_width.max(run.line_w); layout_height = layout_height.max(run.line_top + run.line_height); @@ -902,7 +931,7 @@ impl CaptionsLayer { for (offset_x, offset_y) in outline_offsets.iter() { text_areas.push(TextArea { - buffer: &self.text_buffer, + buffer: &self.text.buffer, left: text_left + offset_x, top: text_top + offset_y, scale: render_scale, @@ -921,7 +950,7 @@ impl CaptionsLayer { ); text_areas.push(TextArea { - buffer: &self.text_buffer, + buffer: &self.text.buffer, left: text_left, top: text_top, scale: render_scale, @@ -1288,6 +1317,379 @@ mod tests { } } +#[cfg(test)] +mod shaping_tests { + use cap_project::CaptionWord; + use glyphon::{Attrs, Buffer, Color, Family, FontSystem, Metrics, Shaping, Weight}; + + use super::{ + BASE_TEXT_OPACITY, CaptionText, CaptionTextColors, ShapedCaptionText, WordHighlight, + calculate_caption_fade, calculate_word_highlight, caption_rgba, word_highlight_spans, + }; + + const FPS: f32 = 60.0; + const FADE: f64 = 0.15; + + fn spoken_words(sentence: &str, start: f32) -> Vec { + sentence + .split_whitespace() + .enumerate() + .map(|(index, text)| CaptionWord { + text: text.to_string(), + start: start + index as f32 * 0.3, + end: start + index as f32 * 0.3 + 0.25, + }) + .collect() + } + + struct Frame<'a> { + text: String, + words: &'a [CaptionWord], + uppercase: bool, + current_time: f32, + highlight: WordHighlight, + font_size: f32, + wrap_width: f32, + family: Family<'static>, + weight: Weight, + } + + impl Frame<'_> { + fn key(&self) -> ShapedCaptionText { + ShapedCaptionText { + text: self.text.clone(), + font_size: self.font_size.to_bits(), + wrap_width: self.wrap_width.to_bits(), + family: self.family, + weight: self.weight, + colors: CaptionTextColors::Spans(word_highlight_spans( + &self.text, + self.words, + self.uppercase, + self.current_time, + &self.highlight, + )), + } + } + + fn uncached_buffer(&self, font_system: &mut FontSystem) -> Buffer { + let WordHighlight { + base_rgb: base_color, + highlight_rgb: highlight_color_rgb, + base_alpha, + highlight_alpha, + transition_duration: word_transition_duration, + } = self.highlight; + let (font_family, weight, font_size) = (self.family, self.weight, self.font_size); + let mut rich_text: Vec<(&str, Attrs)> = Vec::new(); + let full_text = self.text.as_str(); + let mut last_end = 0usize; + + for (idx, word) in self.words.iter().enumerate() { + let needle = if self.uppercase { + word.text.to_uppercase() + } else { + word.text.clone() + }; + if let Some(start_pos) = full_text.get(last_end..).and_then(|s| s.find(&needle)) { + let abs_start = last_end + start_pos; + + if abs_start > last_end { + let space = &full_text[last_end..abs_start]; + rich_text.push(( + space, + Attrs::new() + .family(font_family) + .weight(weight) + .color(Color::rgba( + (base_color[0] * 255.0) as u8, + (base_color[1] * 255.0) as u8, + (base_color[2] * 255.0) as u8, + (base_alpha * 255.0) as u8, + )), + )); + } + + let word_highlight = calculate_word_highlight( + self.current_time, + word, + idx, + self.words, + word_transition_duration, + ); + + let blended_color = [ + base_color[0] + (highlight_color_rgb[0] - base_color[0]) * word_highlight, + base_color[1] + (highlight_color_rgb[1] - base_color[1]) * word_highlight, + base_color[2] + (highlight_color_rgb[2] - base_color[2]) * word_highlight, + ]; + + let blended_alpha = + base_alpha + (highlight_alpha - base_alpha) * word_highlight; + + let word_end = abs_start + needle.len(); + rich_text.push(( + &full_text[abs_start..word_end], + Attrs::new() + .family(font_family) + .weight(weight) + .color(Color::rgba( + (blended_color[0] * 255.0) as u8, + (blended_color[1] * 255.0) as u8, + (blended_color[2] * 255.0) as u8, + (blended_alpha * 255.0) as u8, + )), + )); + last_end = word_end; + } + } + + if last_end < full_text.len() { + rich_text.push(( + &full_text[last_end..], + Attrs::new() + .family(font_family) + .weight(weight) + .color(Color::rgba( + (base_color[0] * 255.0) as u8, + (base_color[1] * 255.0) as u8, + (base_color[2] * 255.0) as u8, + (base_alpha * 255.0) as u8, + )), + )); + } + + let mut buffer = Buffer::new(font_system, Metrics::new(font_size, font_size * 1.2)); + buffer.set_size(font_system, Some(self.wrap_width), None); + buffer.set_wrap(font_system, glyphon::Wrap::None); + buffer.set_rich_text( + font_system, + rich_text, + &Attrs::new().family(font_family).weight(weight), + Shaping::Advanced, + None, + ); + buffer + } + } + + fn glyphs(buffer: &Buffer) -> Vec { + buffer + .layout_runs() + .flat_map(|run| { + std::iter::once(format!( + "line {} y {} top {} height {} width {}", + run.line_i, run.line_y, run.line_top, run.line_height, run.line_w + )) + .chain(run.glyphs.iter().map(|glyph| format!("{glyph:?}"))) + }) + .collect() + } + + fn highlight( + highlight_rgb: [f32; 3], + transition_duration: f32, + fade_opacity: f32, + ) -> WordHighlight { + WordHighlight { + base_rgb: [1.0, 1.0, 1.0], + highlight_rgb, + base_alpha: (fade_opacity * BASE_TEXT_OPACITY).clamp(0.0, 1.0), + highlight_alpha: fade_opacity.clamp(0.0, 1.0), + transition_duration, + } + } + + #[test] + fn cached_word_highlight_layout_matches_an_uncached_shape_every_frame() { + let mut cached_fonts = super::super::new_font_system(); + let mut uncached_fonts = super::super::new_font_system(); + let sentence = "Ship it today, then share the link with Straße ʼn everyone"; + let words = spoken_words(sentence, 0.3); + let start = f64::from(words[0].start); + let end = f64::from(words.last().unwrap().end); + let variants = [ + (false, [1.0, 1.0, 1.0], 0.25), + (true, [1.0, 0.843, 0.0], 0.1), + (false, [0.2, 0.6, 1.0], 0.0), + ]; + + for (uppercase, highlight_rgb, transition_duration) in variants { + let text = if uppercase { + sentence.to_uppercase() + } else { + sentence.to_string() + }; + let mut cache = CaptionText::new(); + let (mut reshapes, mut visible_frames) = (0, 0); + let mut distinct_layouts = Vec::new(); + for frame in 0..((end + FADE + 0.2) * f64::from(FPS)) as u32 { + let current_time = f64::from(frame) / f64::from(FPS); + let fade_opacity = calculate_caption_fade(current_time, start, end, FADE); + if fade_opacity <= 0.0 { + continue; + } + visible_frames += 1; + let width = [1920.0, 1280.0, 640.0][(frame / 45) as usize % 3]; + let font_size = 48.0; + let inputs = Frame { + text: text.clone(), + words: &words, + uppercase, + current_time: current_time as f32, + highlight: highlight(highlight_rgb, transition_duration, fade_opacity), + font_size, + wrap_width: (width - width * 0.1_f32).max(font_size), + family: Family::SansSerif, + weight: Weight::BOLD, + }; + + reshapes += usize::from(cache.update(&mut cached_fonts, inputs.key())); + let expected = glyphs(&inputs.uncached_buffer(&mut uncached_fonts)); + assert_eq!( + glyphs(&cache.buffer), + expected, + "uppercase {uppercase}, transition {transition_duration}, time {current_time}" + ); + if !distinct_layouts.contains(&expected) { + distinct_layouts.push(expected); + } + } + + assert!(distinct_layouts.len() > 2); + assert!(reshapes >= distinct_layouts.len()); + assert!( + reshapes < visible_frames, + "{reshapes} reshapes for {visible_frames} frames" + ); + } + } + + #[test] + fn word_highlight_cache_reshapes_when_any_layout_input_changes() { + let mut cached_fonts = super::super::new_font_system(); + let mut uncached_fonts = super::super::new_font_system(); + let sentence = "Ship it today then share the link"; + let words = spoken_words(sentence, 0.0); + let other_words = spoken_words("Ship it now then share the link", 0.0); + let base = || Frame { + text: sentence.to_string(), + words: &words, + uppercase: false, + current_time: 0.35, + highlight: highlight([1.0, 0.843, 0.0], 0.25, 1.0), + font_size: 48.0, + wrap_width: 1728.0, + family: Family::SansSerif, + weight: Weight::BOLD, + }; + let mutations = [ + ( + "text", + Frame { + text: "Ship it now then share the link".to_string(), + words: &other_words, + ..base() + }, + ), + ( + "uppercase", + Frame { + text: sentence.to_uppercase(), + uppercase: true, + ..base() + }, + ), + ( + "font size", + Frame { + font_size: 60.0, + ..base() + }, + ), + ( + "wrap width", + Frame { + wrap_width: 1152.0, + ..base() + }, + ), + ( + "font family", + Frame { + family: Family::Monospace, + ..base() + }, + ), + ( + "font weight", + Frame { + weight: Weight::NORMAL, + ..base() + }, + ), + ( + "highlight color", + Frame { + highlight: highlight([0.0, 1.0, 0.0], 0.25, 1.0), + ..base() + }, + ), + ( + "transition duration", + Frame { + highlight: highlight([1.0, 0.843, 0.0], 0.5, 1.0), + ..base() + }, + ), + ( + "active word", + Frame { + current_time: 0.65, + ..base() + }, + ), + ( + "caption fade", + Frame { + highlight: highlight([1.0, 0.843, 0.0], 0.25, 0.5), + ..base() + }, + ), + ]; + + for (name, changed) in mutations { + let mut cache = CaptionText::new(); + assert!(cache.update(&mut cached_fonts, base().key())); + assert!( + !cache.update(&mut cached_fonts, base().key()), + "{name}: identical inputs reshaped" + ); + assert!( + cache.update(&mut cached_fonts, changed.key()), + "{name}: the cached layout was reused" + ); + assert_eq!( + glyphs(&cache.buffer), + glyphs(&changed.uncached_buffer(&mut uncached_fonts)), + "{name}" + ); + } + + let mut cache = CaptionText::new(); + let plain = |color: Color| ShapedCaptionText { + colors: CaptionTextColors::Uniform(color), + ..base().key() + }; + let white = caption_rgba([1.0, 1.0, 1.0], 1.0); + assert!(cache.update(&mut cached_fonts, plain(white))); + assert!(!cache.update(&mut cached_fonts, plain(white))); + assert!(cache.update(&mut cached_fonts, base().key())); + assert!(cache.update(&mut cached_fonts, plain(white))); + assert!(cache.update(&mut cached_fonts, plain(caption_rgba([1.0, 1.0, 1.0], 0.5)))); + } +} + #[cfg(test)] mod gpu_tests { use cap_project::{ @@ -1521,6 +1923,21 @@ mod gpu_tests { }, CaptionSettings { active_word_highlight: true, + ..enabled.clone() + }, + CaptionSettings { + active_word_highlight: true, + highlight_color: "#FFD700".to_string(), + outline: true, + uppercase: true, + animation: "pop".to_string(), + ..enabled.clone() + }, + CaptionSettings { + active_word_highlight: true, + highlight_color: "#3399FF".to_string(), + word_transition_duration: 0.0, + fade_duration: 0.4, ..enabled }, ]; @@ -1632,6 +2049,81 @@ mod gpu_tests { plain.clone(), base_size, ), + ( + "highlighted font size", + color_highlight.clone(), + CaptionSettings { + size: 80, + ..color_highlight.clone() + }, + base_size, + ), + ( + "highlighted font family", + color_highlight.clone(), + CaptionSettings { + font: "System Monospace".to_string(), + ..color_highlight.clone() + }, + base_size, + ), + ( + "highlighted font weight", + color_highlight.clone(), + CaptionSettings { + font_weight: 400, + ..color_highlight.clone() + }, + base_size, + ), + ( + "highlighted uppercase", + color_highlight.clone(), + CaptionSettings { + uppercase: true, + ..color_highlight.clone() + }, + base_size, + ), + ( + "highlight color", + color_highlight.clone(), + CaptionSettings { + highlight_color: "#00FF00".to_string(), + ..color_highlight.clone() + }, + base_size, + ), + ( + "base color under highlight", + color_highlight.clone(), + CaptionSettings { + color: "#00FFFF".to_string(), + ..color_highlight.clone() + }, + base_size, + ), + ( + "word transition", + color_highlight.clone(), + CaptionSettings { + word_transition_duration: 0.0, + ..color_highlight.clone() + }, + base_size, + ), + ( + "highlighted output height", + color_highlight.clone(), + color_highlight.clone(), + XY::new(512, 216), + ), + ( + "highlighted output width", + color_highlight.clone(), + color_highlight.clone(), + XY::new(384, 288), + ), ]; for (name, before, after, size) in cases { @@ -1669,6 +2161,42 @@ mod gpu_tests { ); } + #[test] + #[ignore = "requires a working wgpu adapter"] + fn highlighted_frames_follow_output_size_changes_like_a_fresh_layer() { + let sizes = [XY::new(512, 288), XY::new(384, 288), XY::new(512, 216)]; + let constants = constants(sizes[0]); + let segments = spoken_segments(&["Hello there", "Ship it today"]); + let settings = CaptionSettings { + enabled: true, + size: 60, + active_word_highlight: true, + highlight_color: "#FFD700".to_string(), + ..Default::default() + }; + let last_frame = ((segments.last().unwrap().end + 0.3) * f64::from(FPS)) as u32; + let mut reused = CaptionsLayer::new(&constants.device, &constants.queue); + let mut drawn_frames = 0; + for frame in 0..last_frame { + let size = sizes[(frame / 7) as usize % sizes.len()]; + let decoded = frames(size); + let mut uniforms = uniforms(&constants, settings.clone(), &segments, size); + uniforms.frame_number = frame; + reused.prepare(&uniforms, &decoded, size, &constants); + let mut fresh = CaptionsLayer::new(&constants.device, &constants.queue); + fresh.prepare(&uniforms, &decoded, size, &constants); + assert_eq!(reused.has_caption, fresh.has_caption, "frame {frame}"); + let pixels = render_pixels(&constants, &reused, size); + assert_eq!( + pixels, + render_pixels(&constants, &fresh, size), + "frame {frame}" + ); + drawn_frames += usize::from(pixels.chunks_exact(4).any(|pixel| pixel[3] != 0)); + } + assert!(drawn_frames > 60); + } + #[test] #[ignore = "benchmark; requires a working wgpu adapter"] fn benchmark_captions_layer_prepare() { @@ -1737,4 +2265,105 @@ mod gpu_tests { ); } } + + fn pixel_digest(digest: u64, pixels: &[u8]) -> u64 { + pixels.chunks_exact(8).fold(digest, |hash, chunk| { + (hash ^ u64::from_le_bytes(chunk.try_into().unwrap())).wrapping_mul(0x0100_0000_01b3) + }) + } + + #[test] + #[ignore = "benchmark; requires a working wgpu adapter"] + fn benchmark_word_highlight_prepare() { + const FRAMES: u32 = 600; + const PASSES: usize = 5; + let size = XY::new(1920, 1080); + let constants = constants(size); + let frames = frames(size); + let spaced = spoken_segments(&[ + "Today I want to walk you through the new export flow", + "First pick a preset and the resolution you need", + "Then trim the start and end so the video gets to the point", + "When you are happy with it press export and share the link", + ]); + let mut back_to_back = spaced.clone(); + for segment in &mut back_to_back { + let next_starts = segment + .words + .iter() + .skip(1) + .map(|word| word.start) + .collect::>(); + for (word, next_start) in segment.words.iter_mut().zip(next_starts) { + word.end = next_start; + } + } + let base = CaptionSettings { + enabled: true, + ..Default::default() + }; + let highlight = CaptionSettings { + active_word_highlight: true, + ..base.clone() + }; + let cases = [ + ("plain", base, &spaced), + ("color highlight", highlight.clone(), &spaced), + ( + "color highlight gold outline", + CaptionSettings { + highlight_color: "#FFD700".to_string(), + outline: true, + ..highlight.clone() + }, + &spaced, + ), + ( + "color highlight back-to-back words", + highlight, + &back_to_back, + ), + ]; + for (name, settings, segments) in cases { + let mut uniforms = uniforms(&constants, settings, segments, size); + let mut layer = CaptionsLayer::new(&constants.device, &constants.queue); + let mut samples = Vec::new(); + let mut pass_totals = Vec::new(); + for pass in 0..PASSES { + let mut pass_total = 0.0; + for frame in 0..FRAMES { + uniforms.frame_number = frame; + let started = std::time::Instant::now(); + layer.prepare(&uniforms, &frames, size, &constants); + let elapsed = started.elapsed().as_secs_f64() * 1e6; + pass_total += elapsed; + if pass > 0 && layer.has_caption { + samples.push(elapsed); + } + } + if pass > 0 { + pass_totals.push(pass_total / 1e3); + } + constants.device.poll(wgpu::PollType::Wait).unwrap(); + } + samples.sort_by(f64::total_cmp); + pass_totals.sort_by(f64::total_cmp); + + let mut checked = CaptionsLayer::new(&constants.device, &constants.queue); + let mut digest = 0xcbf2_9ce4_8422_2325; + for frame in 0..FRAMES { + uniforms.frame_number = frame; + checked.prepare(&uniforms, &frames, size, &constants); + digest = pixel_digest(digest, &render_pixels(&constants, &checked, size)); + } + + println!( + "word highlight prepare [{name}] ({FRAMES} frames, {} visible samples): p50 {:.1} us, p95 {:.1} us, {FRAMES}-frame total p50 {:.2} ms, pixels {digest:016x}", + samples.len(), + samples[samples.len() / 2], + samples[samples.len() * 95 / 100], + pass_totals[pass_totals.len() / 2], + ); + } + } }