diff --git a/crates/rendering/src/layers/captions.rs b/crates/rendering/src/layers/captions.rs index 15af3ff477f..d90e94c0d9d 100644 --- a/crates/rendering/src/layers/captions.rs +++ b/crates/rendering/src/layers/captions.rs @@ -261,6 +261,16 @@ fn calculate_word_highlight( 0.0 } +#[derive(PartialEq)] +struct ShapedCaptionText { + text: String, + font_size: u32, + wrap_width: u32, + family: Family<'static>, + weight: Weight, + color: Color, +} + pub struct CaptionsLayer { _settings_buffer: wgpu::Buffer, font_system: FontSystem, @@ -268,6 +278,7 @@ pub struct CaptionsLayer { text_atlas: TextAtlas, text_renderer: TextRenderer, text_buffer: Buffer, + shaped: Option, current_text: Option, current_segment_start: f32, current_segment_end: f32, @@ -412,6 +423,7 @@ impl CaptionsLayer { text_atlas, text_renderer, text_buffer, + shaped: None, current_text: None, current_segment_start: 0.0, current_segment_end: 0.0, @@ -605,12 +617,13 @@ impl CaptionsLayer { .clamp(0.0, 1.0); let font_size = caption_data.settings.size as f32 * (height as f32 / 1080.0); - let metrics = Metrics::new(font_size, font_size * 1.2); - - let mut updated_buffer = Buffer::new(&mut self.font_system, metrics); let wrap_width = (width as f32 - margin * 2.0).max(font_size); - updated_buffer.set_size(&mut self.font_system, Some(wrap_width), None); - updated_buffer.set_wrap(&mut self.font_system, glyphon::Wrap::None); + let new_buffer = |font_system: &mut FontSystem| { + let mut buffer = Buffer::new(font_system, Metrics::new(font_size, font_size * 1.2)); + buffer.set_size(font_system, Some(wrap_width), None); + buffer.set_wrap(font_system, glyphon::Wrap::None); + buffer + }; let font_family = match caption_data.settings.font.as_str() { "System Serif" => Family::Serif, @@ -708,13 +721,16 @@ impl CaptionsLayer { )); } - updated_buffer.set_rich_text( + let mut buffer = new_buffer(&mut self.font_system); + buffer.set_rich_text( &mut self.font_system, rich_text, &Attrs::new().family(font_family).weight(weight), Shaping::Advanced, None, ); + self.text_buffer = buffer; + self.shaped = None; } else { let color = Color::rgba( (base_color[0] * 255.0) as u8, @@ -722,19 +738,31 @@ impl CaptionsLayer { (base_color[2] * 255.0) as u8, (highlight_alpha * 255.0) as u8, ); - let attrs = Attrs::new().family(font_family).weight(weight).color(color); - updated_buffer.set_text( - &mut self.font_system, - caption_text.as_str(), - &attrs, - Shaping::Advanced, - ); + let shaped = ShapedCaptionText { + text: caption_text, + font_size: font_size.to_bits(), + wrap_width: wrap_width.to_bits(), + family: font_family, + weight, + color, + }; + if self.shaped.as_ref() != Some(&shaped) { + let mut buffer = new_buffer(&mut self.font_system); + buffer.set_text( + &mut self.font_system, + &shaped.text, + &Attrs::new().family(font_family).weight(weight).color(color), + Shaping::Advanced, + ); + self.text_buffer = buffer; + self.shaped = Some(shaped); + } } let mut layout_width: f32 = 0.0; let mut layout_height: f32 = 0.0; let mut highlight_extent: Option<(f32, f32, f32, f32)> = None; - for run in LayoutRunIter::new(&updated_buffer) { + for run in LayoutRunIter::new(&self.text_buffer) { layout_width = layout_width.max(run.line_w); layout_height = layout_height.max(run.line_top + run.line_height); @@ -844,7 +872,6 @@ impl CaptionsLayer { bottom: (text_top + draw_text_height + 2.0).ceil() as i32, }; - self.text_buffer = updated_buffer; self.viewport.update(queue, Resolution { width, height }); let mut text_areas = Vec::new(); @@ -1260,3 +1287,446 @@ mod tests { assert_eq!(word_byte_range("missing", &words, 0, false), None); } } + +#[cfg(test)] +mod gpu_tests { + use cap_project::{ + CaptionSettings, CaptionTrackSegment, CaptionWord, CaptionsData, CursorEvents, + ProjectConfiguration, RecordingMeta, RecordingMetaInner, StudioRecordingMeta, + TimelineConfiguration, XY, + }; + + use super::CaptionsLayer; + use crate::{ + DecodedSegmentFrames, ProjectUniforms, RenderOptions, RenderVideoConstants, + ZoomTransformTimeline, + }; + + const FPS: u32 = 60; + + fn constants(size: XY) -> RenderVideoConstants { + let meta: StudioRecordingMeta = serde_json::from_value(serde_json::json!({ + "display": { "path": "synthetic.mp4", "fps": FPS } + })) + .unwrap(); + let recording = RecordingMeta { + platform: None, + project_path: Default::default(), + pretty_name: "Caption overlay test".into(), + sharing: None, + inner: RecordingMetaInner::Studio(Box::new(meta.clone())), + upload: None, + }; + pollster::block_on(RenderVideoConstants::new_with_options( + RenderOptions { + screen_size: size, + camera_size: None, + preserve_screen_alpha: false, + }, + recording, + meta, + )) + .unwrap() + } + + fn spoken_segments(sentences: &[&str]) -> Vec { + let mut start = 0.3; + sentences + .iter() + .enumerate() + .map(|(index, sentence)| { + let words = sentence + .split_whitespace() + .enumerate() + .map(|(word_index, text)| CaptionWord { + text: text.to_string(), + start: start as f32 + word_index as f32 * 0.3, + end: start as f32 + word_index as f32 * 0.3 + 0.25, + }) + .collect::>(); + let end = f64::from(words.last().map_or(start as f32, |word| word.end)); + let segment = CaptionTrackSegment { + id: format!("caption-{index}"), + start, + end, + text: sentence.to_string(), + words, + fade_duration_override: None, + linger_duration_override: None, + position_override: None, + color_override: None, + background_color_override: None, + font_size_override: None, + }; + start = end + 0.4; + segment + }) + .collect() + } + + fn frames(size: XY) -> DecodedSegmentFrames { + DecodedSegmentFrames { + screen_size: size, + screen_frame: None, + camera_frame: None, + segment_time: 0.0, + recording_time: 0.0, + segment_has_camera: false, + } + } + + fn uniforms( + constants: &RenderVideoConstants, + settings: CaptionSettings, + segments: &[CaptionTrackSegment], + size: XY, + ) -> ProjectUniforms { + let project = ProjectConfiguration::default(); + let cursor = CursorEvents::default(); + let frames = frames(size); + let mut zoom = ZoomTransformTimeline::from_project(&project, &cursor, 60.0, size); + zoom.ensure_precomputed_until(60.0); + let mut uniforms = ProjectUniforms::new( + constants, &project, 0, FPS, size, &cursor, &frames, 60.0, &zoom, + ); + uniforms.output_size = (size.x, size.y); + uniforms.project.captions = Some(CaptionsData { + segments: vec![], + settings, + source_timed: false, + }); + uniforms.project.timeline = Some(TimelineConfiguration { + segments: vec![], + transitions: vec![], + zoom_segments: vec![], + scene_segments: vec![], + mask_segments: vec![], + text_segments: vec![], + caption_segments: segments.to_vec(), + keyboard_segments: vec![], + audio_segments: vec![], + style_segments: vec![], + image_segments: vec![], + camera3d_segments: vec![], + }); + uniforms + } + + fn render_pixels( + constants: &RenderVideoConstants, + layer: &CaptionsLayer, + size: XY, + ) -> Vec { + let device = &constants.device; + let target = device.create_texture(&wgpu::TextureDescriptor { + label: Some("Caption Test Target"), + size: wgpu::Extent3d { + width: size.x, + height: size.y, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Rgba8Unorm, + usage: wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::COPY_SRC, + view_formats: &[], + }); + let view = target.create_view(&Default::default()); + let readback = device.create_buffer(&wgpu::BufferDescriptor { + label: Some("Caption Test Readback"), + size: u64::from(size.x * size.y * 4), + usage: wgpu::BufferUsages::COPY_DST | wgpu::BufferUsages::MAP_READ, + mapped_at_creation: false, + }); + let mut encoder = device.create_command_encoder(&Default::default()); + { + let mut pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("Caption Test Pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &view, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT), + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + }); + layer.render(&mut pass); + } + encoder.copy_texture_to_buffer( + wgpu::TexelCopyTextureInfo { + texture: &target, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + wgpu::TexelCopyBufferInfo { + buffer: &readback, + layout: wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(size.x * 4), + rows_per_image: Some(size.y), + }, + }, + wgpu::Extent3d { + width: size.x, + height: size.y, + depth_or_array_layers: 1, + }, + ); + constants.queue.submit([encoder.finish()]); + readback.slice(..).map_async(wgpu::MapMode::Read, |_| {}); + device.poll(wgpu::PollType::Wait).unwrap(); + let pixels = readback.slice(..).get_mapped_range().to_vec(); + readback.unmap(); + pixels + } + + #[test] + #[ignore = "requires a working wgpu adapter"] + fn every_caption_frame_renders_like_a_fresh_layer() { + let size = XY::new(512, 288); + let constants = constants(size); + let segments = spoken_segments(&["Hello there", "Ship it today"]); + let frames = frames(size); + let enabled = CaptionSettings { + enabled: true, + size: 60, + ..Default::default() + }; + let variants = [ + enabled.clone(), + CaptionSettings { + outline: true, + uppercase: true, + animation: "pop".to_string(), + ..enabled.clone() + }, + CaptionSettings { + active_word_highlight: true, + highlight_style: "pill".to_string(), + ..enabled.clone() + }, + CaptionSettings { + active_word_highlight: true, + ..enabled + }, + ]; + let last_frame = (segments.last().unwrap().end + 0.3) * f64::from(FPS); + for settings in variants { + let mut uniforms = uniforms(&constants, settings, &segments, size); + let mut reused = CaptionsLayer::new(&constants.device, &constants.queue); + let mut drawn_frames = 0; + for frame in 0..last_frame as u32 { + uniforms.frame_number = frame; + reused.prepare(&uniforms, &frames, size, &constants); + let mut fresh = CaptionsLayer::new(&constants.device, &constants.queue); + fresh.prepare(&uniforms, &frames, size, &constants); + assert_eq!(reused.has_caption, fresh.has_caption, "frame {frame}"); + let pixels = render_pixels(&constants, &reused, size); + assert_eq!( + pixels, + render_pixels(&constants, &fresh, size), + "frame {frame}" + ); + drawn_frames += usize::from(pixels.chunks_exact(4).any(|pixel| pixel[3] != 0)); + } + assert!(drawn_frames > 60); + } + } + + #[test] + #[ignore = "requires a working wgpu adapter"] + fn reused_layer_follows_style_and_output_size_changes() { + let base_size = XY::new(512, 288); + let constants = constants(base_size); + let segments = spoken_segments(&["Ship it today"]); + let frames = frames(base_size); + let render = |layer: &mut CaptionsLayer, settings: &CaptionSettings, size: XY| { + let mut uniforms = uniforms(&constants, settings.clone(), &segments, size); + uniforms.frame_number = 45; + layer.prepare(&uniforms, &frames, size, &constants); + assert!(layer.has_caption); + render_pixels(&constants, layer, size) + }; + let plain = CaptionSettings { + enabled: true, + size: 60, + ..Default::default() + }; + let color_highlight = CaptionSettings { + active_word_highlight: true, + highlight_color: "#FF0000".to_string(), + ..plain.clone() + }; + let cases = [ + ( + "font size", + plain.clone(), + CaptionSettings { + size: 80, + ..plain.clone() + }, + base_size, + ), + ( + "font family", + plain.clone(), + CaptionSettings { + font: "System Monospace".to_string(), + ..plain.clone() + }, + base_size, + ), + ( + "font weight", + plain.clone(), + CaptionSettings { + font_weight: 400, + ..plain.clone() + }, + base_size, + ), + ( + "text color", + plain.clone(), + CaptionSettings { + color: "#FF0000".to_string(), + ..plain.clone() + }, + base_size, + ), + ( + "output height", + plain.clone(), + plain.clone(), + XY::new(512, 216), + ), + ( + "output width", + plain.clone(), + plain.clone(), + XY::new(384, 288), + ), + ( + "plain to color highlight", + plain.clone(), + color_highlight.clone(), + base_size, + ), + ( + "color highlight to plain", + color_highlight.clone(), + plain.clone(), + base_size, + ), + ]; + + for (name, before, after, size) in cases { + let expected = render( + &mut CaptionsLayer::new(&constants.device, &constants.queue), + &after, + size, + ); + let previous = render( + &mut CaptionsLayer::new(&constants.device, &constants.queue), + &before, + base_size, + ); + assert!(expected != previous, "{name} edit should change the output"); + + let mut reused = CaptionsLayer::new(&constants.device, &constants.queue); + render(&mut reused, &before, base_size); + assert!( + render(&mut reused, &after, size) == expected, + "{name} edit reused a stale layout" + ); + } + + let mut reused = CaptionsLayer::new(&constants.device, &constants.queue); + render(&mut reused, &plain, base_size); + render(&mut reused, &color_highlight, base_size); + let expected = render( + &mut CaptionsLayer::new(&constants.device, &constants.queue), + &plain, + base_size, + ); + assert!( + render(&mut reused, &plain, base_size) == expected, + "returning to plain text reused the highlighted layout" + ); + } + + #[test] + #[ignore = "benchmark; requires a working wgpu adapter"] + fn benchmark_captions_layer_prepare() { + let size = XY::new(1920, 1080); + let constants = constants(size); + let segments = spoken_segments(&[ + "Today I want to walk you through the new export flow", + "First pick a preset and the resolution you need", + "Then trim the start and end so the video gets to the point", + "When you are happy with it press export and share the link", + ]); + let frames = frames(size); + let last_frame = (segments.last().unwrap().end * f64::from(FPS)) as u32; + let base = CaptionSettings { + enabled: true, + ..Default::default() + }; + let cases = [ + ("default", base.clone()), + ( + "outline", + CaptionSettings { + outline: true, + ..base.clone() + }, + ), + ( + "pill highlight", + CaptionSettings { + active_word_highlight: true, + highlight_style: "pill".to_string(), + ..base.clone() + }, + ), + ( + "color highlight", + CaptionSettings { + active_word_highlight: true, + ..base + }, + ), + ]; + for (name, settings) in cases { + let mut uniforms = uniforms(&constants, settings, &segments, size); + let mut layer = CaptionsLayer::new(&constants.device, &constants.queue); + let mut samples = Vec::new(); + for pass in 0..4 { + for frame in 0..last_frame { + uniforms.frame_number = frame; + let started = std::time::Instant::now(); + layer.prepare(&uniforms, &frames, size, &constants); + let elapsed = started.elapsed().as_secs_f64() * 1e6; + if pass > 0 && layer.has_caption { + samples.push(elapsed); + } + } + } + constants.device.poll(wgpu::PollType::Wait).unwrap(); + samples.sort_by(f64::total_cmp); + println!( + "captions layer prepare [{name}] ({} visible frames): p50 {:.1} us, p95 {:.1} us, mean {:.1} us", + samples.len(), + samples[samples.len() / 2], + samples[samples.len() * 95 / 100], + samples.iter().sum::() / samples.len() as f64, + ); + } + } +}