From f14139ae33dacac2730a48a86930142191402231 Mon Sep 17 00:00:00 2001 From: dazzatronus Date: Mon, 19 Oct 2026 21:26:04 +1100 Subject: [PATCH] fix(export): honour clip speed in video frames and the audio mix --- src/components/canvas/players/video-player.ts | 7 +- src/core/export/audio-processor.ts | 34 ++++++-- src/core/export/export-coordinator.ts | 2 +- src/core/export/export-timing.ts | 3 - src/core/export/video-frame-processor.ts | 13 ++- tests/export-audio.test.ts | 85 +++++++++++++++++++ tests/export-coordinator.test.ts | 53 +++++++++++- tests/export-timing.test.ts | 23 +---- 8 files changed, 174 insertions(+), 46 deletions(-) create mode 100644 tests/export-audio.test.ts diff --git a/src/components/canvas/players/video-player.ts b/src/components/canvas/players/video-player.ts index 24dbc7ad..26f5fb8c 100644 --- a/src/components/canvas/players/video-player.ts +++ b/src/components/canvas/players/video-player.ts @@ -15,8 +15,7 @@ const SPEED_STEP_SECONDS = 0.01; type AnimatedSpeed = { curve: KeyframeBuilder; sourceOffsets: Float64Array }; -function createAnimatedSpeed(asset: VideoAsset, length: number): AnimatedSpeed | null { - const tweens = asset.speed; +export function createAnimatedSpeed(tweens: VideoAsset["speed"], length: number): AnimatedSpeed | null { if (!Array.isArray(tweens) || tweens.length === 0) return null; const [first] = [...tweens].sort((a, b) => (a.start ?? 0) - (b.start ?? 0)); // The render holds the first tween's speed before it starts; KeyframeBuilder would otherwise ramp from 0. @@ -55,7 +54,7 @@ export class VideoPlayer extends Player { const videoAsset = this.clipConfiguration.asset as VideoAsset; this.volumeKeyframeBuilder = new KeyframeBuilder(videoAsset.volume ?? 1, this.getLength()); - this.animatedSpeed = createAnimatedSpeed(videoAsset, this.getLength()); + this.animatedSpeed = createAnimatedSpeed(videoAsset.speed, this.getLength()); this.syncTimer = 0; this.activeSyncTimer = 0; this.skipVideoUpdate = false; @@ -203,7 +202,7 @@ export class VideoPlayer extends Player { const videoAsset = this.clipConfiguration.asset as VideoAsset; this.volumeKeyframeBuilder = new KeyframeBuilder(videoAsset.volume ?? 1, this.getLength()); - this.animatedSpeed = createAnimatedSpeed(videoAsset, this.getLength()); + this.animatedSpeed = createAnimatedSpeed(videoAsset.speed, this.getLength()); } private async loadVideo(): Promise { diff --git a/src/core/export/audio-processor.ts b/src/core/export/audio-processor.ts index 5da4625b..d04b45a2 100644 --- a/src/core/export/audio-processor.ts +++ b/src/core/export/audio-processor.ts @@ -1,9 +1,13 @@ import { PlayerType, type Player } from "@canvas/players/player"; +import { createAnimatedSpeed } from "@canvas/players/video-player"; +import type { VideoAsset } from "@schemas"; import { buildVolumeAutomation, type VolumeTween } from "./export-timing"; const OUTPUT_SAMPLE_RATE = 48_000; const OUTPUT_CHANNELS = 2; +// Matches the step VideoPlayer integrates its speed curve at. +const SPEED_STEP_SECONDS = 0.01; // Player types whose `src` is (or, for text-to-speech, becomes) an audio-bearing file. const AUDIO_BEARING = new Set([PlayerType.Audio, PlayerType.Video, PlayerType.TextToSpeech]); @@ -15,6 +19,7 @@ interface AudioBearingClip { trim: number; volume: number | VolumeTween[] | undefined; effect: string | undefined; + speed: VideoAsset["speed"]; } /** @@ -42,8 +47,10 @@ export class AudioProcessor { source.connect(gain).connect(ctx.destination); try { - // when = timeline start, offset = trim into the source, duration = clip length - source.start(Math.max(0, clip.start), Math.max(0, clip.trim), Math.max(0, clip.length)); + const when = Math.max(0, clip.start); + const offset = this.applySpeed(source.playbackRate, clip, when); + source.start(when, Math.max(0, offset)); + source.stop(when + Math.max(0, clip.length)); scheduled += 1; } catch (error) { // e.g. trim past the source end — contribute nothing rather than fail the export. @@ -56,6 +63,23 @@ export class AudioProcessor { return ctx.startRendering(); } + /** + * Mirrors the preview's source time: a constant speed scales the trim as well as the rate, + * while a speed curve starts at the trim and integrates from there. Returns the source offset. + */ + private applySpeed(rate: AudioParam, clip: AudioBearingClip, when: number): number { + const animated = createAnimatedSpeed(clip.speed, clip.length); + if (!animated) { + const speed = typeof clip.speed === "number" ? clip.speed : 1; + rate.setValueAtTime(speed, when); + return speed * clip.trim; + } + const samples = Math.max(2, Math.ceil(clip.length / SPEED_STEP_SECONDS) + 1); + const curve = Float32Array.from({ length: samples }, (_, i) => animated.curve.getValue((i * clip.length) / (samples - 1))); + rate.setValueCurveAtTime(curve, when, clip.length); + return clip.trim; + } + private applyVolume(param: AudioParam, clip: AudioBearingClip): void { const points = buildVolumeAutomation(clip.volume, clip.effect, clip.length); const base = Math.max(0, clip.start); @@ -98,8 +122,7 @@ export class AudioProcessor { private asAudioClip(clip: Player): AudioBearingClip | null { if (!AUDIO_BEARING.has(clip.playerType)) return null; const asset = clip.clipConfiguration?.asset as - | { src?: unknown; trim?: number; volume?: unknown; volumeEffect?: string; effect?: string } - | undefined; + { src?: unknown; trim?: number; volume?: unknown; volumeEffect?: string; effect?: string; speed?: VideoAsset["speed"] } | undefined; if (typeof asset?.src !== "string" || !asset.src) return null; return { @@ -108,7 +131,8 @@ export class AudioProcessor { length: clip.getLength(), trim: asset.trim ?? 0, volume: asset.volume as number | VolumeTween[] | undefined, - effect: asset.volumeEffect ?? asset.effect + effect: asset.volumeEffect ?? asset.effect, + speed: asset.speed }; } } diff --git a/src/core/export/export-coordinator.ts b/src/core/export/export-coordinator.ts index 87192afb..7dfa677b 100644 --- a/src/core/export/export-coordinator.ts +++ b/src/core/export/export-coordinator.ts @@ -213,7 +213,7 @@ export class ExportCoordinator { const start = player.getStart?.() || 0; const end = player.getEnd?.() || start + (player.getLength?.() || 0); if (frameTime >= start && frameTime < end) { - await this.videoProcessor.replaceVideoTexture(player, frameTime); + await this.videoProcessor.replaceVideoTexture(player); } } diff --git a/src/core/export/export-timing.ts b/src/core/export/export-timing.ts index ea21b05f..d7243c6e 100644 --- a/src/core/export/export-timing.ts +++ b/src/core/export/export-timing.ts @@ -1,7 +1,4 @@ /** Pure timing math for export. */ -export function videoSourceTime(timestamp: number, clipStart: number, trim: number): number { - return timestamp - clipStart + trim; -} /** A linear volume segment, clip-local seconds. */ export interface VolumeTween { diff --git a/src/core/export/video-frame-processor.ts b/src/core/export/video-frame-processor.ts index 659bd9b0..ec196917 100644 --- a/src/core/export/video-frame-processor.ts +++ b/src/core/export/video-frame-processor.ts @@ -1,7 +1,6 @@ import { Input, UrlSource, MP4, WEBM, CanvasSink, type InputVideoTrack } from "mediabunny"; import * as pixi from "pixi.js"; -import { videoSourceTime } from "./export-timing"; import type { ExportIssue } from "./export-utils"; export interface VideoPlayerExtended { @@ -10,6 +9,7 @@ export interface VideoPlayerExtended { sprite?: { texture?: pixi.Texture }; clipConfiguration?: { asset?: { src?: string; trim?: number; type?: string } }; skipVideoUpdate?: boolean; + getSourceTime(): number; getStart?(): number; getEnd?(): number; getLength?(): number; @@ -28,8 +28,6 @@ interface ClipDecoder { input: Input; track: InputVideoTrack; sink: CanvasSink; - trim: number; - start: number; canvas?: HTMLCanvasElement; ctx?: CanvasRenderingContext2D; texture?: pixi.Texture; @@ -60,8 +58,6 @@ export class VideoFrameProcessor { input, track, sink: new CanvasSink(track, { poolSize: 2 }), - trim: player.clipConfiguration?.asset?.trim ?? 0, - start: player.getStart?.() ?? 0, originalTexture: player.texture }); } catch (error) { @@ -75,12 +71,13 @@ export class VideoFrameProcessor { return issues; } - /** Point a clip's texture at the decoded source frame for the given timeline time. */ - async replaceVideoTexture(player: VideoPlayerExtended, timestamp: number): Promise { + /** Point a clip's texture at the decoded source frame for the edit's current playback time. */ + async replaceVideoTexture(player: VideoPlayerExtended): Promise { const decoder = this.decoders.get(player); if (!decoder) return; - const sourceTime = Math.max(0, videoSourceTime(timestamp, decoder.start, decoder.trim)); + // The preview's own mapping, so trim, speed and speed curves land on the same source frame. + const sourceTime = Math.max(0, player.getSourceTime()); const wrapped = await decoder.sink.getCanvas(sourceTime); if (!wrapped) return; diff --git a/tests/export-audio.test.ts b/tests/export-audio.test.ts new file mode 100644 index 00000000..8278c228 --- /dev/null +++ b/tests/export-audio.test.ts @@ -0,0 +1,85 @@ +import { PlayerType, type Player } from "@canvas/players/player"; +import { AudioProcessor } from "@core/export/audio-processor"; + +// Player modules extend pixi filters at load; the mix itself never touches pixi. +jest.mock("pixi.js", () => ({ + // eslint-disable-next-line global-require, @typescript-eslint/no-require-imports -- jest.mock factories can't use imports + ...require("./helpers/pixi-mock-filters").pixiFilterStubs +})); + +type Scheduled = { start: unknown[][]; stop: unknown[][]; rate: { setValueAtTime: jest.Mock; setValueCurveAtTime: jest.Mock } }; + +const scheduled: Scheduled[] = []; + +function createBufferSource() { + const source = { + buffer: null as unknown, + playbackRate: { setValueAtTime: jest.fn(), setValueCurveAtTime: jest.fn() }, + connect: (node: unknown) => node, + start: jest.fn(), + stop: jest.fn() + }; + scheduled.push({ start: source.start.mock.calls, stop: source.stop.mock.calls, rate: source.playbackRate }); + return source; +} + +const createGain = () => ({ gain: { setValueAtTime: jest.fn(), linearRampToValueAtTime: jest.fn() }, connect: (node: unknown) => node }); + +function mediaClip(playerType: PlayerType, asset: Record, start: number, length: number): Player { + return { playerType, clipConfiguration: { asset }, getStart: () => start, getLength: () => length } as unknown as Player; +} + +beforeAll(() => { + // Node has no Web Audio; the mix only needs these calls to schedule sources. + Object.assign(globalThis, { + OfflineAudioContext: jest.fn(() => ({ + destination: {}, + createBufferSource, + createGain, + decodeAudioData: async () => ({ duration: 60 }), + startRendering: async () => ({ length: 1 }) + })), + fetch: jest.fn(async () => ({ ok: true, arrayBuffer: async () => new ArrayBuffer(8) })) + }); +}); + +beforeEach(() => { + scheduled.length = 0; +}); + +describe("AudioProcessor speed", () => { + it("plays a constant speed at that rate from speed × trim, for the clip's length", async () => { + await new AudioProcessor().renderMix( + [[mediaClip(PlayerType.Audio, { type: "audio", src: "https://example.com/a.mp3", trim: 1, speed: 2 }, 3, 4)]], + 10 + ); + + expect(scheduled[0].rate.setValueAtTime).toHaveBeenCalledWith(2, 3); + expect(scheduled[0].start).toEqual([[3, 2]]); + expect(scheduled[0].stop).toEqual([[7]]); + }); + + it("plays at normal speed from the trim when the clip has no speed", async () => { + await new AudioProcessor().renderMix([[mediaClip(PlayerType.Audio, { type: "audio", src: "https://example.com/a.mp3", trim: 1 }, 0, 4)]], 10); + + expect(scheduled[0].rate.setValueAtTime).toHaveBeenCalledWith(1, 0); + expect(scheduled[0].start).toEqual([[0, 1]]); + }); + + it("follows a speed curve from the unscaled trim", async () => { + const speed = [{ from: 1, to: 3, start: 0, length: 2 }]; + + await new AudioProcessor().renderMix( + [[mediaClip(PlayerType.Video, { type: "video", src: "https://example.com/v.mp4", trim: 1, speed }, 0.5, 2)]], + 10 + ); + + const [curve, when, duration] = scheduled[0].rate.setValueCurveAtTime.mock.calls[0]; + expect(when).toBe(0.5); + expect(duration).toBe(2); + expect(curve[0]).toBeCloseTo(1); + expect(curve[curve.length - 1]).toBeCloseTo(3); + expect(scheduled[0].start).toEqual([[0.5, 1]]); + expect(scheduled[0].stop).toEqual([[2.5]]); + }); +}); diff --git a/tests/export-coordinator.test.ts b/tests/export-coordinator.test.ts index 125ebb35..decc752f 100644 --- a/tests/export-coordinator.test.ts +++ b/tests/export-coordinator.test.ts @@ -5,7 +5,7 @@ import type { Edit } from "@core/edit-session"; import { ASSET_SETTLE_TIMEOUT_MS, ExportCoordinator } from "@core/export/export-coordinator"; import { ExportProgressUI } from "@core/export/export-progress-ui"; import { ExportError } from "@core/export/export-utils"; -import { canEncodeVideo, CanvasSource, Output } from "mediabunny"; +import { canEncodeVideo, CanvasSink, CanvasSource, Input, Output } from "mediabunny"; const mockOutputs: Array<{ cancel: jest.Mock; state: string }> = []; @@ -45,7 +45,7 @@ jest.mock("mediabunny", () => ({ jest.mock("pixi.js", () => ({ Rectangle: jest.fn().mockImplementation((x: number, y: number, width: number, height: number) => ({ x, y, width, height })), - Texture: { from: jest.fn() } + Texture: { from: jest.fn(() => ({ source: { update: jest.fn() }, destroy: jest.fn() })) } })); // The canvas package needs TextEncoder, which jsdom lacks; the export only uses its font-name parser. @@ -99,7 +99,7 @@ beforeAll(() => { Object.assign(globalThis, { ImageData: jest.fn((data: Uint8ClampedArray, width: number, height: number) => ({ data, width, height })) }); - Object.assign(HTMLCanvasElement.prototype, { getContext: () => ({ putImageData: jest.fn() }) }); + Object.assign(HTMLCanvasElement.prototype, { getContext: () => ({ putImageData: jest.fn(), drawImage: jest.fn() }) }); }); beforeEach(() => { @@ -332,6 +332,53 @@ describe("off-screen export", () => { }); }); +describe("speed", () => { + it("decodes each video frame at the player's own source time for that frame", async () => { + const { exporter, edit } = createHarness(); + // isVideoPlayer recognises a player by its class name; getSourceTime stands in for the preview's speed mapping. + class VideoPlayer { + clipId = "fast-clip"; + + needsResolution = false; + + clipConfiguration = { asset: { type: "video", src: "https://example.com/clip.mp4" } }; + + start = 0; + + length = 1; + + update = jest.fn(); + + constructor(private readonly session: { playbackTime: number }) {} + + getSourceTime(): number { + return this.session.playbackTime * 3; + } + + getStart(): number { + return this.start; + } + + getEnd(): number { + return this.start + this.length; + } + } + edit.tracks = [[new VideoPlayer(edit)]]; + const getCanvas = jest.fn(async () => ({ canvas: { width: 2, height: 2 } })); + jest + .mocked(Input) + .mockImplementationOnce( + () => ({ getPrimaryVideoTrack: async () => ({ canDecode: async () => true }), dispose: jest.fn() }) as unknown as InstanceType + ); + jest.mocked(CanvasSink).mockImplementationOnce(() => ({ getCanvas }) as unknown as InstanceType); + + await exporter.exportBlob(); + + expect(getCanvas.mock.calls).toEqual([[0], [1.5]]); + exporter.dispose(); + }); +}); + describe("editor restart", () => { it("restarts the app and rejects when pausing playback throws", async () => { const { exporter, edit, app } = createHarness(); diff --git a/tests/export-timing.test.ts b/tests/export-timing.test.ts index 1af36394..f8bf44c3 100644 --- a/tests/export-timing.test.ts +++ b/tests/export-timing.test.ts @@ -7,28 +7,7 @@ * length. These tests pin the units to seconds so the regression cannot recur. */ -import { buildVolumeAutomation, videoSourceTime } from "../src/core/export/export-timing"; - -describe("export timing math", () => { - describe("videoSourceTime", () => { - it("maps a timeline timestamp to source-relative seconds", () => { - expect(videoSourceTime(5, 2, 0)).toBe(3); - expect(videoSourceTime(0, 0, 0)).toBe(0); - }); - - it("applies trim as a source offset", () => { - expect(videoSourceTime(1, 0, 1.5)).toBe(2.5); - expect(videoSourceTime(5, 2, 1.5)).toBe(4.5); - }); - - it("does not collapse seconds to milliseconds (the /1000 regression)", () => { - // 3s into a clip must seek ~3s of source, not 0.003s. - expect(videoSourceTime(3, 0, 0)).toBe(3); - expect(videoSourceTime(3, 0, 0)).toBeGreaterThan(0.5); - }); - }); - -}); +import { buildVolumeAutomation } from "../src/core/export/export-timing"; describe("buildVolumeAutomation", () => { it("defaults to constant unit gain", () => {