[4/7] Honour clip speed in exported frames and audio

src/components/canvas/players/video-player.ts

@@ -15,8 +15,7 @@ const SPEED_STEP_SECONDS = 0.01;
 
 type AnimatedSpeed = { curve: KeyframeBuilder; sourceOffsets: Float64Array };
 
-function createAnimatedSpeed(asset: VideoAsset, length: number): AnimatedSpeed | null {
-	const tweens = asset.speed;
+export function createAnimatedSpeed(tweens: VideoAsset["speed"], length: number): AnimatedSpeed | null {
 	if (!Array.isArray(tweens) || tweens.length === 0) return null;
 	const [first] = [...tweens].sort((a, b) => (a.start ?? 0) - (b.start ?? 0));
 	// The render holds the first tween's speed before it starts; KeyframeBuilder would otherwise ramp from 0.
@@ -55,7 +54,7 @@ export class VideoPlayer extends Player {
 		const videoAsset = this.clipConfiguration.asset as VideoAsset;
 
 		this.volumeKeyframeBuilder = new KeyframeBuilder(videoAsset.volume ?? 1, this.getLength());
-		this.animatedSpeed = createAnimatedSpeed(videoAsset, this.getLength());
+		this.animatedSpeed = createAnimatedSpeed(videoAsset.speed, this.getLength());
 		this.syncTimer = 0;
 		this.activeSyncTimer = 0;
 		this.skipVideoUpdate = false;
@@ -203,7 +202,7 @@ export class VideoPlayer extends Player {
 
 		const videoAsset = this.clipConfiguration.asset as VideoAsset;
 		this.volumeKeyframeBuilder = new KeyframeBuilder(videoAsset.volume ?? 1, this.getLength());
-		this.animatedSpeed = createAnimatedSpeed(videoAsset, this.getLength());
+		this.animatedSpeed = createAnimatedSpeed(videoAsset.speed, this.getLength());
 	}
 
 	private async loadVideo(): Promise<void> {

src/core/export/audio-processor.ts

@@ -1,9 +1,13 @@
 import { PlayerType, type Player } from "@canvas/players/player";
+import { createAnimatedSpeed } from "@canvas/players/video-player";
+import type { VideoAsset } from "@schemas";
 
 import { buildVolumeAutomation, type VolumeTween } from "./export-timing";
 
 const OUTPUT_SAMPLE_RATE = 48_000;
 const OUTPUT_CHANNELS = 2;
+// Matches the step VideoPlayer integrates its speed curve at.
+const SPEED_STEP_SECONDS = 0.01;
 
 // Player types whose `src` is (or, for text-to-speech, becomes) an audio-bearing file.
 const AUDIO_BEARING = new Set<string>([PlayerType.Audio, PlayerType.Video, PlayerType.TextToSpeech]);
@@ -15,6 +19,7 @@ interface AudioBearingClip {
 	trim: number;
 	volume: number | VolumeTween[] | undefined;
 	effect: string | undefined;
+	speed: VideoAsset["speed"];
 }
 
 /**
@@ -42,8 +47,10 @@ export class AudioProcessor {
 				source.connect(gain).connect(ctx.destination);
 
 				try {
-					// when = timeline start, offset = trim into the source, duration = clip length
-					source.start(Math.max(0, clip.start), Math.max(0, clip.trim), Math.max(0, clip.length));
+					const when = Math.max(0, clip.start);
+					const offset = this.applySpeed(source.playbackRate, clip, when);
+					source.start(when, Math.max(0, offset));
+					source.stop(when + Math.max(0, clip.length));
 					scheduled += 1;
 				} catch (error) {
 					// e.g. trim past the source end — contribute nothing rather than fail the export.
@@ -56,6 +63,23 @@ export class AudioProcessor {
 		return ctx.startRendering();
 	}
 
+	/**
+	 * Mirrors the preview's source time: a constant speed scales the trim as well as the rate,
+	 * while a speed curve starts at the trim and integrates from there. Returns the source offset.
+	 */
+	private applySpeed(rate: AudioParam, clip: AudioBearingClip, when: number): number {
+		const animated = createAnimatedSpeed(clip.speed, clip.length);
+		if (!animated) {
+			const speed = typeof clip.speed === "number" ? clip.speed : 1;
+			rate.setValueAtTime(speed, when);
+			return speed * clip.trim;
+		}
+		const samples = Math.max(2, Math.ceil(clip.length / SPEED_STEP_SECONDS) + 1);
+		const curve = Float32Array.from({ length: samples }, (_, i) => animated.curve.getValue((i * clip.length) / (samples - 1)));
+		rate.setValueCurveAtTime(curve, when, clip.length);
+		return clip.trim;
+	}
+
 	private applyVolume(param: AudioParam, clip: AudioBearingClip): void {
 		const points = buildVolumeAutomation(clip.volume, clip.effect, clip.length);
 		const base = Math.max(0, clip.start);
@@ -98,8 +122,7 @@ export class AudioProcessor {
 	private asAudioClip(clip: Player): AudioBearingClip | null {
 		if (!AUDIO_BEARING.has(clip.playerType)) return null;
 		const asset = clip.clipConfiguration?.asset as
-			| { src?: unknown; trim?: number; volume?: unknown; volumeEffect?: string; effect?: string }
-			| undefined;
+			{ src?: unknown; trim?: number; volume?: unknown; volumeEffect?: string; effect?: string; speed?: VideoAsset["speed"] } | undefined;
 		if (typeof asset?.src !== "string" || !asset.src) return null;
 
 		return {
@@ -108,7 +131,8 @@ export class AudioProcessor {
 			length: clip.getLength(),
 			trim: asset.trim ?? 0,
 			volume: asset.volume as number | VolumeTween[] | undefined,
-			effect: asset.volumeEffect ?? asset.effect
+			effect: asset.volumeEffect ?? asset.effect,
+			speed: asset.speed
 		};
 	}
 }

src/core/export/export-coordinator.ts

@@ -213,7 +213,7 @@ export class ExportCoordinator {
 					const start = player.getStart?.() || 0;
 					const end = player.getEnd?.() || start + (player.getLength?.() || 0);
 					if (frameTime >= start && frameTime < end) {
-						await this.videoProcessor.replaceVideoTexture(player, frameTime);
+						await this.videoProcessor.replaceVideoTexture(player);
 					}
 				}
 

src/core/export/export-timing.ts

@@ -1,7 +1,4 @@
 /** Pure timing math for export. */
-export function videoSourceTime(timestamp: number, clipStart: number, trim: number): number {
-	return timestamp - clipStart + trim;
-}
 
 /** A linear volume segment, clip-local seconds. */
 export interface VolumeTween {

src/core/export/video-frame-processor.ts

@@ -1,7 +1,6 @@
 import { Input, UrlSource, MP4, WEBM, CanvasSink, type InputVideoTrack } from "mediabunny";
 import * as pixi from "pixi.js";
 
-import { videoSourceTime } from "./export-timing";
 import type { ExportIssue } from "./export-utils";
 
 export interface VideoPlayerExtended {
@@ -10,6 +9,7 @@ export interface VideoPlayerExtended {
 	sprite?: { texture?: pixi.Texture };
 	clipConfiguration?: { asset?: { src?: string; trim?: number; type?: string } };
 	skipVideoUpdate?: boolean;
+	getSourceTime(): number;
 	getStart?(): number;
 	getEnd?(): number;
 	getLength?(): number;
@@ -28,8 +28,6 @@ interface ClipDecoder {
 	input: Input;
 	track: InputVideoTrack;
 	sink: CanvasSink;
-	trim: number;
-	start: number;
 	canvas?: HTMLCanvasElement;
 	ctx?: CanvasRenderingContext2D;
 	texture?: pixi.Texture;
@@ -60,8 +58,6 @@ export class VideoFrameProcessor {
 						input,
 						track,
 						sink: new CanvasSink(track, { poolSize: 2 }),
-						trim: player.clipConfiguration?.asset?.trim ?? 0,
-						start: player.getStart?.() ?? 0,
 						originalTexture: player.texture
 					});
 				} catch (error) {
@@ -75,12 +71,13 @@ export class VideoFrameProcessor {
 		return issues;
 	}
 
-	/** Point a clip's texture at the decoded source frame for the given timeline time. */
-	async replaceVideoTexture(player: VideoPlayerExtended, timestamp: number): Promise<void> {
+	/** Point a clip's texture at the decoded source frame for the edit's current playback time. */
+	async replaceVideoTexture(player: VideoPlayerExtended): Promise<void> {
 		const decoder = this.decoders.get(player);
 		if (!decoder) return;
 
-		const sourceTime = Math.max(0, videoSourceTime(timestamp, decoder.start, decoder.trim));
+		// The preview's own mapping, so trim, speed and speed curves land on the same source frame.
+		const sourceTime = Math.max(0, player.getSourceTime());
 		const wrapped = await decoder.sink.getCanvas(sourceTime);
 		if (!wrapped) return;
 

tests/export-audio.test.ts

@@ -0,0 +1,85 @@
+import { PlayerType, type Player } from "@canvas/players/player";
+import { AudioProcessor } from "@core/export/audio-processor";
+
+// Player modules extend pixi filters at load; the mix itself never touches pixi.
+jest.mock("pixi.js", () => ({
+	// eslint-disable-next-line global-require, @typescript-eslint/no-require-imports -- jest.mock factories can't use imports
+	...require("./helpers/pixi-mock-filters").pixiFilterStubs
+}));
+
+type Scheduled = { start: unknown[][]; stop: unknown[][]; rate: { setValueAtTime: jest.Mock; setValueCurveAtTime: jest.Mock } };
+
+const scheduled: Scheduled[] = [];
+
+function createBufferSource() {
+	const source = {
+		buffer: null as unknown,
+		playbackRate: { setValueAtTime: jest.fn(), setValueCurveAtTime: jest.fn() },
+		connect: (node: unknown) => node,
+		start: jest.fn(),
+		stop: jest.fn()
+	};
+	scheduled.push({ start: source.start.mock.calls, stop: source.stop.mock.calls, rate: source.playbackRate });
+	return source;
+}
+
+const createGain = () => ({ gain: { setValueAtTime: jest.fn(), linearRampToValueAtTime: jest.fn() }, connect: (node: unknown) => node });
+
+function mediaClip(playerType: PlayerType, asset: Record<string, unknown>, start: number, length: number): Player {
+	return { playerType, clipConfiguration: { asset }, getStart: () => start, getLength: () => length } as unknown as Player;
+}
+
+beforeAll(() => {
+	// Node has no Web Audio; the mix only needs these calls to schedule sources.
+	Object.assign(globalThis, {
+		OfflineAudioContext: jest.fn(() => ({
+			destination: {},
+			createBufferSource,
+			createGain,
+			decodeAudioData: async () => ({ duration: 60 }),
+			startRendering: async () => ({ length: 1 })
+		})),
+		fetch: jest.fn(async () => ({ ok: true, arrayBuffer: async () => new ArrayBuffer(8) }))
+	});
+});
+
+beforeEach(() => {
+	scheduled.length = 0;
+});
+
+describe("AudioProcessor speed", () => {
+	it("plays a constant speed at that rate from speed × trim, for the clip's length", async () => {
+		await new AudioProcessor().renderMix(
+			[[mediaClip(PlayerType.Audio, { type: "audio", src: "https://example.com/a.mp3", trim: 1, speed: 2 }, 3, 4)]],
+			10
+		);
+
+		expect(scheduled[0].rate.setValueAtTime).toHaveBeenCalledWith(2, 3);
+		expect(scheduled[0].start).toEqual([[3, 2]]);
+		expect(scheduled[0].stop).toEqual([[7]]);
+	});
+
+	it("plays at normal speed from the trim when the clip has no speed", async () => {
+		await new AudioProcessor().renderMix([[mediaClip(PlayerType.Audio, { type: "audio", src: "https://example.com/a.mp3", trim: 1 }, 0, 4)]], 10);
+
+		expect(scheduled[0].rate.setValueAtTime).toHaveBeenCalledWith(1, 0);
+		expect(scheduled[0].start).toEqual([[0, 1]]);
+	});
+
+	it("follows a speed curve from the unscaled trim", async () => {
+		const speed = [{ from: 1, to: 3, start: 0, length: 2 }];
+
+		await new AudioProcessor().renderMix(
+			[[mediaClip(PlayerType.Video, { type: "video", src: "https://example.com/v.mp4", trim: 1, speed }, 0.5, 2)]],
+			10
+		);
+
+		const [curve, when, duration] = scheduled[0].rate.setValueCurveAtTime.mock.calls[0];
+		expect(when).toBe(0.5);
+		expect(duration).toBe(2);
+		expect(curve[0]).toBeCloseTo(1);
+		expect(curve[curve.length - 1]).toBeCloseTo(3);
+		expect(scheduled[0].start).toEqual([[0.5, 1]]);
+		expect(scheduled[0].stop).toEqual([[2.5]]);
+	});
+});

tests/export-coordinator.test.ts

@@ -5,7 +5,7 @@ import type { Edit } from "@core/edit-session";
 import { ASSET_SETTLE_TIMEOUT_MS, ExportCoordinator } from "@core/export/export-coordinator";
 import { ExportProgressUI } from "@core/export/export-progress-ui";
 import { ExportError } from "@core/export/export-utils";
-import { canEncodeVideo, CanvasSource, Output } from "mediabunny";
+import { canEncodeVideo, CanvasSink, CanvasSource, Input, Output } from "mediabunny";
 
 const mockOutputs: Array<{ cancel: jest.Mock; state: string }> = [];
 
@@ -45,7 +45,7 @@ jest.mock("mediabunny", () => ({
 
 jest.mock("pixi.js", () => ({
 	Rectangle: jest.fn().mockImplementation((x: number, y: number, width: number, height: number) => ({ x, y, width, height })),
-	Texture: { from: jest.fn() }
+	Texture: { from: jest.fn(() => ({ source: { update: jest.fn() }, destroy: jest.fn() })) }
 }));
 
 // The canvas package needs TextEncoder, which jsdom lacks; the export only uses its font-name parser.
@@ -99,7 +99,7 @@ beforeAll(() => {
 	Object.assign(globalThis, {
 		ImageData: jest.fn((data: Uint8ClampedArray, width: number, height: number) => ({ data, width, height }))
 	});
-	Object.assign(HTMLCanvasElement.prototype, { getContext: () => ({ putImageData: jest.fn() }) });
+	Object.assign(HTMLCanvasElement.prototype, { getContext: () => ({ putImageData: jest.fn(), drawImage: jest.fn() }) });
 });
 
 beforeEach(() => {
@@ -332,6 +332,53 @@ describe("off-screen export", () => {
 	});
 });
 
+describe("speed", () => {
+	it("decodes each video frame at the player's own source time for that frame", async () => {
+		const { exporter, edit } = createHarness();
+		// isVideoPlayer recognises a player by its class name; getSourceTime stands in for the preview's speed mapping.
+		class VideoPlayer {
+			clipId = "fast-clip";
+
+			needsResolution = false;
+
+			clipConfiguration = { asset: { type: "video", src: "https://example.com/clip.mp4" } };
+
+			start = 0;
+
+			length = 1;
+
+			update = jest.fn();
+
+			constructor(private readonly session: { playbackTime: number }) {}
+
+			getSourceTime(): number {
+				return this.session.playbackTime * 3;
+			}
+
+			getStart(): number {
+				return this.start;
+			}
+
+			getEnd(): number {
+				return this.start + this.length;
+			}
+		}
+		edit.tracks = [[new VideoPlayer(edit)]];
+		const getCanvas = jest.fn(async () => ({ canvas: { width: 2, height: 2 } }));
+		jest
+			.mocked(Input)
+			.mockImplementationOnce(
+				() => ({ getPrimaryVideoTrack: async () => ({ canDecode: async () => true }), dispose: jest.fn() }) as unknown as InstanceType<typeof Input>
+			);
+		jest.mocked(CanvasSink).mockImplementationOnce(() => ({ getCanvas }) as unknown as InstanceType<typeof CanvasSink>);
+
+		await exporter.exportBlob();
+
+		expect(getCanvas.mock.calls).toEqual([[0], [1.5]]);
+		exporter.dispose();
+	});
+});
+
 describe("editor restart", () => {
 	it("restarts the app and rejects when pausing playback throws", async () => {
 		const { exporter, edit, app } = createHarness();

tests/export-timing.test.ts

@@ -7,28 +7,7 @@
  * length. These tests pin the units to seconds so the regression cannot recur.
  */
 
-import { buildVolumeAutomation, videoSourceTime } from "../src/core/export/export-timing";
-
-describe("export timing math", () => {
-	describe("videoSourceTime", () => {
-		it("maps a timeline timestamp to source-relative seconds", () => {
-			expect(videoSourceTime(5, 2, 0)).toBe(3);
-			expect(videoSourceTime(0, 0, 0)).toBe(0);
-		});
-
-		it("applies trim as a source offset", () => {
-			expect(videoSourceTime(1, 0, 1.5)).toBe(2.5);
-			expect(videoSourceTime(5, 2, 1.5)).toBe(4.5);
-		});
-
-		it("does not collapse seconds to milliseconds (the /1000 regression)", () => {
-			// 3s into a clip must seek ~3s of source, not 0.003s.
-			expect(videoSourceTime(3, 0, 0)).toBe(3);
-			expect(videoSourceTime(3, 0, 0)).toBeGreaterThan(0.5);
-		});
-	});
-
-});
+import { buildVolumeAutomation } from "../src/core/export/export-timing";
 
 describe("buildVolumeAutomation", () => {
 	it("defaults to constant unit gain", () => {