From f1f7bc372ff8ec2da5f0a8063bcf38de441744e6 Mon Sep 17 00:00:00 2001 From: Gal Sasson Date: Mon, 29 Jun 2026 09:52:30 +0300 Subject: [PATCH 1/3] feat(video): replay recorded tracking data via VideoSourceInput.trackingDataUrl MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add optional `trackingDataUrl` to VideoSourceInput. When set, the URL is fetched and passed to `createVideoSource` as `trackingData`, so the lens is driven by recorded tracking (camera pose, etc.) instead of live tracking — enabling recorded-environment previews of world-facing lenses. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/internal/sourceUtils.test.ts | 47 +++++++++++++++++++++++++++++++ src/internal/sourceUtils.ts | 48 ++++++++++++++++++++++++-------- src/types.ts | 7 +++++ 3 files changed, 90 insertions(+), 12 deletions(-) diff --git a/src/internal/sourceUtils.test.ts b/src/internal/sourceUtils.test.ts index d9fdccc..4636aa7 100644 --- a/src/internal/sourceUtils.test.ts +++ b/src/internal/sourceUtils.test.ts @@ -239,6 +239,7 @@ describe("sourceUtils", () => { afterEach(() => { jest.restoreAllMocks(); + delete (globalThis as { fetch?: typeof fetch }).fetch; }); it("should create a video source with autoplay", async () => { @@ -320,6 +321,52 @@ describe("sourceUtils", () => { videoElement: videoElement, }); }); + + it("should fetch tracking data and pass it to createVideoSource when trackingDataUrl is set", async () => { + const buffer = new ArrayBuffer(8); + const fetchMock = jest.fn().mockResolvedValue({ ok: true, arrayBuffer: () => Promise.resolve(buffer) }); + (globalThis as { fetch?: typeof fetch }).fetch = fetchMock as typeof fetch; + + const source = { + kind: "video" as const, + url: "https://example.com/video.mp4", + trackingDataUrl: "https://example.com/clip.td", + }; + + const promise = createCameraKitSource(source); + videoElement.dispatchEvent(new Event("canplay")); + await promise; + + expect(fetchMock).toHaveBeenCalledWith("https://example.com/clip.td"); + expect(mockCreateVideoSource).toHaveBeenCalledWith(videoElement, { trackingData: buffer }); + }); + + it("should not pass tracking data when trackingDataUrl is omitted", async () => { + const source = { kind: "video" as const, url: "https://example.com/video.mp4" }; + + const promise = createCameraKitSource(source); + videoElement.dispatchEvent(new Event("canplay")); + await promise; + + expect(mockCreateVideoSource).toHaveBeenCalledWith(videoElement, undefined); + }); + + it("should reject when tracking data fails to load", async () => { + (globalThis as { fetch?: typeof fetch }).fetch = jest + .fn() + .mockResolvedValue({ ok: false, status: 404 } as Response) as typeof fetch; + + const source = { + kind: "video" as const, + url: "https://example.com/video.mp4", + trackingDataUrl: "https://example.com/missing.td", + }; + + const promise = createCameraKitSource(source); + videoElement.dispatchEvent(new Event("canplay")); + + await expect(promise).rejects.toThrow("Unable to load tracking data"); + }); }); describe("createCameraKitSource - Image", () => { diff --git a/src/internal/sourceUtils.ts b/src/internal/sourceUtils.ts index f358b8f..650fd68 100644 --- a/src/internal/sourceUtils.ts +++ b/src/internal/sourceUtils.ts @@ -43,6 +43,7 @@ export async function createCameraKitSource(source: SourceInput): Promise((res, rej) => { autoplay = autoplay ?? true; const videoInput = document.createElement("video"); @@ -118,17 +127,24 @@ function createCameraKitVideoSource({ videoUrl, autoplay }: { videoUrl: string; videoInput.addEventListener( "canplay", async () => { - if (autoplay) await videoInput.play(); - res({ - cameraKitSource: createVideoSource(videoInput), - transform: Transform2D.Identity, - inputSize: [videoInput.videoWidth, videoInput.videoHeight], - initializedSourceInput: { - kind: "video", - url: videoUrl, - videoElement: videoInput, - }, - }); + try { + if (autoplay) await videoInput.play(); + // When a tracking-data sidecar is provided, fetch it and hand it to createVideoSource so the + // recorded tracking (camera pose, etc.) is replayed against the video instead of live tracking. + const trackingData = trackingDataUrl ? await fetchTrackingData(trackingDataUrl) : undefined; + res({ + cameraKitSource: createVideoSource(videoInput, trackingData ? { trackingData } : undefined), + transform: Transform2D.Identity, + inputSize: [videoInput.videoWidth, videoInput.videoHeight], + initializedSourceInput: { + kind: "video", + url: videoUrl, + videoElement: videoInput, + }, + }); + } catch (cause) { + rej(cause instanceof Error ? cause : new Error(String(cause))); + } }, { once: true }, ); @@ -146,6 +162,14 @@ function createCameraKitVideoSource({ videoUrl, autoplay }: { videoUrl: string; }); } +async function fetchTrackingData(url: string): Promise { + const response = await fetch(url); + if (!response.ok) { + throw new Error(`Unable to load tracking data from ${url} (HTTP ${response.status}).`); + } + return response.arrayBuffer(); +} + function createCameraKitImageSource({ imageUrl }: { imageUrl: string }) { return new Promise((res, rej) => { const imageInput = document.createElement("img"); diff --git a/src/types.ts b/src/types.ts index 1abf956..32732d5 100644 --- a/src/types.ts +++ b/src/types.ts @@ -48,6 +48,13 @@ export type VideoSourceInput = { kind: "video"; url: string; autoplay?: boolean; + /** + * Optional URL of a recorded tracking-data buffer to replay against the video (e.g. a Lens Studio + * `.td` file). When provided, it is fetched and passed to `createVideoSource` as `trackingData`, so + * the lens is driven by the recorded camera pose / tracking instead of live tracking. Useful for + * previewing world-facing lenses from a recorded environment. + */ + trackingDataUrl?: string; }; export type ImageSourceInput = { kind: "image"; url: string }; From 5c1c0cd7c980124f950baa9b996fdb56a07948a1 Mon Sep 17 00:00:00 2001 From: Gal Sasson Date: Mon, 29 Jun 2026 17:28:20 +0300 Subject: [PATCH 2/3] fix(video): fetch tracking data before starting playback Fetch the tracking-data sidecar before videoInput.play() so the looping video does not advance before Camera Kit receives the source (which would offset replay tracking from the video timeline), and a failed fetch rejects before playback starts (no orphaned playing video). Co-Authored-By: Claude Opus 4.8 (1M context) --- src/internal/sourceUtils.test.ts | 2 ++ src/internal/sourceUtils.ts | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/internal/sourceUtils.test.ts b/src/internal/sourceUtils.test.ts index 4636aa7..6e60b76 100644 --- a/src/internal/sourceUtils.test.ts +++ b/src/internal/sourceUtils.test.ts @@ -366,6 +366,8 @@ describe("sourceUtils", () => { videoElement.dispatchEvent(new Event("canplay")); await expect(promise).rejects.toThrow("Unable to load tracking data"); + // Tracking data is fetched before playback starts, so a failed fetch must not leave the video playing. + expect(videoElement.play).not.toHaveBeenCalled(); }); }); diff --git a/src/internal/sourceUtils.ts b/src/internal/sourceUtils.ts index 650fd68..2233c7a 100644 --- a/src/internal/sourceUtils.ts +++ b/src/internal/sourceUtils.ts @@ -128,10 +128,13 @@ function createCameraKitVideoSource({ "canplay", async () => { try { - if (autoplay) await videoInput.play(); // When a tracking-data sidecar is provided, fetch it and hand it to createVideoSource so the // recorded tracking (camera pose, etc.) is replayed against the video instead of live tracking. + // Fetch BEFORE starting playback: the looping video must not advance before Camera Kit has the + // source, or replay tracking drifts from the video timeline; and a fetch failure then rejects + // before any playback has started, so there is no orphaned playing video to clean up. const trackingData = trackingDataUrl ? await fetchTrackingData(trackingDataUrl) : undefined; + if (autoplay) await videoInput.play(); res({ cameraKitSource: createVideoSource(videoInput, trackingData ? { trackingData } : undefined), transform: Transform2D.Identity, From 695ac0d1be6844bbae326073f8c7d11dd1c287af Mon Sep 17 00:00:00 2001 From: Gal Sasson Date: Mon, 29 Jun 2026 18:01:00 +0300 Subject: [PATCH 3/3] bumped npm version - 0.4.0 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 9ba73f9..d897954 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@snap/react-camera-kit", - "version": "0.3.1", + "version": "0.4.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@snap/react-camera-kit", - "version": "0.3.1", + "version": "0.4.0", "license": "MIT", "dependencies": { "stable-hash": "^0.0.6" diff --git a/package.json b/package.json index 1fbc716..21bd925 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@snap/react-camera-kit", - "version": "0.3.1", + "version": "0.4.0", "description": "React Camera Kit for web applications", "type": "module", "main": "./dist/cjs/index.js",