feat(video): replay recorded tracking data via VideoSourceInput.trackingDataUrl

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@snap/react-camera-kit",
-  "version": "0.3.1",
+  "version": "0.4.0",
   "description": "React Camera Kit for web applications",
   "type": "module",
   "main": "./dist/cjs/index.js",

src/internal/sourceUtils.test.ts

@@ -239,6 +239,7 @@ describe("sourceUtils", () => {
 
     afterEach(() => {
       jest.restoreAllMocks();
+      delete (globalThis as { fetch?: typeof fetch }).fetch;
     });
 
     it("should create a video source with autoplay", async () => {
@@ -320,6 +321,54 @@ describe("sourceUtils", () => {
         videoElement: videoElement,
       });
     });
+
+    it("should fetch tracking data and pass it to createVideoSource when trackingDataUrl is set", async () => {
+      const buffer = new ArrayBuffer(8);
+      const fetchMock = jest.fn().mockResolvedValue({ ok: true, arrayBuffer: () => Promise.resolve(buffer) });
+      (globalThis as { fetch?: typeof fetch }).fetch = fetchMock as typeof fetch;
+
+      const source = {
+        kind: "video" as const,
+        url: "https://example.com/video.mp4",
+        trackingDataUrl: "https://example.com/clip.td",
+      };
+
+      const promise = createCameraKitSource(source);
+      videoElement.dispatchEvent(new Event("canplay"));
+      await promise;
+
+      expect(fetchMock).toHaveBeenCalledWith("https://example.com/clip.td");
+      expect(mockCreateVideoSource).toHaveBeenCalledWith(videoElement, { trackingData: buffer });
+    });
+
+    it("should not pass tracking data when trackingDataUrl is omitted", async () => {
+      const source = { kind: "video" as const, url: "https://example.com/video.mp4" };
+
+      const promise = createCameraKitSource(source);
+      videoElement.dispatchEvent(new Event("canplay"));
+      await promise;
+
+      expect(mockCreateVideoSource).toHaveBeenCalledWith(videoElement, undefined);
+    });
+
+    it("should reject when tracking data fails to load", async () => {
+      (globalThis as { fetch?: typeof fetch }).fetch = jest
+        .fn()
+        .mockResolvedValue({ ok: false, status: 404 } as Response) as typeof fetch;
+
+      const source = {
+        kind: "video" as const,
+        url: "https://example.com/video.mp4",
+        trackingDataUrl: "https://example.com/missing.td",
+      };
+
+      const promise = createCameraKitSource(source);
+      videoElement.dispatchEvent(new Event("canplay"));
+
+      await expect(promise).rejects.toThrow("Unable to load tracking data");
+      // Tracking data is fetched before playback starts, so a failed fetch must not leave the video playing.
+      expect(videoElement.play).not.toHaveBeenCalled();
+    });
   });
 
   describe("createCameraKitSource - Image", () => {

src/internal/sourceUtils.ts

@@ -43,6 +43,7 @@ export async function createCameraKitSource(source: SourceInput): Promise<Source
     return createCameraKitVideoSource({
       videoUrl: source.url,
       autoplay: source.autoplay,
+      trackingDataUrl: source.trackingDataUrl,
     });
   } else if (source.kind === "image") {
     return createCameraKitImageSource({
@@ -105,7 +106,15 @@ async function createCameraStreamSource({
   };
 }
 
-function createCameraKitVideoSource({ videoUrl, autoplay }: { videoUrl: string; autoplay?: boolean }) {
+function createCameraKitVideoSource({
+  videoUrl,
+  autoplay,
+  trackingDataUrl,
+}: {
+  videoUrl: string;
+  autoplay?: boolean;
+  trackingDataUrl?: string;
+}) {
   return new Promise<SourceApplication>((res, rej) => {
     autoplay = autoplay ?? true;
     const videoInput = document.createElement("video");
@@ -118,17 +127,27 @@ function createCameraKitVideoSource({ videoUrl, autoplay }: { videoUrl: string;
     videoInput.addEventListener(
       "canplay",
       async () => {
-        if (autoplay) await videoInput.play();
-        res({
-          cameraKitSource: createVideoSource(videoInput),
-          transform: Transform2D.Identity,
-          inputSize: [videoInput.videoWidth, videoInput.videoHeight],
-          initializedSourceInput: {
-            kind: "video",
-            url: videoUrl,
-            videoElement: videoInput,
-          },
-        });
+        try {
+          // When a tracking-data sidecar is provided, fetch it and hand it to createVideoSource so the
+          // recorded tracking (camera pose, etc.) is replayed against the video instead of live tracking.
+          // Fetch BEFORE starting playback: the looping video must not advance before Camera Kit has the
+          // source, or replay tracking drifts from the video timeline; and a fetch failure then rejects
+          // before any playback has started, so there is no orphaned playing video to clean up.
+          const trackingData = trackingDataUrl ? await fetchTrackingData(trackingDataUrl) : undefined;
+          if (autoplay) await videoInput.play();
+          res({
+            cameraKitSource: createVideoSource(videoInput, trackingData ? { trackingData } : undefined),
+            transform: Transform2D.Identity,
+            inputSize: [videoInput.videoWidth, videoInput.videoHeight],
+            initializedSourceInput: {
+              kind: "video",
+              url: videoUrl,
+              videoElement: videoInput,
+            },
+          });
+        } catch (cause) {
+          rej(cause instanceof Error ? cause : new Error(String(cause)));
+        }
       },
       { once: true },
     );
@@ -146,6 +165,14 @@ function createCameraKitVideoSource({ videoUrl, autoplay }: { videoUrl: string;
   });
 }
 
+async function fetchTrackingData(url: string): Promise<ArrayBuffer> {
+  const response = await fetch(url);
+  if (!response.ok) {
+    throw new Error(`Unable to load tracking data from ${url} (HTTP ${response.status}).`);
+  }
+  return response.arrayBuffer();
+}
+
 function createCameraKitImageSource({ imageUrl }: { imageUrl: string }) {
   return new Promise<SourceApplication>((res, rej) => {
     const imageInput = document.createElement("img");

src/types.ts

@@ -48,6 +48,13 @@ export type VideoSourceInput = {
   kind: "video";
   url: string;
   autoplay?: boolean;
+  /**
+   * Optional URL of a recorded tracking-data buffer to replay against the video (e.g. a Lens Studio
+   * `.td` file). When provided, it is fetched and passed to `createVideoSource` as `trackingData`, so
+   * the lens is driven by the recorded camera pose / tracking instead of live tracking. Useful for
+   * previewing world-facing lenses from a recorded environment.
+   */
+  trackingDataUrl?: string;
 };
 export type ImageSourceInput = { kind: "image"; url: string };