Add frame extraction bridging recording output to the labeling tool

Slice a recorded session's lossless video into individual frame PNGs
(ffmpeg -vsync 0, no drop/dup) via a new `extract-frames` subcommand,
with frame count cross-checked against the manifest and input log so
any capture-rate drift surfaces immediately instead of silently
misaligning frames and logged input later. Includes a real end-to-end
test against an ffmpeg-generated synthetic video (skipped when ffmpeg
isn't installed, e.g. bare WSL).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

recording/README.md

@@ -26,6 +26,13 @@ below).
   resolution/frame count. Because both are ticked from the same start time at the same
   fixed rate, frame `i` in the log lines up with frame `i` of the video without
   post-hoc alignment.
+- **Frame extraction** (`frames.py`): slices a session's video into individual
+  `frame_%06d.png` images (via `ffmpeg -vsync 0`, so no frames are silently dropped or
+  duplicated during extraction) — the bridge to the Labeling Tool, which works on
+  frame images rather than video. Cross-checks the extracted frame count against the
+  manifest's `frame_count` and the input log's line count, and warns if they disagree
+  — the one place actual capture drift (if the recording PC couldn't sustain 30fps)
+  would surface, since it's not otherwise possible to detect from the video alone.
 
 ## Setup
 
@@ -52,6 +59,9 @@ spelunkai-record record --input-device /dev/input/event3 --output-dir recordings
 
 # fixed-length session
 spelunkai-record record --input-device /dev/input/event3 --duration 120 --name run01
+
+# slice a recorded session into individual frame images for labeling
+spelunkai-record extract-frames --output-dir recordings --name run01
 ```
 
 ## Testing
@@ -64,5 +74,10 @@ ffmpeg binary required, so they run anywhere (including WSL):
 pytest
 ```
 
+`test_frames.py` additionally runs a real end-to-end extraction against an
+`ffmpeg`-generated synthetic test video (no display or game needed) when `ffmpeg` is
+on `PATH`; it's skipped automatically otherwise (e.g. a bare WSL install without
+`ffmpeg`).
+
 Actual capture (`VideoCapture`/`KeyboardState` against real hardware) still needs to be
 validated end-to-end on the real recording PC with Spelunky Classic HD running.

recording/src/spelunkai_recording/cli.py

@@ -7,6 +7,7 @@ from pathlib import Path
 
 from .capture import CaptureConfig, VideoCapture
 from .devices import list_keyboard_devices
+from .frames import extract_frames
 from .input_logger import KeyboardState
 from .session import RecordingSession, session_paths
 
@@ -41,13 +42,36 @@ def main() -> None:
 
     subparsers.add_parser("list-devices", help="List input devices that look like keyboards")
 
+    extract = subparsers.add_parser(
+        "extract-frames",
+        help="Slice a recorded session's video into individual frame images",
+    )
+    extract.add_argument(
+        "--output-dir", type=Path, default=Path("recordings"),
+        help="Directory containing the session's video/log/manifest",
+    )
+    extract.add_argument("--name", required=True, help="Session name (file prefix used during recording)")
+    extract.add_argument(
+        "--frames-dir", type=Path, default=None,
+        help="Where to write frame images (default: <output-dir>/<name>_frames)",
+    )
+
     args = parser.parse_args()
 
     if args.command == "list-devices":
-        for path, name in list_keyboard_devices():
-            print(f"{path}\t{name}")
-        return
+        _run_list_devices()
+    elif args.command == "record":
+        _run_record(args)
+    elif args.command == "extract-frames":
+        _run_extract_frames(args)
 
+
+def _run_list_devices() -> None:
+    for path, name in list_keyboard_devices():
+        print(f"{path}\t{name}")
+
+
+def _run_record(args: argparse.Namespace) -> None:
     session_name = args.name or datetime.now().strftime("%Y%m%d_%H%M%S")
     paths = session_paths(args.output_dir, session_name)
     config = CaptureConfig(
@@ -72,5 +96,18 @@ def main() -> None:
     print(f"Stopped. Frames logged: {frame_count if frame_count is not None else 'interrupted'}")
 
 
+def _run_extract_frames(args: argparse.Namespace) -> None:
+    paths = session_paths(args.output_dir, args.name)
+    frames_dir = args.frames_dir or (args.output_dir / f"{args.name}_frames")
+    result = extract_frames(paths, frames_dir)
+    print(f"Extracted {result.extracted_frame_count} frames to {result.frames_dir}")
+    if not result.is_in_sync:
+        print(
+            f"WARNING: frame count mismatch (video={result.extracted_frame_count}, "
+            f"manifest={result.manifest_frame_count}, input_log={result.input_log_line_count}). "
+            "Video capture may have dropped or duplicated frames relative to the input log."
+        )
+
+
 if __name__ == "__main__":
     main()

recording/src/spelunkai_recording/frames.py

@@ -0,0 +1,64 @@
+"""Extract individual frames from a recorded session's lossless video.
+
+Bridges the Recording Tool's video output to the Labeling Tool, which
+operates on individual frame images rather than video. Frame filenames are
+zero-padded by frame index so they line up 1:1 with the `frame` field in
+the session's input log.
+"""
+from __future__ import annotations
+
+import json
+import shutil
+import subprocess
+from dataclasses import dataclass
+from pathlib import Path
+
+from .session import RecordingPaths
+
+FRAME_FILENAME_PATTERN = "frame_%06d.png"
+
+
+@dataclass
+class FrameExtractionResult:
+    frames_dir: Path
+    extracted_frame_count: int
+    manifest_frame_count: int
+    input_log_line_count: int
+
+    @property
+    def is_in_sync(self) -> bool:
+        return self.extracted_frame_count == self.manifest_frame_count == self.input_log_line_count
+
+
+def extract_frames(paths: RecordingPaths, frames_dir: Path) -> FrameExtractionResult:
+    if shutil.which("ffmpeg") is None:
+        raise RuntimeError("ffmpeg not found on PATH")
+    if not paths.video.exists():
+        raise FileNotFoundError(f"video not found: {paths.video}")
+
+    frames_dir.mkdir(parents=True, exist_ok=True)
+    cmd = [
+        "ffmpeg", "-y",
+        "-i", str(paths.video),
+        "-vsync", "0",
+        "-start_number", "0",
+        str(frames_dir / FRAME_FILENAME_PATTERN),
+    ]
+    subprocess.run(cmd, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+
+    return FrameExtractionResult(
+        frames_dir=frames_dir,
+        extracted_frame_count=len(list(frames_dir.glob("frame_*.png"))),
+        manifest_frame_count=_read_manifest_frame_count(paths.manifest),
+        input_log_line_count=_count_lines(paths.input_log),
+    )
+
+
+def _read_manifest_frame_count(manifest_path: Path) -> int:
+    manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+    return manifest["frame_count"]
+
+
+def _count_lines(path: Path) -> int:
+    with path.open("r", encoding="utf-8") as f:
+        return sum(1 for _ in f)

recording/tests/test_frames.py

@@ -0,0 +1,67 @@
+import json
+import shutil
+import subprocess
+from pathlib import Path
+
+import pytest
+
+from spelunkai_recording.frames import FrameExtractionResult, extract_frames
+from spelunkai_recording.session import session_paths
+
+FFMPEG_AVAILABLE = shutil.which("ffmpeg") is not None
+
+
+def test_is_in_sync_true_when_counts_match():
+    result = FrameExtractionResult(
+        frames_dir=Path("."), extracted_frame_count=3, manifest_frame_count=3, input_log_line_count=3,
+    )
+    assert result.is_in_sync
+
+
+def test_is_in_sync_false_when_counts_differ():
+    result = FrameExtractionResult(
+        frames_dir=Path("."), extracted_frame_count=3, manifest_frame_count=3, input_log_line_count=2,
+    )
+    assert not result.is_in_sync
+
+
+@pytest.mark.skipif(not FFMPEG_AVAILABLE, reason="ffmpeg not installed")
+def test_extract_frames_matches_synthetic_video(tmp_path: Path):
+    paths = session_paths(tmp_path, "synthetic")
+    frame_count = 5
+
+    subprocess.run(
+        [
+            "ffmpeg", "-y",
+            "-f", "lavfi", "-i", "testsrc=size=64x64:rate=10",
+            "-frames:v", str(frame_count),
+            "-c:v", "libx264rgb", "-qp", "0",
+            str(paths.video),
+        ],
+        check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
+    )
+
+    paths.manifest.write_text(
+        json.dumps({
+            "started_at": "2026-01-01T00:00:00+00:00",
+            "video_file": paths.video.name,
+            "input_log_file": paths.input_log.name,
+            "fps": 10,
+            "width": 64,
+            "height": 64,
+            "frame_count": frame_count,
+        }),
+        encoding="utf-8",
+    )
+
+    with paths.input_log.open("w", encoding="utf-8") as f:
+        for i in range(frame_count):
+            f.write(json.dumps({"frame": i, "t": i / 10, "keys": []}) + "\n")
+
+    frames_dir = tmp_path / "synthetic_frames"
+    result = extract_frames(paths, frames_dir)
+
+    assert result.is_in_sync
+    assert result.extracted_frame_count == frame_count
+    assert (frames_dir / "frame_000000.png").exists()
+    assert (frames_dir / f"frame_{frame_count - 1:06d}.png").exists()