diff --git a/recording/README.md b/recording/README.md index 3269554..226478a 100644 --- a/recording/README.md +++ b/recording/README.md @@ -26,6 +26,13 @@ below). resolution/frame count. Because both are ticked from the same start time at the same fixed rate, frame `i` in the log lines up with frame `i` of the video without post-hoc alignment. +- **Frame extraction** (`frames.py`): slices a session's video into individual + `frame_%06d.png` images (via `ffmpeg -vsync 0`, so no frames are silently dropped or + duplicated during extraction) — the bridge to the Labeling Tool, which works on + frame images rather than video. Cross-checks the extracted frame count against the + manifest's `frame_count` and the input log's line count, and warns if they disagree + — the one place actual capture drift (if the recording PC couldn't sustain 30fps) + would surface, since it's not otherwise possible to detect from the video alone. ## Setup @@ -52,6 +59,9 @@ spelunkai-record record --input-device /dev/input/event3 --output-dir recordings # fixed-length session spelunkai-record record --input-device /dev/input/event3 --duration 120 --name run01 + +# slice a recorded session into individual frame images for labeling +spelunkai-record extract-frames --output-dir recordings --name run01 ``` ## Testing @@ -64,5 +74,10 @@ ffmpeg binary required, so they run anywhere (including WSL): pytest ``` +`test_frames.py` additionally runs a real end-to-end extraction against an +`ffmpeg`-generated synthetic test video (no display or game needed) when `ffmpeg` is +on `PATH`; it's skipped automatically otherwise (e.g. a bare WSL install without +`ffmpeg`). + Actual capture (`VideoCapture`/`KeyboardState` against real hardware) still needs to be validated end-to-end on the real recording PC with Spelunky Classic HD running. diff --git a/recording/src/spelunkai_recording/cli.py b/recording/src/spelunkai_recording/cli.py index a63e80a..341c206 100644 --- a/recording/src/spelunkai_recording/cli.py +++ b/recording/src/spelunkai_recording/cli.py @@ -7,6 +7,7 @@ from pathlib import Path from .capture import CaptureConfig, VideoCapture from .devices import list_keyboard_devices +from .frames import extract_frames from .input_logger import KeyboardState from .session import RecordingSession, session_paths @@ -41,13 +42,36 @@ def main() -> None: subparsers.add_parser("list-devices", help="List input devices that look like keyboards") + extract = subparsers.add_parser( + "extract-frames", + help="Slice a recorded session's video into individual frame images", + ) + extract.add_argument( + "--output-dir", type=Path, default=Path("recordings"), + help="Directory containing the session's video/log/manifest", + ) + extract.add_argument("--name", required=True, help="Session name (file prefix used during recording)") + extract.add_argument( + "--frames-dir", type=Path, default=None, + help="Where to write frame images (default: /_frames)", + ) + args = parser.parse_args() if args.command == "list-devices": - for path, name in list_keyboard_devices(): - print(f"{path}\t{name}") - return + _run_list_devices() + elif args.command == "record": + _run_record(args) + elif args.command == "extract-frames": + _run_extract_frames(args) + +def _run_list_devices() -> None: + for path, name in list_keyboard_devices(): + print(f"{path}\t{name}") + + +def _run_record(args: argparse.Namespace) -> None: session_name = args.name or datetime.now().strftime("%Y%m%d_%H%M%S") paths = session_paths(args.output_dir, session_name) config = CaptureConfig( @@ -72,5 +96,18 @@ def main() -> None: print(f"Stopped. Frames logged: {frame_count if frame_count is not None else 'interrupted'}") +def _run_extract_frames(args: argparse.Namespace) -> None: + paths = session_paths(args.output_dir, args.name) + frames_dir = args.frames_dir or (args.output_dir / f"{args.name}_frames") + result = extract_frames(paths, frames_dir) + print(f"Extracted {result.extracted_frame_count} frames to {result.frames_dir}") + if not result.is_in_sync: + print( + f"WARNING: frame count mismatch (video={result.extracted_frame_count}, " + f"manifest={result.manifest_frame_count}, input_log={result.input_log_line_count}). " + "Video capture may have dropped or duplicated frames relative to the input log." + ) + + if __name__ == "__main__": main() diff --git a/recording/src/spelunkai_recording/frames.py b/recording/src/spelunkai_recording/frames.py new file mode 100644 index 0000000..25c1814 --- /dev/null +++ b/recording/src/spelunkai_recording/frames.py @@ -0,0 +1,64 @@ +"""Extract individual frames from a recorded session's lossless video. + +Bridges the Recording Tool's video output to the Labeling Tool, which +operates on individual frame images rather than video. Frame filenames are +zero-padded by frame index so they line up 1:1 with the `frame` field in +the session's input log. +""" +from __future__ import annotations + +import json +import shutil +import subprocess +from dataclasses import dataclass +from pathlib import Path + +from .session import RecordingPaths + +FRAME_FILENAME_PATTERN = "frame_%06d.png" + + +@dataclass +class FrameExtractionResult: + frames_dir: Path + extracted_frame_count: int + manifest_frame_count: int + input_log_line_count: int + + @property + def is_in_sync(self) -> bool: + return self.extracted_frame_count == self.manifest_frame_count == self.input_log_line_count + + +def extract_frames(paths: RecordingPaths, frames_dir: Path) -> FrameExtractionResult: + if shutil.which("ffmpeg") is None: + raise RuntimeError("ffmpeg not found on PATH") + if not paths.video.exists(): + raise FileNotFoundError(f"video not found: {paths.video}") + + frames_dir.mkdir(parents=True, exist_ok=True) + cmd = [ + "ffmpeg", "-y", + "-i", str(paths.video), + "-vsync", "0", + "-start_number", "0", + str(frames_dir / FRAME_FILENAME_PATTERN), + ] + subprocess.run(cmd, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + return FrameExtractionResult( + frames_dir=frames_dir, + extracted_frame_count=len(list(frames_dir.glob("frame_*.png"))), + manifest_frame_count=_read_manifest_frame_count(paths.manifest), + input_log_line_count=_count_lines(paths.input_log), + ) + + +def _read_manifest_frame_count(manifest_path: Path) -> int: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + return manifest["frame_count"] + + +def _count_lines(path: Path) -> int: + with path.open("r", encoding="utf-8") as f: + return sum(1 for _ in f) diff --git a/recording/tests/test_frames.py b/recording/tests/test_frames.py new file mode 100644 index 0000000..152f4c8 --- /dev/null +++ b/recording/tests/test_frames.py @@ -0,0 +1,67 @@ +import json +import shutil +import subprocess +from pathlib import Path + +import pytest + +from spelunkai_recording.frames import FrameExtractionResult, extract_frames +from spelunkai_recording.session import session_paths + +FFMPEG_AVAILABLE = shutil.which("ffmpeg") is not None + + +def test_is_in_sync_true_when_counts_match(): + result = FrameExtractionResult( + frames_dir=Path("."), extracted_frame_count=3, manifest_frame_count=3, input_log_line_count=3, + ) + assert result.is_in_sync + + +def test_is_in_sync_false_when_counts_differ(): + result = FrameExtractionResult( + frames_dir=Path("."), extracted_frame_count=3, manifest_frame_count=3, input_log_line_count=2, + ) + assert not result.is_in_sync + + +@pytest.mark.skipif(not FFMPEG_AVAILABLE, reason="ffmpeg not installed") +def test_extract_frames_matches_synthetic_video(tmp_path: Path): + paths = session_paths(tmp_path, "synthetic") + frame_count = 5 + + subprocess.run( + [ + "ffmpeg", "-y", + "-f", "lavfi", "-i", "testsrc=size=64x64:rate=10", + "-frames:v", str(frame_count), + "-c:v", "libx264rgb", "-qp", "0", + str(paths.video), + ], + check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + + paths.manifest.write_text( + json.dumps({ + "started_at": "2026-01-01T00:00:00+00:00", + "video_file": paths.video.name, + "input_log_file": paths.input_log.name, + "fps": 10, + "width": 64, + "height": 64, + "frame_count": frame_count, + }), + encoding="utf-8", + ) + + with paths.input_log.open("w", encoding="utf-8") as f: + for i in range(frame_count): + f.write(json.dumps({"frame": i, "t": i / 10, "keys": []}) + "\n") + + frames_dir = tmp_path / "synthetic_frames" + result = extract_frames(paths, frames_dir) + + assert result.is_in_sync + assert result.extracted_frame_count == frame_count + assert (frames_dir / "frame_000000.png").exists() + assert (frames_dir / f"frame_{frame_count - 1:06d}.png").exists()