Cheap Eyes

local file, or a direct media URL
    → look.py
    → Gemini 3.5 Flash          # looks, never decides
    → text description
    → Claude Code               # decides, never looks
    → the component, and the fix

#!/usr/bin/env python3
"""cheap eyes: describe a video/image without ever handing it to the model that reasons.

look.py <file|url|files/id> [question...]            — Gemini eyes
look.py --opus [--at t1,t2,...] <file> [question...] — second opinion: frames -> Claude Opus vision
"""
import base64, functools, json, math, mimetypes, os, socket, subprocess, sys, tempfile, time
import urllib.error, urllib.parse, urllib.request

BASE = "https://generativelanguage.googleapis.com"
MODEL = os.environ.get("LOOK_MODEL", "gemini-3.5-flash")
try:
    THINK = int(os.environ.get("LOOK_THINK", "0"))  # 0 = no thinking tokens billed
except ValueError:
    THINK = 0
DEFAULT_Q = ("Describe this recording shot by shot: timestamps, every screen shown, "
             "all on-screen text, every click/scroll/interaction, everything said "
             "aloud, and any moments of hesitation, confusion, or UI glitches.")
INLINE_LIMIT = 19_000_000  # measured: 18.9MB raw (~25MB base64) returns 200 — do not lower this
MIMES = {  # what the Gemini API accepts, plus the aliases Python's mimetypes emits
    "video/mp4", "video/mpeg", "video/mov", "video/quicktime", "video/avi", "video/x-msvideo",
    "video/x-flv", "video/mpg", "video/webm", "video/wmv", "video/x-ms-wmv", "video/3gpp",
    "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif",
    "audio/wav", "audio/x-wav", "audio/mp3", "audio/mpeg", "audio/aac", "audio/x-aac",
    "audio/ogg", "audio/flac", "audio/x-flac", "audio/aiff", "audio/x-aiff", "audio/mp4",
}
HINT = ""  # extra stderr advice printed alongside a 400

ANTHROPIC = "https://api.anthropic.com/v1/messages"
OPUS_MODEL = os.environ.get("LOOK_OPUS_MODEL", "claude-opus-5")
OPUS_FRAMES = 6  # evenly spaced grabs when --at is not given
OPUS_MIMES = {"image/png", "image/jpeg", "image/webp"}  # sent whole; anything else gets frame-grabbed
OPUS_Q = ("These are frames sampled from a screen recording. For each frame, describe exactly "
          "what is on screen: the app/page, all visible text, UI state, dialogs, errors, and "
          "anything unusual. Note differences between frames.")


def die(msg):
    sys.exit("look.py: " + msg)


@functools.cache
def key():
    k = os.environ.get("GEMINI_API_KEY") or ""
    if not k:
        try:
            k = open(os.path.expanduser("~/.config/gemini/key")).read().strip()
        except OSError:
            die("no GEMINI_API_KEY set and no readable ~/.config/gemini/key")
    if not k:
        die("empty API key — set GEMINI_API_KEY or fill ~/.config/gemini/key")
    return k


def api(path, data, headers, timeout=600, soft400=False):
    req = urllib.request.Request(BASE + path, data=data,
                                 headers={"x-goog-api-key": key(), **headers})
    try:
        with urllib.request.urlopen(req, timeout=timeout) as r:
            return json.load(r)
    except urllib.error.HTTPError as e:
        body = e.read().decode("utf-8", "replace").strip().replace("\n", " ")[:2000]
        if soft400 and e.code == 400 and "Request contains an invalid argument" in body:
            return None  # flash-lite rejecting thinkingBudget=0 — caller retries without it
        print(f"ERROR {e.code}: {body or '(empty body)'}", file=sys.stderr)
        if e.code == 400 and HINT:
            print(HINT, file=sys.stderr)
        if e.code == 404 and "/" in MODEL:
            print(f"LOOK_MODEL={MODEL} has a provider prefix; use a bare id "
                  f"like gemini-3.5-flash", file=sys.stderr)
        sys.exit(1)
    except (socket.timeout, TimeoutError):
        die(f"timed out after {timeout}s on {path}")
    except urllib.error.URLError as e:
        die(f"network error on {path}: {e.reason}")


def mime_of(path):
    m = mimetypes.guess_type(path)[0] or subprocess.run(
        ["file", "-b", "--mime-type", path], capture_output=True, text=True).stdout.strip()
    if m not in MIMES:
        die(f"unsupported media type {m or '(unknown)'} for {path}")
    return m


def upload(path, mime):
    try:
        blob = open(path, "rb").read()
    except OSError as e:
        die(f"cannot read {path}: {e.strerror}")
    if len(blob) < INLINE_LIMIT:
        return {"inline_data": {"mime_type": mime, "data": base64.b64encode(blob).decode()}}
    up = api("/upload/v1beta/files", blob,
             {"X-Goog-Upload-Protocol": "raw", "Content-Type": mime})
    name, uri = up["file"]["name"], up["file"]["uri"]
    for _ in range(90):  # videos need server-side processing before they can be used
        state = api(f"/v1beta/{name}", None, {}, timeout=30)["state"]
        if state == "ACTIVE":
            print(f"uploaded {name} (expires ~48h) — pass this id instead of the file "
                  f"for follow-up questions", file=sys.stderr)
            return {"file_data": {"file_uri": uri, "mime_type": mime}}
        if state == "FAILED":
            die(f"Gemini failed to process {name}")
        time.sleep(2)
    die(f"still processing after 3min — it may finish anyway; re-run: look.py {name} <question>")


def media_part(inp):
    global HINT
    if os.path.exists(inp):  # a local file always wins over URL/id interpretation
        if os.path.isdir(inp):
            die(f"{inp} is a directory (a .cap project is a directory, not a video) — run: "
                f"cap export {inp} -o /tmp/out.mp4 --json, then pass /tmp/out.mp4")
        return upload(inp, mime_of(inp))
    if urllib.parse.urlparse(inp).scheme in ("http", "https"):
        HINT = (f"if Google cannot fetch that host, download it first: "
                f"curl -L -o /tmp/video.mp4 '{inp}' — then pass /tmp/video.mp4")
        return {"file_data": {"file_uri": inp}}
    if inp.startswith("files/"):  # reuse of an earlier upload
        return {"file_data": {"file_uri": f"{BASE}/v1beta/{inp}"}}
    die(f"no such file: {inp}")


@functools.cache
def opus_key():
    k = os.environ.get("ANTHROPIC_API_KEY") or ""
    if not k:
        try:
            k = open(os.path.expanduser("~/.config/anthropic/key")).read().strip()
        except OSError:
            die("no ANTHROPIC_API_KEY set and no readable ~/.config/anthropic/key")
    if not k:
        die("empty API key — set ANTHROPIC_API_KEY or fill ~/.config/anthropic/key")
    return k


def ff(cmd, soft=False):
    try:
        p = subprocess.run(cmd, capture_output=True, text=True)
    except FileNotFoundError:
        die(f"{cmd[0]} not found — --opus needs ffmpeg (brew install ffmpeg)")
    if p.returncode:
        if soft:
            return None
        die(f"{cmd[0]} failed: {' '.join((p.stderr or '').split())[-300:] or p.returncode}")
    return p.stdout


def grab(path, at):
    # video-stream duration, not container duration — audio can outrun the last video frame
    dur = ff(["ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries",
              "stream=duration", "-of", "csv=p=0", path]).strip()
    if not dur or dur == "N/A":  # some containers (webm) only carry it at format level
        dur = (ff(["ffprobe", "-v", "error", "-show_entries", "format=duration",
                   "-of", "csv=p=0", path]) or "").strip()
    try:
        dur = float(dur)
    except ValueError:
        die(f"ffprobe read no duration from {path} — is it a video?")
    if at and any(not math.isfinite(t) or t < 0 for t in at):
        die("--at wants finite, non-negative seconds")
    stamps = at or [dur * (i + 0.5) / OPUS_FRAMES for i in range(OPUS_FRAMES)]
    stamps = [min(t, max(dur - 0.1, 0.0)) for t in stamps]  # clamp past-the-end grabs
    frames = []
    with tempfile.TemporaryDirectory(prefix="look-opus-") as tmp:
        for i, t in enumerate(stamps):
            jpg = os.path.join(tmp, f"{i:02d}.jpg")
            out = ff(["ffmpeg", "-v", "error", "-nostdin", "-y", "-ss", f"{t:g}", "-i", path,
                      "-frames:v", "1", "-q:v", "2",  # 1568px long edge caps image tokens per frame
                      "-vf", "scale='min(1568,iw)':'min(1568,ih)':force_original_aspect_ratio=decrease",
                      jpg], soft=True)
            if out is None or not os.path.exists(jpg):
                print(f"look.py: no frame at {t:g}s (duration {dur:.1f}s) — skipping", file=sys.stderr)
                continue
            frames.append((t, open(jpg, "rb").read()))
    if not frames:
        die(f"ffmpeg produced no frames from {path}")
    return frames


def img(mime, blob):
    return {"type": "image", "source": {"type": "base64", "media_type": mime,
                                        "data": base64.b64encode(blob).decode()}}


def ask_opus(content):
    body = json.dumps({"model": OPUS_MODEL, "max_tokens": 16000, "fallbacks": "default",
                       "messages": [{"role": "user", "content": content}]}).encode()
    req = urllib.request.Request(ANTHROPIC, data=body, headers={
        "x-api-key": opus_key(), "anthropic-version": "2023-06-01",
        "content-type": "application/json",
        "anthropic-beta": "server-side-fallback-2026-07-01"})  # refusals retry on a fallback model
    try:
        with urllib.request.urlopen(req, timeout=600) as r:
            return json.load(r)
    except urllib.error.HTTPError as e:
        body = e.read().decode("utf-8", "replace").strip().replace("\n", " ")[:2000]
        print(f"ERROR {e.code}: {body or '(empty body)'}", file=sys.stderr)
        sys.exit(1)
    except (socket.timeout, TimeoutError):
        die("timed out after 600s on /v1/messages")
    except urllib.error.URLError as e:
        die(f"network error on /v1/messages: {e.reason}")


def opus(argv):
    at = None
    if argv and argv[0] in ("-h", "--help"):
        sys.exit(__doc__)
    if argv and argv[0] == "--at":
        if len(argv) < 2:
            die("--at wants seconds, e.g. --at 12,41.5,88")
        try:
            at = [float(s) for s in argv[1].split(",") if s.strip()]
        except ValueError:
            die(f"--at wants seconds as numbers, e.g. --at 12,41.5,88 (got {argv[1]})")
        if not at:
            die("--at wants seconds, e.g. --at 12,41.5,88")
        argv = argv[2:]
    if not argv:
        sys.exit(__doc__)
    inp, q = argv[0], " ".join(argv[1:]) or OPUS_Q
    for a in argv[1:]:
        if a in ("--at", "--opus"):
            die(f"{a} goes before the file: look.py --opus --at 12,41 video.mp4 [question]")
    opus_key()  # fail fast, before any ffmpeg work
    if not os.path.exists(inp):
        die(f"--opus needs a local file, not a URL or files/ id — fetch it first: "
            f"curl -fL -o /tmp/video.mp4 '{inp}' — then pass /tmp/video.mp4")
    if os.path.isdir(inp):
        die(f"{inp} is a directory (a .cap project is a directory, not a video) — run: "
            f"cap export {inp} -o /tmp/out.mp4 --json, then pass /tmp/out.mp4")
    mime = mime_of(inp)
    if mime in OPUS_MIMES:
        content = [img(mime, open(inp, "rb").read())]
    elif not mime.startswith("video/"):
        die(f"--opus wants a video or a png/jpeg/webp image, not {mime}")
    else:
        content = [part for t, blob in grab(inp, at)
                   for part in ({"type": "text", "text": f"Frame at {t:.1f}s:"},
                                img("image/jpeg", blob))]
    resp = ask_opus(content + [{"type": "text", "text": q}])
    if resp.get("stop_reason") == "refusal":
        det = resp.get("stop_details")
        print(f"look.py: {OPUS_MODEL} refused these frames"
              + (f": {json.dumps(det)}" if det else ""), file=sys.stderr)
        sys.exit(1)
    blocks = resp.get("content") or []
    text = "".join(b.get("text", "") for b in blocks if b.get("type") == "text")  # skip thinking
    if not text.strip():
        die("no text in response: " + json.dumps(resp)[:2000])
    print(text)


def main():
    if len(sys.argv) < 2:
        sys.exit(__doc__)
    if sys.argv[1] == "--opus":
        return opus(sys.argv[2:])
    inp, q = sys.argv[1], " ".join(sys.argv[2:]) or DEFAULT_Q
    path, hdrs = f"/v1beta/models/{MODEL}:generateContent", {"Content-Type": "application/json"}
    body = {"contents": [{"parts": [{"text": q}, media_part(inp)]}],
            "generationConfig": {"thinkingConfig": {"thinkingBudget": THINK}}}
    resp = api(path, json.dumps(body).encode(), hdrs, soft400=(THINK == 0))
    if resp is None:  # gemini-3.5-flash-lite rejects thinkingBudget=0; retry with its default
        del body["generationConfig"]
        resp = api(path, json.dumps(body).encode(), hdrs)
    cand = (resp.get("candidates") or [{}])[0]
    if cand.get("finishReason", "STOP") != "STOP":
        print(f"WARNING: finishReason={cand['finishReason']}", file=sys.stderr)
    parts = cand.get("content", {}).get("parts") or []
    text = "".join(p["text"] for p in parts if "text" in p)  # 3.x replies are multi-part
    if not text.strip():
        die("no text in response: " + json.dumps(resp)[:2000])
    print(text)


if __name__ == "__main__":
    main()

mkdir -p ~/.config/gemini ~/.claude/scripts
printf '%s' 'YOUR_KEY' > ~/.config/gemini/key && chmod 600 ~/.config/gemini/key
# save the script above as ~/.claude/scripts/look.py, then:
chmod +x ~/.claude/scripts/look.py
~/.claude/scripts/look.py photo.png "what color is this?"

curl -fsSL https://cap.so/install-cli.sh | sh

cap auth status --json              # uploads need a Cap login or CAP_API_KEY
cap record                          # start a recording (or use the desktop app)
cap recordings list --json          # find recordings in the desktop library
cap export <project.cap> -o out.mp4 # a .cap project is a directory, not a video
cap upload out.mp4                  # returns a shareable cap.so link

https://cap.so/api/playlist?videoId=<id>&videoType=mp4

---
name: cap-review
description: Watch a cofounder's cap.so screen recording (or any video/screenshot) with cheap Gemini "eyes" and turn it into concrete UI / onboarding fixes. Use when the user says "cap review", "watch this video", "my cofounder sent a recording", pastes a cap.so link, or points at an .mp4/.mov/.webm showing a UI problem. Pass the whole video to look.py in one call; never scrub frames with ffmpeg unless one specific frame decides something. Claude never ingests the video itself, the script returns a text description to reason over. When the description is confusing or the video is complicated, pepper in a second opinion from Claude Opus vision via look.py --opus.
---

# cap-review

`~/.claude/scripts/look.py <file|url|files/id> [question]` sends media to the Gemini API at
generativelanguage.googleapis.com and prints text. Key: `GEMINI_API_KEY`, else `~/.config/gemini/key`.
Model `gemini-3.5-flash`; `LOOK_MODEL=gemini-3.5-flash-lite` for bulk runs. Use bare model ids. A
`google/` prefix is OpenRouter syntax and 404s. `gemini-3.6-flash` and `gemini-flash-latest` exist
if you ever upgrade.

`~/.local/bin/caplook [file|url|cap-link|latest] [question]` resolves cap links, `.cap` projects and
the latest library recording (steps 1, 2 and 4 below), then calls look.py.

## Input resolution

0. Screenshot pasted into the conversation: look at it. No look.py round trip.
1. Local file: pass the path.
2. cap.so or cap.link share page: fetch the HTML, take the videoId from the `og:video` meta, pass
   `https://cap.so/api/playlist?videoId=<id>&videoType=mp4` straight to look.py. Gemini fetches that
   URL itself, so nothing downloads. Without `&videoType=mp4` the endpoint returns a 547-byte error blob.
3. `ERROR 400: Cannot fetch content from the provided URL` means the host refused Google's fetcher.
   Run `curl -L -o /tmp/video.mp4 '<url>'` and pass the file.
4. `.cap` project (a directory, not a video): `cap export <project.cap> -o /tmp/out.mp4 --json` first.
5. Follow-up question about a big video: look.py prints `uploaded files/<id>` on stderr. Pass that id
   back instead of uploading again. It expires after about 48 hours.

## Workflow

1. Ask the eyes to describe, never to judge. They have no product context, so "is anything broken or
   stale or wrong here" gets a confident no. They are the witness; you are the judge.
2. One call, whole video. The built-in default question already asks for timestamps, on-screen text,
   clicks, hesitation and everything said aloud, so a narrated recording comes back with the speech
   attributed per timestamp. Write your own only when the review has a specific focus, e.g. "give me
   only the transcript, verbatim" when the words matter more than the screens.
3. Map the description onto the codebase (fff / codegraph) to find the screens and components.
4. Report: timestamp, what the video shows, the code location, the proposed fix. Then fix what the
   user asked for.
5. The one exception to the no-frames rule: when a single frame decides something subtle (pixel
   alignment, exact copy), run `ffmpeg -ss <t> -i video.mp4 -frames:v 1 /tmp/frame.png` and read that
   frame directly. If the frame needs judgment rather than reading — is this layout broken, which of
   these two states is wrong — send it to Opus vision with `--opus --at <t>` instead of pulling it
   into your own context.

## Second opinion: Opus eyes

Gemini stays the default witness. When its description contradicts itself, stays ambiguous, or the
incident is complicated to reconstruct, pepper in a couple of Opus vision calls on real frames rather
than re-asking Gemini the same question.

- `look.py --opus <video> [question]` — 6 evenly spaced frames to `claude-opus-5` in one call.
- `look.py --opus --at 12,41,88 <video> [question]` — frames at the exact timestamps the Gemini
  description made confusing. This is the usual second call: even sampling first, then targeted
  timestamps.

Needs a local file (curl remote videos down first) and `ANTHROPIC_API_KEY`, else
`~/.config/anthropic/key`. The frames go to Anthropic; a safety refusal retries on a fallback model
server-side, and only if that also refuses does the command exit 1. `LOOK_OPUS_MODEL` overrides the
model (default `claude-opus-5`).

Cost: frames are capped at 1568px on the long edge, so a 6-frame call is a few thousand input tokens
at Opus pricing — still cheap, but Gemini remains the bulk-work eyes.

## Mechanics

Files under 19MB go inline; larger ones upload and wait up to 3 minutes for server-side processing
("still processing" means re-run, it may have finished by then). Thinking is off by default, set
`LOOK_THINK=<n>` to turn it back on.

Audio rides along. A Cap recording's voice track is ingested and transcribed with the video, over a
remote URL as well as inline — no separate audio call, no ffmpeg extraction. Measured on a 118s Cap
recording (`gemini-3.5-flash`, `usageMetadata`): 7,080 input tokens with the audio stripped, 9,991
with it, so ~60 tokens/sec of video plus ~25 tokens/sec of speech, all reported under the VIDEO
modality. A 3-minute narrated recording is ~15k input tokens, a single frame ~1,080. Do not trust
`countTokens` for this — it estimates from container metadata and reports identical AUDIO tokens for
a file whose audio track has been removed.

## Sending a recording out

Preflight `cap auth status --json`. If it reports unauthenticated, tell Matthias to sign in to Cap
Desktop or set `CAP_API_KEY` rather than continuing silently. Then `cap record`,
`cap export`, `cap upload`, and hand back the link. Before recording: deploy the fix, reset the app
or auth state, verify it live, then hit record.

## Cap video workflow (core)

Screen recordings are a core feedback loop with my cofounder, both directions:

- Incoming — a recording (cap.so link or file) showing a UI/onboarding
  problem: use the `cap-review` skill. Cheap eyes via look.py, map findings
  to code, fix.
- Outgoing — whenever I demo something and want my cofounder to see it:
  use the `cap` CLI to record → export → upload and give me the shareable
  link to send. Preflight `cap auth status --json`; if it reports
  unauthenticated, tell me to sign into Cap Desktop or set `CAP_API_KEY`
  instead of continuing.

Zero ceremony: a pasted cap link or video file IS the request. Watch it.
Don't ask permission, don't make me name the skill.

Never hand-transcribe a video or paste frames to the main model — the
look.py eyes exist for that. One exception: a single decisive frame pulled
with `ffmpeg -ss <t> -i video.mp4 -frames:v 1`.

# pass 1 — even sampling
look.py --opus recording.mp4
# pass 2 — the timestamps Gemini made confusing
look.py --opus --at 41,42,43 recording.mp4 "what exactly does the error dialog say?"