Cheap Eyes

local file, or a direct media URL
    → look.py
    → Gemini 3.5 Flash          # looks, never decides
    → text description
    → Claude Code               # decides, never looks
    → the component, and the fix

#!/usr/bin/env python3
"""look.py <file|url|files/id> [question...] — cheap eyes: describe a video/image via the Gemini API."""
import base64, functools, json, mimetypes, os, socket, subprocess, sys, time
import urllib.error, urllib.parse, urllib.request

BASE = "https://generativelanguage.googleapis.com"
MODEL = os.environ.get("LOOK_MODEL", "gemini-3.5-flash")
try:
    THINK = int(os.environ.get("LOOK_THINK", "0"))  # 0 = no thinking tokens billed
except ValueError:
    THINK = 0
DEFAULT_Q = ("Describe this recording shot by shot: timestamps, every screen shown, "
             "all on-screen text, every click/scroll/interaction, everything said "
             "aloud, and any moments of hesitation, confusion, or UI glitches.")
INLINE_LIMIT = 19_000_000  # measured: 18.9MB raw (~25MB base64) returns 200 — do not lower this
MIMES = {  # what the Gemini API accepts, plus the aliases Python's mimetypes emits
    "video/mp4", "video/mpeg", "video/mov", "video/quicktime", "video/avi", "video/x-msvideo",
    "video/x-flv", "video/mpg", "video/webm", "video/wmv", "video/x-ms-wmv", "video/3gpp",
    "image/png", "image/jpeg", "image/webp", "image/heic", "image/heif",
    "audio/wav", "audio/x-wav", "audio/mp3", "audio/mpeg", "audio/aac", "audio/x-aac",
    "audio/ogg", "audio/flac", "audio/x-flac", "audio/aiff", "audio/x-aiff", "audio/mp4",
}
HINT = ""  # extra stderr advice printed alongside a 400


def die(msg):
    sys.exit("look.py: " + msg)


@functools.cache
def key():
    k = os.environ.get("GEMINI_API_KEY") or ""
    if not k:
        try:
            k = open(os.path.expanduser("~/.config/gemini/key")).read().strip()
        except OSError:
            die("no GEMINI_API_KEY set and no readable ~/.config/gemini/key")
    return k


def api(path, data, headers, timeout=600, soft400=False):
    req = urllib.request.Request(BASE + path, data=data,
                                 headers={"x-goog-api-key": key(), **headers})
    try:
        with urllib.request.urlopen(req, timeout=timeout) as r:
            return json.load(r)
    except urllib.error.HTTPError as e:
        body = e.read().decode("utf-8", "replace").strip().replace("\n", " ")[:2000]
        if soft400 and e.code == 400 and "Request contains an invalid argument" in body:
            return None  # flash-lite rejecting thinkingBudget=0 — caller retries without it
        print(f"ERROR {e.code}: {body or '(empty body)'}", file=sys.stderr)
        if e.code == 400 and HINT:
            print(HINT, file=sys.stderr)
        if e.code == 404 and "/" in MODEL:
            print(f"LOOK_MODEL={MODEL} has a provider prefix; use a bare id "
                  f"like gemini-3.5-flash", file=sys.stderr)
        sys.exit(1)
    except (socket.timeout, TimeoutError):
        die(f"timed out after {timeout}s on {path}")
    except urllib.error.URLError as e:
        die(f"network error on {path}: {e.reason}")


def mime_of(path):
    m = mimetypes.guess_type(path)[0] or subprocess.run(
        ["file", "-b", "--mime-type", path], capture_output=True, text=True).stdout.strip()
    if m not in MIMES:
        die(f"unsupported media type {m or '(unknown)'} for {path}")
    return m


def upload(path, mime):
    try:
        blob = open(path, "rb").read()
    except OSError as e:
        die(f"cannot read {path}: {e.strerror}")
    if len(blob) < INLINE_LIMIT:
        return {"inline_data": {"mime_type": mime, "data": base64.b64encode(blob).decode()}}
    up = api("/upload/v1beta/files", blob,
             {"X-Goog-Upload-Protocol": "raw", "Content-Type": mime})
    name, uri = up["file"]["name"], up["file"]["uri"]
    for _ in range(90):  # videos need server-side processing before they can be used
        state = api(f"/v1beta/{name}", None, {}, timeout=30)["state"]
        if state == "ACTIVE":
            print(f"uploaded {name} (expires ~48h) — pass this id instead of the file "
                  f"for follow-up questions", file=sys.stderr)
            return {"file_data": {"file_uri": uri, "mime_type": mime}}
        if state == "FAILED":
            die(f"Gemini failed to process {name}")
        time.sleep(2)
    die(f"still processing after 3min — it may finish anyway; re-run: look.py {name} <question>")


def media_part(inp):
    global HINT
    if os.path.exists(inp):  # a local file always wins over URL/id interpretation
        if os.path.isdir(inp):
            die(f"{inp} is a directory (a .cap project is a directory, not a video) — run: "
                f"cap export {inp} -o /tmp/out.mp4 --json, then pass /tmp/out.mp4")
        return upload(inp, mime_of(inp))
    if urllib.parse.urlparse(inp).scheme in ("http", "https"):
        HINT = (f"if Google cannot fetch that host, download it first: "
                f"curl -L -o /tmp/video.mp4 '{inp}' — then pass /tmp/video.mp4")
        return {"file_data": {"file_uri": inp}}
    if inp.startswith("files/"):  # reuse of an earlier upload
        return {"file_data": {"file_uri": f"{BASE}/v1beta/{inp}"}}
    die(f"no such file: {inp}")


def main():
    if len(sys.argv) < 2:
        sys.exit(__doc__)
    inp, q = sys.argv[1], " ".join(sys.argv[2:]) or DEFAULT_Q
    path, hdrs = f"/v1beta/models/{MODEL}:generateContent", {"Content-Type": "application/json"}
    body = {"contents": [{"parts": [{"text": q}, media_part(inp)]}],
            "generationConfig": {"thinkingConfig": {"thinkingBudget": THINK}}}
    resp = api(path, json.dumps(body).encode(), hdrs, soft400=(THINK == 0))
    if resp is None:  # gemini-3.5-flash-lite rejects thinkingBudget=0; retry with its default
        del body["generationConfig"]
        resp = api(path, json.dumps(body).encode(), hdrs)
    cand = (resp.get("candidates") or [{}])[0]
    if cand.get("finishReason", "STOP") != "STOP":
        print(f"WARNING: finishReason={cand['finishReason']}", file=sys.stderr)
    parts = cand.get("content", {}).get("parts") or []
    text = "".join(p["text"] for p in parts if "text" in p)  # 3.x replies are multi-part
    if not text.strip():
        die("no text in response: " + json.dumps(resp)[:2000])
    print(text)


if __name__ == "__main__":
    main()

mkdir -p ~/.config/gemini ~/.claude/scripts
printf '%s' 'YOUR_KEY' > ~/.config/gemini/key && chmod 600 ~/.config/gemini/key
# save the script above as ~/.claude/scripts/look.py, then:
chmod +x ~/.claude/scripts/look.py
~/.claude/scripts/look.py photo.png "what color is this?"

curl -fsSL https://cap.so/install-cli.sh | sh

cap auth status --json              # uploads need a Cap login or CAP_API_KEY
cap record                          # start a recording (or use the desktop app)
cap recordings list --json          # find recordings in the desktop library
cap export <project.cap> -o out.mp4 # a .cap project is a directory, not a video
cap upload out.mp4                  # returns a shareable cap.so link

https://cap.so/api/playlist?videoId=<id>&videoType=mp4

---
name: cap-review
description: Watch a cofounder's cap.so screen recording (or any video/screenshot) with cheap Gemini "eyes" and turn it into concrete UI / onboarding fixes. Use when the user says "cap review", "watch this video", "my cofounder sent a recording", pastes a cap.so link, or points at an .mp4/.mov/.webm showing a UI problem. Pass the whole video to look.py in one call; never scrub frames with ffmpeg unless one specific frame decides something. Claude never ingests the video itself, the script returns a text description to reason over.
---

# cap-review

`~/.claude/scripts/look.py <file|url|files/id> [question]` sends media to the Gemini API at
generativelanguage.googleapis.com and prints text. Key: `GEMINI_API_KEY`, else `~/.config/gemini/key`.
Model `gemini-3.5-flash`; `LOOK_MODEL=gemini-3.5-flash-lite` for bulk runs. Use bare model ids. A
`google/` prefix is OpenRouter syntax and 404s. `gemini-3.6-flash` and `gemini-flash-latest` exist
if you ever upgrade.

`~/.local/bin/caplook [file|url|cap-link|latest] [question]` resolves cap links, `.cap` projects and
the latest library recording (steps 1, 2 and 4 below), then calls look.py.

## Input resolution

0. Screenshot pasted into the conversation: look at it. No look.py round trip.
1. Local file: pass the path.
2. cap.so or cap.link share page: fetch the HTML, take the videoId from the `og:video` meta, pass
   `https://cap.so/api/playlist?videoId=<id>&videoType=mp4` straight to look.py. Gemini fetches that
   URL itself, so nothing downloads. Without `&videoType=mp4` the endpoint returns a 547-byte error blob.
3. `ERROR 400: Cannot fetch content from the provided URL` means the host refused Google's fetcher.
   Run `curl -L -o /tmp/video.mp4 '<url>'` and pass the file.
4. `.cap` project (a directory, not a video): `cap export <project.cap> -o /tmp/out.mp4 --json` first.
5. Follow-up question about a big video: look.py prints `uploaded files/<id>` on stderr. Pass that id
   back instead of uploading again. It expires after about 48 hours.

## Workflow

1. Ask the eyes to describe, never to judge. They have no product context, so "is anything broken or
   stale or wrong here" gets a confident no. They are the witness; you are the judge.
2. One call, whole video. The built-in default question already asks for timestamps, on-screen text,
   clicks, hesitation and everything said aloud, so a narrated recording comes back with the speech
   attributed per timestamp. Write your own only when the review has a specific focus, e.g. "give me
   only the transcript, verbatim" when the words matter more than the screens.
3. Map the description onto the codebase (fff / codegraph) to find the screens and components.
4. Report: timestamp, what the video shows, the code location, the proposed fix. Then fix what the
   user asked for.
5. The one exception to the no-frames rule: when a single frame decides something subtle (pixel
   alignment, exact copy), run `ffmpeg -ss <t> -i video.mp4 -frames:v 1 /tmp/frame.png` and read that
   frame directly.

## Mechanics

Files under 19MB go inline; larger ones upload and wait up to 3 minutes for server-side processing
("still processing" means re-run, it may have finished by then). Thinking is off by default, set
`LOOK_THINK=<n>` to turn it back on.

Audio rides along. A Cap recording's voice track is ingested and transcribed with the video, over a
remote URL as well as inline — no separate audio call, no ffmpeg extraction. Measured on a 118s Cap
recording (`gemini-3.5-flash`, `usageMetadata`): 7,080 input tokens with the audio stripped, 9,991
with it, so ~60 tokens/sec of video plus ~25 tokens/sec of speech, all reported under the VIDEO
modality. A 3-minute narrated recording is ~15k input tokens, a single frame ~1,080. Do not trust
`countTokens` for this — it estimates from container metadata and reports identical AUDIO tokens for
a file whose audio track has been removed.

## Sending a recording out

Preflight `cap auth status --json`. If it reports unauthenticated, tell Matthias to sign in to Cap
Desktop or set `CAP_API_KEY` rather than continuing silently. Then `cap record`,
`cap export`, `cap upload`, and hand back the link. Before recording: deploy the fix, reset the app
or auth state, verify it live, then hit record.

## Cap video workflow (core)

Screen recordings are a core feedback loop with my cofounder, both directions:

- Incoming — a recording (cap.so link or file) showing a UI/onboarding
  problem: use the `cap-review` skill. Cheap eyes via look.py, map findings
  to code, fix.
- Outgoing — whenever I demo something and want my cofounder to see it:
  use the `cap` CLI to record → export → upload and give me the shareable
  link to send. Preflight `cap auth status --json`; if it reports
  unauthenticated, tell me to sign into Cap Desktop or set `CAP_API_KEY`
  instead of continuing.

Zero ceremony: a pasted cap link or video file IS the request. Watch it.
Don't ask permission, don't make me name the skill.

Never hand-transcribe a video or paste frames to the main model — the
look.py eyes exist for that. One exception: a single decisive frame pulled
with `ffmpeg -ss <t> -i video.mp4 -frames:v 1`.