dsh: multimodal video/screen-recording analyzer (GLM-5v + ffmpeg frames); fix File/UploadFile imports + ffmpeg in image

dsh/app.py

@@ -16,6 +16,8 @@ import json
 import os
 import base64
 import re
+import shutil
+import subprocess
 import threading
 from pathlib import Path
 from typing import Annotated
@@ -23,7 +25,7 @@ from uuid import uuid4
 from datetime import datetime, timezone
 
 import httpx
-from fastapi import FastAPI, Form, Request
+from fastapi import FastAPI, File, Form, Request, UploadFile
 from fastapi.responses import HTMLResponse, StreamingResponse
 from fastapi.staticfiles import StaticFiles
 from fastapi.templating import Jinja2Templates
@@ -49,6 +51,7 @@ VISION_BASE = os.getenv("DSH_VISION_BASE", "https://openrouter.ai/api/v1")
 VISION_MODEL = os.getenv("DSH_VISION_MODEL", "openai/gpt-5")
 VISION_KEY = os.getenv("DSH_VISION_KEY", "")
 IMAGE_MODEL = os.getenv("DSH_IMAGE_MODEL", "openai/gpt-image-1")
+VIDEO_MODEL = os.getenv("DSH_VIDEO_MODEL", "z-ai/glm-5v-turbo")
 
 # --- per-user conversations (multiple sessions, persist in the history volume) ---
 HISTORY_LIMIT = 60
@@ -150,6 +153,45 @@ async def _complete(messages) -> str:
     return data["choices"][0]["message"]["content"]
 
 
+async def _vision_complete(messages) -> str:
+    """Non-streaming OpenRouter call to the multimodal video/vision model."""
+    key = VISION_KEY
+    base = VISION_BASE
+    headers = {"Authorization": f"Bearer {key}"} if key else {}
+    url = f"{base.rstrip('/')}/chat/completions"
+    payload = {"model": VIDEO_MODEL, "messages": messages, "stream": False, "temperature": 0.3}
+    async with httpx.AsyncClient(timeout=240) as client:
+        r = await client.post(url, json=payload, headers=headers)
+        r.raise_for_status()
+    return r.json()["choices"][0]["message"]["content"]
+
+
+def _extract_frames(video: str, max_frames: int = 6):
+    """Sample up to N JPEG frames from a video with ffmpeg; return data URIs."""
+    outdir = f"/tmp/frames_{uuid4().hex}"
+    os.makedirs(outdir, exist_ok=True)
+    try:
+        dur = subprocess.run(
+            ["ffprobe", "-v", "error", "-show_entries", "format=duration",
+             "-of", "default=nw=1:nk=1", video],
+            capture_output=True, text=True).stdout.strip()
+        dur = float(dur) if dur else 10
+        fps = max(0.1, min(1.0, max_frames / max(dur, 1)))
+        subprocess.run(
+            ["ffmpeg", "-v", "error", "-i", video, "-vf", f"fps={fps}",
+             "-frames:v", str(max_frames), f"{outdir}/f%03d.jpg"],
+            capture_output=True)
+        frames = []
+        for f in sorted(os.listdir(outdir)):
+            p = os.path.join(outdir, f)
+            if os.path.isfile(p) and os.path.getsize(p) > 0:
+                with open(p, "rb") as fh:
+                    frames.append("data:image/jpeg;base64," + base64.b64encode(fh.read()).decode())
+        return frames
+    finally:
+        shutil.rmtree(outdir, ignore_errors=True)
+
+
 @app.post("/api/summarize/web")
 async def summarize_web(url: Annotated[str, Form()]) -> dict:
     """Fetch a URL and summarise its text content via the LLM."""
@@ -298,6 +340,39 @@ async def chat(message: Annotated[str, Form()],
 
 
 
+@app.post("/api/tool/video")
+async def analyze_video(file: Annotated[UploadFile, File()]) -> dict:
+    """Analyze a video / screen recording: extract keyframes, send to the
+    multimodal GLM model, return a description."""
+    if file is None or not file.filename:
+        return {"ok": False, "error": "No video chosen."}
+    if not VISION_KEY:
+        return {"ok": False, "error": "No vision API key configured."}
+    vid = f"/tmp/video_{uuid4().hex}.mp4"
+    try:
+        data = await file.read()
+        if len(data) > 200 * 1024 * 1024:
+            return {"ok": False, "error": "Video too large (max 200 MB)."}
+        with open(vid, "wb") as fh:
+            fh.write(data)
+        frames = _extract_frames(vid, max_frames=6)
+        if not frames:
+            return {"ok": False, "error": "Could not extract frames — is this a video?"}
+        content = [
+            {"type": "text", "text": "Analyze this video/screen recording: describe what is shown, "
+                                     "the main actions, and any visible text or UI. Give a concise bullet summary."}
+        ]
+        for b64 in frames:
+            content.append({"type": "image_url", "image_url": {"url": b64}})
+        summary = await _vision_complete([{"role": "user", "content": content}])
+        return {"ok": True, "frames": len(frames), "summary": summary}
+    except Exception as exc:  # noqa: BLE001
+        return {"ok": False, "error": f"{type(exc).__name__}: {exc}"}
+    finally:
+        if os.path.exists(vid):
+            os.remove(vid)
+
+
 @app.post("/api/tool/image")
 async def generate_image(prompt: Annotated[str, Form()]) -> dict:
     """Generate an image from a prompt via OpenRouter (gpt-image-1); save to workspace."""